...
 
Commits (2)
__pycache__
......@@ -15,10 +15,10 @@ APPENDIX_TO_DOWNLOAD = [
'dimension_label': 'Frequency',
},
'api': {
'datasets_list_url': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=TableName',
'dimensions_values_list_url': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=Frequency',
'datasets_list_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=TableName',
'dimensions_values_list_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=Frequency',
'dimensions_values_keyname': 'FrequencyID',
'series_data_url': 'https://www.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={appendix_code}&TableName={dataset_code}&Frequency={dimension_value_code}&Year=ALL',
'series_data_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={appendix_code}&TableName={dataset_code}&Frequency={dimension_value_code}&Year=ALL',
},
'dimensions': [
{
......@@ -37,10 +37,10 @@ APPENDIX_TO_DOWNLOAD = [
'dimension_label': 'Frequency',
},
'api': {
'datasets_list_url': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=TableName',
'dimensions_values_list_url': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=Frequency',
'datasets_list_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=TableName',
'dimensions_values_list_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=Frequency',
'dimensions_values_keyname': 'FrequencyID',
'series_data_url': 'https://www.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={appendix_code}&TableName={dataset_code}&Frequency={dimension_value_code}&Year=ALL',
'series_data_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={appendix_code}&TableName={dataset_code}&Frequency={dimension_value_code}&Year=ALL',
},
'dimensions': [
{
......@@ -56,8 +56,8 @@ APPENDIX_TO_DOWNLOAD = [
'name': 'Appendix D – Fixed Assets',
'api': {
# "TableID" is deprecated in API doc, but "TableName" is not supported here ...
'datasets_list_url': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=TableID',
'series_data_url': 'https://www.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={appendix_code}&TableID={dataset_code}&Year=X',
'datasets_list_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=TableID',
'series_data_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={appendix_code}&TableID={dataset_code}&Year=X',
},
'dimensions': [
{
......@@ -75,17 +75,17 @@ APPENDIX_TO_DOWNLOAD = [
'code': 'GDPbyIndustry',
'name': 'Appendix F – Gross Domestic Product by Industry (GDPbyIndustry)',
# Special case for GDPbyIndustry (#217): series are presented as a tree => the same industry code have more than one "IndustrYDescription"
'industries_labels_url': "https://bea.gov/api/data/?UserID={api_user_id}&method=GetParameterValues&DataSetName=GDPBYINDUSTRY&ParameterName=INDUSTRY",
'industries_labels_url': "https://apps.bea.gov/api/data/?UserID={api_user_id}&method=GetParameterValues&DataSetName=GDPBYINDUSTRY&ParameterName=INDUSTRY",
'cut_on': {
'dimension_code': 'FREQ',
'dimension_label': 'Frequency',
},
'api': {
# "TableID" is deprecated in API doc, but "TableName" is not supported here ...
'datasets_list_url': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=TableID',
'dimensions_values_list_url': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=Frequency',
'datasets_list_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=TableID',
'dimensions_values_list_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=Frequency',
'dimensions_values_keyname': 'Key',
'series_data_url': 'https://www.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={appendix_code}&TableID={dataset_code}&Frequency={dimension_value_code}&Industry=ALL&Year=ALL',
'series_data_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={appendix_code}&TableID={dataset_code}&Frequency={dimension_value_code}&Industry=ALL&Year=ALL',
},
'dimensions': [
{
......@@ -106,13 +106,13 @@ APPENDIX_TO_DOWNLOAD = [
# 'ITA', # Appendix G – ITA (International Transactions)
# cut sur AreaOrCountry:
# https://www.bea.gov/api/data/?&UserID=BD92A097-6B55-4FB6-A522-F89DF5A452A7&method=GetParameterValues&DataSetName=ITA&ParameterName=AreaOrCountry
# https://apps.bea.gov/api/data/?&UserID=BD92A097-6B55-4FB6-A522-F89DF5A452A7&method=GetParameterValues&DataSetName=ITA&ParameterName=AreaOrCountry
# => TODO
# => only one table ? (no table list)
# 'IIP', # Appendix H – IIP (International Investment Position)
# cut sur TypeOfInvestment
# https://www.bea.gov/api/data/?&UserID=BD92A097-6B55-4FB6-A522-F89DF5A452A7&method=GetParameterValues&DataSetName=IIP&ParameterName=TypeOfInvestment&ResultFormat=xml
# https://apps.bea.gov/api/data/?&UserID=BD92A097-6B55-4FB6-A522-F89DF5A452A7&method=GetParameterValues&DataSetName=IIP&ParameterName=TypeOfInvestment&ResultFormat=xml
# => TODO
# => only one table ? (no table list)
......@@ -124,10 +124,10 @@ APPENDIX_TO_DOWNLOAD = [
'dimension_label': 'Frequency',
},
'api': {
'datasets_list_url': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=TableId',
'dimensions_values_list_url': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=Frequency',
'datasets_list_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=TableId',
'dimensions_values_list_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=Frequency',
'dimensions_values_keyname': 'Key',
'series_data_url': 'https://www.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName=underlyingGDPbyIndustry&Year=ALL&Industry=ALL&tableID={dataset_code}&Frequency={dimension_value_code}'
'series_data_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName=underlyingGDPbyIndustry&Year=ALL&Industry=ALL&tableID={dataset_code}&Frequency={dimension_value_code}'
},
'dimensions': [
{
......@@ -145,10 +145,10 @@ APPENDIX_TO_DOWNLOAD = [
# 'dimension_code': ''
# 'dimension_label':
# 'api': {
# 'datasets_list_url': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=TableName',
# 'dimensions_values_list_url': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=Frequency',
# 'datasets_list_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=TableName',
# 'dimensions_values_list_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=Frequency',
# 'dimensions_values_keyname': 'FrequencyID',
# 'series_data_url': 'https://www.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={appendix_code}&TableName={dataset_code}&Frequency={dimension_value_code}&Year=ALL',
# 'series_data_url': 'https://apps.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={appendix_code}&TableName={dataset_code}&Frequency={dimension_value_code}&Year=ALL',
# }
# => TODO
# => only one table ? (no table list)
......
......@@ -214,6 +214,7 @@ def get_from_api(url, raise_on_errors=True):
log.debug("Tomcat error")
waiting_time = sleeps_times[sleep_time_index]
log.info("Exceeded request quota per minute or server internal error. Waiting {} seconds ...".format(waiting_time))
log.info('Url: {}'.format(url))
time.sleep(sleeps_times[sleep_time_index])
if sleep_time_index < len(sleeps_times) - 1:
sleep_time_index += 1
......