Commit 5880b75a authored by Bruno Duyé's avatar Bruno Duyé

Download: add FixedAssets appendix

parent 41f73fa9
......@@ -29,10 +29,16 @@ APPENDIX_TO_DOWNLOAD = [
'series_data_url': 'https://www.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={appendix_code}&TableName={dataset_code}&Frequency={dimension_value_code}&Year=ALL',
}
},
# pas de fréquences
# 'datasets_list': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=TableID',
# 'series_data': 'https://www.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={appendix_code}&TableID={dataset_code}&Year=X',
# dataset_code = dataset_info['TableID']
{
'code': 'FixedAssets',
'name': 'Appendix D – Fixed Assets',
'cut_on_dimension': False,
'api': {
# "TableID" is deprecated in API doc, but "TableName" is not supported here ...
'datasets_list_url': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=TableID',
'series_data_url': 'https://www.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={appendix_code}&TableID={dataset_code}&Year=X',
}
}
# 'MNE', # Appendix E – Direct Investment and Multinational Enterprises (MNEs)
# Direct Investment (DI) OR Activities of Multinational Enterprises (AMNE)
# => TODO
......
......@@ -37,6 +37,7 @@ import os
import sys
import time
import toolz
from docopt import docopt
from requests import Session, exceptions
from requests.adapters import HTTPAdapter
......@@ -90,12 +91,12 @@ def main():
datasets_json = [] # Data to be written to datasets.json
# For each dataset
for dataset_info in datasets_list["ParamValue"]:
dataset_code = dataset_info.get('TableName') or dataset_info.get('TableNumber')
dataset_code = dataset_info.get('TableName') or dataset_info.get('TableID') or dataset_info.get('TableNumber')
assert dataset_code, "Error: can't find dataset code in dataset_info (dataset {!r}):\n{!r}".format(appendix_code, dataset_info)
log.debug("* {}".format(dataset_code))
# Download series data and write series file
if args['--limit_nb_datasets'] and nb_tables_downloaded >= int(args['--limit_nb_datasets']):
break
log.debug("* {}".format(dataset_code))
nb_tables_downloaded += 1
dataset_json = [] # dataset information to be written in datasets.json
for dimension_value_code, dataset_observations_json in iter_data(dataset_code, appendix_dict):
......@@ -107,12 +108,12 @@ def main():
'dimension_value_code': dimension_value_code,
'filename': dataset_filename,
})
datasets_json.append({
datasets_json.append(toolz.valfilter(lambda e: e, { # filter None values
'dataset_code': dataset_code,
'dimension_label': appendix_dict['dimension_label'],
'dimension_code': appendix_dict['dimension_code'],
'dimension_label': appendix_dict['dimension_label'] if appendix_dict['cut_on_dimension'] else None,
'dimension_code': appendix_dict['dimension_code'] if appendix_dict['cut_on_dimension'] else None,
'files': dataset_json,
})
}))
# Write datasets.json
write_json_file(os.path.join(appendix_path, "datasets.json"), datasets_json)
......@@ -136,6 +137,12 @@ def iter_data(dataset_code, appendix_dict):
dataset_observations_json = get_from_api(url, raise_on_errors=False)
if dataset_observations_json:
yield dimension_value_code, dataset_observations_json
else:
# No dimension to "cut" on
url = appendix_dict['api']['series_data_url'].format(api_user_id=api_user_id, appendix_code=appendix_code, dataset_code=dataset_code)
dataset_observations_json = get_from_api(url, raise_on_errors=False)
if dataset_observations_json:
yield None, dataset_observations_json
def get_from_api(url, raise_on_errors=True):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment