Commit a64346bb authored by Bruno Duyé's avatar Bruno Duyé

category_tree: add appendix information level

parent ad136e78
"""
Common code between download and convert
"""
APPENDIX_TO_DOWNLOAD = [
{
'code': 'NIPA',
'name': 'Appendix B – NIPA (National Income and Product Accounts)'
},
{
'code': 'NIUnderlyingDetail',
'name': 'Appendix C – NIUnderlyingDetail (National Income and Product Accounts)'
}
# 'FixedAssets', # Appendix D – Fixed Assets
# pas de fréquences
# 'datasets_list': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={appendix_code}&ParameterName=TableID',
# 'series_data': 'https://www.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={appendix_code}&TableID={dataset_code}&Year=X',
# dataset_code = dataset_info['TableID']
# 'MNE', # Appendix E – Direct Investment and Multinational Enterprises (MNEs)
# Direct Investment (DI) OR Activities of Multinational Enterprises (AMNE)
# => TODO
# 'GDPbyIndustry', # Appendix F – Gross Domestic Product by Industry (GDPbyIndustry)
# frequency_code = frequency_dict['Key']
# 'series_data': 'https://www.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={appendix_code}&TableID={dataset_code}&Frequency={frequency_code}&Industry=ALL&Year=ALL',
# 'ITA', # Appendix G – ITA (International Transactions)
# cut sur AreaOrCountry:
# https://www.bea.gov/api/data/?&UserID=BD92A097-6B55-4FB6-A522-F89DF5A452A7&method=GetParameterValues&DataSetName=ITA&ParameterName=AreaOrCountry
# => TODO
# 'IIP', # Appendix H – IIP (International Investment Position)
# cut sur TypeOfInvestment
# https://www.bea.gov/api/data/?&UserID=BD92A097-6B55-4FB6-A522-F89DF5A452A7&method=GetParameterValues&DataSetName=IIP&ParameterName=TypeOfInvestment&ResultFormat=xml
# => TODO
# 'UnderlyingGDPbyIndustry', # Appendix L – Underlying Gross Domestic Product by Industry (UnderlyingGDPbyIndustry)
# cut sur frequency
# https://www.bea.gov/api/data/?&UserID=BD92A097-6B55-4FB6-A522-F89DF5A452A7&method=GetData&DataSetName=underlyingGDPbyIndustry&Year=ALL&Industry=ALL&tableID=ALL&Frequency=Q&ResultFormat=xml
# 'IntlServTrade', # Appendix M - IntlServTrade (International Services Trade)
]
......@@ -46,6 +46,7 @@ from babel import numbers
from docopt import docopt
from slugify import slugify
import bea_common
import ujson as json
DATAPACKAGE_JSON = {
......@@ -97,21 +98,25 @@ def main():
category_tree = []
for appendix_code in appendix_dirnames:
log.info("* Appendix: {!r}".format(appendix_code))
# Iterate through datasets
appendix_path = os.path.join(source_dir, appendix_code)
# dataset_json_series_list = [] # to be written into "series" key of dataset.json
# dimensions_values_labels = defaultdict(dict)
# dimension_values_codes_by_labels = {} # cache for labels slugifications to make dimensions_codes
appendix_name = next(appendix['name'] for appendix in bea_common.APPENDIX_TO_DOWNLOAD if appendix['code'] == appendix_code)
# open datasets.json and read datasets information
with open(os.path.join(appendix_path, "datasets.json")) as datasets_json_file:
datasets_information = json.load(datasets_json_file)
# Iterate through datasets
category_tree_appendix_datasets = []
for dataset_dict in datasets_information:
dataset_code, dataset_name = treat_dataset(dataset_dict, appendix_path)
# Add dataset information to category_tree
category_tree.append(toolz.valfilter(lambda e: e, { # filter not provided names
category_tree_appendix_datasets.append(toolz.valfilter(lambda e: e, { # filter not provided names
"code": dataset_code,
"name": dataset_name
}))
category_tree.append({
'code': appendix_code,
'name': appendix_name,
'children': category_tree_appendix_datasets
})
# Write categories_tree.json file
write_json_file(os.path.join(target_dir, 'categories_tree.json'), category_tree)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment