Commit 2edfb51e authored by Bruno Duyé's avatar Bruno Duyé

Minor: rewording: series -> table (one table actually includes more than one series)

parent 4ce6eda7
......@@ -29,7 +29,7 @@ Usage:
Options:
--debug show debug output, and log each API request in 'last_api_result.json' file
--only <datasets_codes> only convert given dataset(s)_code(s). Ex: "--only ABA,BGG"
-l --limit_nb_series <number> limit the number of series to download per dataset
-l --limit_nb_tables <number> limit the number of tables to download per dataset
"""
import logging
......@@ -60,9 +60,9 @@ DATASETS_TO_DOWNLOAD = [
]
API_URLs = {
'series_list': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={dataset_code}&ParameterName=TableName',
'tables_list': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={dataset_code}&ParameterName=TableName',
'series_frequencies': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={dataset_code}&ParameterName=Frequency',
'series_data': 'https://www.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={dataset_code}&TableName={series_name}&Frequency={frequency_code}&Year=ALL',
'series_data': 'https://www.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={dataset_code}&TableName={table_name}&Frequency={frequency_code}&Year=ALL',
}
API_ERRORS_WHITELIST = [
......@@ -98,24 +98,25 @@ def main():
dataset_path = os.path.join(target_dir, dataset_code)
os.mkdir(dataset_path)
log.info("** {}".format(dataset_code))
series_list_url = API_URLs['series_list'].format(api_user_id=api_user_id, dataset_code=dataset_code)
series_list = get_from_api(series_list_url)
tables_list_url = API_URLs['tables_list'].format(api_user_id=api_user_id, dataset_code=dataset_code)
tables_list = get_from_api(tables_list_url)
# Get available frequencies for this dataset
dataset_frequencies = get_from_api(API_URLs['series_frequencies'].format(api_user_id=api_user_id, dataset_code=dataset_code))['ParamValue']
nb_series_donwloaded = 0
for series_info in series_list["ParamValue"]:
series_name = series_info['TableName']
log.debug("* {}".format(series_name))
nb_tables_donwloaded = 0
# For each "table" (set of series)
for table_info in tables_list["ParamValue"]:
table_name = table_info['TableName']
log.debug("* {}".format(table_name))
# Download series data and write series file
for frequency_dict in dataset_frequencies:
frequency_code = frequency_dict['FrequencyID']
url = API_URLs['series_data'].format(api_user_id=api_user_id, dataset_code=dataset_code, series_name=series_name, frequency_code=frequency_code)
series_filepath = os.path.join(dataset_path, slugify(series_name) + '-' + frequency_code + '.json')
url = API_URLs['series_data'].format(api_user_id=api_user_id, dataset_code=dataset_code, table_name=table_name, frequency_code=frequency_code)
table_filepath = os.path.join(dataset_path, slugify(table_name) + '-' + frequency_code + '.json')
json = get_from_api(url, raise_on_errors=False)
if json:
write_json_file(series_filepath, json)
nb_series_donwloaded += 1
if args['--limit_nb_series'] and nb_series_donwloaded == int(args['--limit_nb_series']):
write_json_file(table_filepath, json)
nb_tables_donwloaded += 1
if args['--limit_nb_tables'] and nb_tables_donwloaded == int(args['--limit_nb_tables']):
break
log.info('END')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment