Commit 44a81ac6 authored by Bruno Duyé's avatar Bruno Duyé

Download: separate frequencies in different sources files

parent a181ee2f
......@@ -61,7 +61,8 @@ DATASETS_TO_DOWNLOAD = [
API_URLs = {
'series_list': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={dataset_code}&ParameterName=TableName',
'series_data': 'https://www.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={dataset_code}&TableName={series_name}&Frequency=A,Q&Year=ALL',
'series_frequencies': 'https://bea.gov/api/data/?&UserID={api_user_id}&method=GetParameterValues&DataSetName={dataset_code}&ParameterName=Frequency',
'series_data': 'https://www.bea.gov/api/data/?&UserID={api_user_id}&method=GetData&DataSetName={dataset_code}&TableName={series_name}&Frequency={frequency_code}&Year=ALL',
}
......@@ -95,19 +96,23 @@ def main():
log.info("** {}".format(dataset_code))
series_list_url = API_URLs['series_list'].format(api_user_id=api_user_id, dataset_code=dataset_code)
series_list = get_from_api(series_list_url)
# Get available frequencies for this dataset
dataset_frequencies = get_from_api(API_URLs['series_frequencies'].format(api_user_id=api_user_id, dataset_code=dataset_code))['ParamValue']
nb_series_donwloaded = 0
for series_info in series_list["ParamValue"]:
series_name = series_info['TableName']
log.debug("* {}".format(series_name))
series_filepath = os.path.join(dataset_path, slugify(series_name) + '.json')
url = API_URLs['series_data'].format(api_user_id=api_user_id, dataset_code=dataset_code, series_name=series_name)
# Download series data and write series file
json = get_from_api(url, raise_on_errors=False)
if json:
write_json_file(series_filepath, json)
nb_series_donwloaded += 1
if args['--limit_nb_series'] and nb_series_donwloaded == int(args['--limit_nb_series']):
break
for frequency_dict in dataset_frequencies:
frequency_code = frequency_dict['FrequencyID']
url = API_URLs['series_data'].format(api_user_id=api_user_id, dataset_code=dataset_code, series_name=series_name, frequency_code=frequency_code)
series_filepath = os.path.join(dataset_path, slugify(series_name) + '-' + frequency_code + '.json')
json = get_from_api(url, raise_on_errors=False)
if json:
write_json_file(series_filepath, json)
nb_series_donwloaded += 1
if args['--limit_nb_series'] and nb_series_donwloaded == int(args['--limit_nb_series']):
break
log.info('END')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment