Commit 9b9a1139 authored by Bruno Duyé's avatar Bruno Duyé

Handle the case when only one dataset is returned by API

parent 1d111882
......@@ -188,13 +188,15 @@ def fetch_series_by_api_link(api_link, max_nb_series=None,
series_dims_by_dataset_code = {} # TODO doc
series_list = []
for series_infos in iter_series(api_link, max_nb_series=max_nb_series):
complete_dataset_code = series_infos['series']['provider_code'] + \
'/' + series_infos['series']['dataset_code'] # ex 'AMECO/ZUTN'
if datasets_dimensions is None:
datasets_dimensions = series_infos['datasets_dimensions']
series_infos = series_infos['series']
series_list.append(series_infos)
complete_dataset_code = series_infos['provider_code'] + '/' + series_infos['dataset_code'] # ex 'AMECO/ZUTN'
datasets_dimensions = series_infos['datasets_dimensions'] if 'datasets_dimensions' in series_infos else {
complete_dataset_code: series_infos['dataset_dimensions']
}
series_list.append(series_infos['series'])
# Store series dimensions information for future use
series_dims_by_dataset_code[complete_dataset_code] = series_infos['dimensions']
series_dims_by_dataset_code[complete_dataset_code] = series_infos['series']['dimensions']
if len(series_list) == 0:
return pd.DataFrame()
......@@ -317,27 +319,44 @@ def iter_filtered_series(series_list, filters, apply_endpoint_url):
def iter_series(api_link, max_nb_series=None):
# """Iterate through series.docs returned by API"""
# Returns dicts of datasets dimensions and series.
# - dataset_dimensions don't change between calls
# - series is the current series
# Example: like:
# {
# 'dataset_dimensions': {
# "AMECO/ZUTN": {
# "code": "ZUTN",
# "converted_at": "2019-05-08T02:51:04Z",
# "dimensions_codes_order": ["freq", "unit", "geo" ...],
# ...
# },
# "CEPII/CHELEM-TRADE-GTAP": {
# "code": "CHELEM-TRADE-GTAP",
# "converted_at": "2019-01-29T15:53:30Z",
# "dimensions_codes_order": ["exporter", "importer", "secgroup", ...],
# ...
# },
# 'series':
# }
"""Iterate through series.docs returned by API
Returns dicts of dataset(s) dimensions and series.
The answer can have a key 'dataset_dimensions' if only one dataset is returned by API, or 'datasets_dimensions' if
more than one dataset is returned.
- datasets_dimensions or dataset_dimensions don't change between calls
- series is the current series
Example:
{
'datasets_dimensions': {
"AMECO/ZUTN": {
"code": "ZUTN",
"converted_at": "2019-05-08T02:51:04Z",
"dimensions_codes_order": ["freq", "unit", "geo" ...],
...
},
"CEPII/CHELEM-TRADE-GTAP": {
"code": "CHELEM-TRADE-GTAP",
"converted_at": "2019-01-29T15:53:30Z",
"dimensions_codes_order": ["exporter", "importer", "secgroup", ...],
...
},
'series':
}
"""
def yield_series(series, response_json):
"""Handle the cases of one-dataset and multi-datasets answer from API"""
assert 'datasets' in response_json or 'dataset' in response_json
if 'datasets' in response_json:
# Multi-datasets answer
datasets_dimensions_dict = {'datasets_dimensions': response_json['datasets']}
else:
# Mono-dataset answer
datasets_dimensions_dict = {'dataset_dimensions': response_json['dataset']}
yield {
'series': series,
**datasets_dimensions_dict
}
total_nb_series = 0
while True:
......@@ -366,18 +385,12 @@ def iter_series(api_link, max_nb_series=None):
# Do not respond more series than the asked max_nb_series.
nb_remaining_series = page_nb_series - (total_nb_series - max_nb_series)
for series in series_page['docs'][:nb_remaining_series]:
yield {
'datasets_dimensions': response_json['datasets'],
'series': series
}
yield from yield_series(series, response_json)
break
# If user didn't asked for a maximum number of series
for series in series_page['docs']:
yield {
'datasets_dimensions': response_json['datasets'],
'series': series
}
yield from yield_series(series, response_json)
# Stop if we downloaded all the series.
assert total_nb_series <= num_found, (total_nb_series, num_found) # Can't download more series than num_found.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment