...
 
Commits (3)
......@@ -77,7 +77,6 @@ def main():
# Write provider.json
write_json_file(os.path.join(target_dir, 'provider.json'), PROVIDER_JSON)
dataset_series_list = [] # List of series to be written to dataset.json
datasets_list = sorted(f for f in os.listdir(source_dir) if os.path.isfile(
os.path.join(source_dir, f))) # List source_dir sub-dirs
for table_filename in datasets_list:
......@@ -91,10 +90,10 @@ def main():
dataset_dir = os.path.join(target_dir, dataset_code)
os.mkdir(dataset_dir)
# Iterate through series
dataset_series_list = [] # List of series to be written to dataset.json
dimensions_labels = {}
dimensions_values_labels = defaultdict(dict)
nb_series_converted = 1
dataset_series_list.clear()
series_iterator = iter_series(source_filepath)
for series_dict in series_iterator:
series_code = generate_series_code({k: list(v.keys())[0]
......@@ -151,7 +150,7 @@ def iter_series(table_filepath):
...
}
'observations': **iterator** that yields tuples like:
("2017M12", "108.584")
("2017-12", "108.584")
"""
def get_period(observation_dict):
"""
......@@ -161,7 +160,7 @@ def iter_series(table_filepath):
"Anyo": 2017,
"Valor": 108.584
})
"2017M12"
"2017-12"
"""
def parse_roman_quarters(roman_notation):
......@@ -181,11 +180,12 @@ def iter_series(table_filepath):
assert "T3_Periodo" in observation_dict.keys()
if observation_dict["T3_Periodo"].startswith("M"):
# Monthly
return "{}-{}".format(observation_dict["Anyo"], observation_dict["T3_Periodo"][1:])
month_id = observation_dict["T3_Periodo"][1:]
return "{}-{}".format(observation_dict["Anyo"], month_id)
elif observation_dict["T3_Periodo"].startswith("QI"):
# Quarterly
quarter_id = parse_roman_quarters(observation_dict["T3_Periodo"])
return "{}M{}".format(observation_dict["Anyo"], quarter_id)
return "{}-Q{}".format(observation_dict["Anyo"], quarter_id)
else:
raise Exception("Non supported period notation: {!r}".format(observation_dict["T3_Periodo"]))
......
......@@ -86,7 +86,7 @@ def download_table(table_code):
try:
json_response = json.loads(response.content)
except ValueError as e:
print('ERROR: response is not JSON, ignoring table {}!'.format(table_code))
log.info('ERROR: response is not JSON, ignoring table {}!'.format(table_code))
raise e
if isinstance(json_response, list):
# Series list
......