Commit 3d103e46 authored by Bruno Duyé's avatar Bruno Duyé

Add 'uncategorized' category to let "old" datasets to be displayed in UI

parent 7e6eed2a
......@@ -388,3 +388,21 @@
children:
- name: "1.1 National indices: overall and by sectors"
code: 27145
- name: Uncategorized
children:
- name: "National indices and by Autonomous Community: general and by economic destination of the goods"
code: 3284
- name: "National indices and by section Autonomous Community"
code: 3280
- name: "Export price index export of industrial products / National indices: general and by economic destination of the goods"
code: 3285
- name: "Export price index export of industrial products / National section indices"
code: 3286
- name: "Industrial price indices / National indices: general and by economic destination of the goods"
code: 3289
- name: "Industrial price indices / National section indices"
code: 3293
- name: "Import price index import of industrial products / National indices: general and by economic destination of the goods"
code: 3295
- name: "Import price index import of industrial products / National section indices"
code: 3296
......@@ -94,9 +94,6 @@ def main():
# Write provider.json
write_json_file(target_dir / 'provider.json', PROVIDER_JSON)
# init converted dataset_code accumulator
converted_datasets_codes = set()
# For each dataset in category tree
nb_expected_datasets = 0
for dataset_node, _ in ine_fetcher_common.yield_final_nodes(category_tree):
......@@ -110,6 +107,8 @@ def main():
message = " -> Skipping non existant source file {!r}".format(source_filepath)
log.info(message)
errors_artifact.add_dataset_error(dataset_code, message)
continue
log.info("* Parsing {} ({})".format(source_filepath.name,
humanize.naturalsize(source_filepath.stat().st_size, gnu=True)))
# Create dataset dir
......
......@@ -75,24 +75,6 @@ def main():
# Download
nb_expected_datasets = 0
# Download "old" datasets (see https://git.nomics.world/dbnomics-fetchers/management/issues/526#note_12424)
log.info("* Old datasets")
for table_node in ine_fetcher_common.OTHER_DATASETS_TO_TREAT:
nb_expected_datasets += 1
table_code = table_node['code']
try:
json_answer = download_table(table_code)
except:
error = "Exception while downloading \"old\" dataset {}".format(table_code)
log.exception(error)
errors_artifact.add_dataset_error(table_code, error)
continue
if json_answer is not None:
with (target_dir / (str(table_code) + '.json')).open("wb") as _file:
_file.write(json_answer)
log.info("* New datasets")
# Download "new" datasets
for category_dict in category_tree:
log.info("* {!r}".format(category_dict['name']))
for table_dict, path in ine_fetcher_common.yield_final_nodes(category_dict['children'], path=[category_dict['name']]):
......
......@@ -21,45 +21,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# List of datasets codes to download and convert (some updates appends sometimes) but that are not present
# category tree (see https://git.nomics.world/dbnomics-fetchers/management/issues/526#note_12424)
OTHER_DATASETS_TO_TREAT = [
{
'code': '3284',
'name': 'National indices and by Autonomous Community: general and by economic destination of the goods',
},
{
'code': '3280',
'name': 'National indices and by section Autonomous Community',
},
{
'code': '3285',
'name': 'Export price index export of industrial products / National indices: general and by economic destination of the goods',
},
{
'code': '3286',
'name': 'Export price index export of industrial products / National section indices',
},
{
'code': '3289',
'name': 'Industrial price indices / National indices: general and by economic destination of the goods',
},
{
'code': '3293',
'name': 'Industrial price indices / National section indices',
},
{
'code': '3295',
'name': 'Import price index import of industrial products / National indices: general and by economic destination of the goods',
},
{
'code': '3296',
'name': 'Import price index import of industrial products / National section indices',
},
]
def yield_final_nodes(tree, path=[]):
""" Recursively walk given category tree and yield final nodes and their path.
"""
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment