Commit 16f3a180 authored by Bruno Duyé's avatar Bruno Duyé

Convert: add --limit_nb_datasets option + fix --only option

parent c3815640
......@@ -31,8 +31,9 @@ target_dir: path of target directory to write datasets & series in DBnomics form
=> all files will be deleted
Options:
--debug show debug output and make some additional tests during the process
--only <appendix_code> only convert given appendix_code
--debug show debug output and make some additional tests during the process
--only <appendix_code> only convert given appendix_code
-l --limit_nb_datasets <number> limit the number of datasets to convert per appendix
"""
......@@ -98,6 +99,8 @@ def main():
]) # List source_dir subdirs
category_tree = []
for appendix_code in appendix_dirnames:
if args['--only'] and not appendix_code in args['--only']:
continue
log.info("* Appendix: {!r}".format(appendix_code))
appendix_path = os.path.join(source_dir, appendix_code)
appendix_name = next(appendix['name'] for appendix in bea_common.APPENDIX_TO_DOWNLOAD if appendix['code'] == appendix_code)
......@@ -106,13 +109,17 @@ def main():
datasets_information = json.load(datasets_json_file)
# Iterate through datasets
category_tree_appendix_datasets = []
nb_datasets_treated = 0
for dataset_dict in datasets_information:
if args['--limit_nb_datasets'] and nb_datasets_treated >= int(args['--limit_nb_datasets']):
break
dataset_code, dataset_name = treat_dataset(dataset_dict, appendix_path)
# Add dataset information to category_tree
category_tree_appendix_datasets.append(toolz.valfilter(lambda e: e, { # filter not provided names
"code": dataset_code,
"name": dataset_name
}))
nb_datasets_treated += 1
category_tree.append({
'code': appendix_code,
'name': appendix_name,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment