Commit 42eb4a49 authored by Bruno Duyé's avatar Bruno Duyé
Browse files

Validation script - errors codes: add a counter for each encoutered codes

parent 099d12c1
......@@ -26,6 +26,7 @@ import argparse
import io
import logging
import sys
from collections import defaultdict
from pathlib import Path
import jsonschema.exceptions
......@@ -34,7 +35,9 @@ from toolz import get, take
import ujson as json
from dbnomics_data_model import storages, validators
from dbnomics_data_model.exceptions import StorageError
from dbnomics_data_model.observations import NOT_AVAILABLE, detect_period_format_strict, value_to_float
from dbnomics_data_model.observations import (NOT_AVAILABLE,
detect_period_format_strict,
value_to_float)
args = None # Will be defined by main().
log = logging.getLogger(__name__)
......@@ -100,18 +103,18 @@ def main():
}))
return -1
errors_codes = set()
errors_codes = defaultdict(int)
log.debug("Validating provider...")
_, provider_errors = validate_provider(storage)
for error in provider_errors:
errors_codes.add(error['error_code'])
errors_codes[error['error_code']] += 1
print(format_error(error))
log.debug("Validating category tree...")
category_tree_errors = validate_category_tree(storage)
for error in category_tree_errors:
errors_codes.add(error['error_code'])
errors_codes[error['error_code']] += 1
print(format_error(error))
log.debug("Validating datasets...")
......@@ -124,17 +127,17 @@ def main():
log.debug("Validating dataset %s (%d/%d) (except its series)...", dataset_code, dataset_index, nb_datasets)
_, dataset_series, dataset_errors = validate_dataset(dataset_dir)
for error in dataset_errors:
errors_codes.add(error['error_code'])
errors_codes[error['error_code']] += 1
print(format_error(error))
log.debug("Validating series of dataset %r...", dataset_code)
series_errors = validate_series(dataset_dir, dataset_series)
for error in series_errors:
errors_codes.add(error['error_code'])
errors_codes[error['error_code']] += 1
print(format_error(error))
if args.format == "text" and errors_codes:
print("Encountered errors codes: {}".format(", ".join(sorted(errors_codes))))
print("Encountered errors codes: {}".format(", ".join(list("{}: {}".format(k, v) for k, v in errors_codes.items()))))
return len(errors_codes)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment