Commit b21d7245 authored by Christophe Benz's avatar Christophe Benz
Browse files

Do not fail if dataset.json is missing

parent 73372d95
......@@ -6,7 +6,7 @@ from git import Commit
from ..csv_utils import write_csv
from ..typez import DatasetCode, SeriesCode
from .csv_row_builder import CsvRowBuilder
from .csv_row_builder import CsvRowBuilder, MissingDatasetMetadataError
logger = logging.getLogger(__name__)
......@@ -43,7 +43,11 @@ class GitToCsvImporter:
self, dataset_code: DatasetCode, series_json_iter: Iterable[dict], deleted_series_codes: set[SeriesCode]
):
# deleted_series_codes is ignored because we create a new directory for each commit.
rows_iter, fieldnames = self._csv_row_builder.build_series(dataset_code, series_json_iter)
try:
rows_iter, fieldnames = self._csv_row_builder.build_series(dataset_code, series_json_iter)
except MissingDatasetMetadataError as exc:
logger.error(str(exc))
return
dataset_dir = self._commit_dir / dataset_code
dataset_dir.mkdir(exist_ok=True, parents=True)
with (dataset_dir / "series.csv").open("wt") as fp:
......
......@@ -8,6 +8,10 @@ from ..typez import DatasetCode, DimensionCode, DimensionValueCode
logger = logging.getLogger(__name__)
class MissingDatasetMetadataError(Exception):
pass
class CsvRowBuilder:
def __init__(self):
self._dimension_order: dict[DatasetCode, list[DimensionCode]] = {}
......@@ -54,8 +58,9 @@ class CsvRowBuilder:
def build_series(self, dataset_code: DatasetCode, series_json_iter: Iterable[dict]):
dimension_order = self._dimension_order.get(dataset_code)
if dimension_order is None:
raise ValueError(
f"Dataset {dataset_code!r} must be imported before its series " "to know the order of dimensions"
raise MissingDatasetMetadataError(
f"Dataset {dataset_code!r} must be imported before its series to know the order of dimensions, "
"skipping importing series"
)
def iter_rows():
......
......@@ -5,11 +5,9 @@ import sqlalchemy as sa
from git import Commit
from import_json_data.dolt_client import DoltClient
from sqlalchemy.dialects.mysql import LONGTEXT, TINYTEXT
from toolz import pluck
from ..logging_utils import change_logger_level
from ..typez import DatasetCode, SeriesCode, TableImportMode
from .csv_row_builder import CsvRowBuilder
from .csv_row_builder import CsvRowBuilder, MissingDatasetMetadataError
logger = logging.getLogger(__name__)
......@@ -109,7 +107,11 @@ class GitToDoltImporter:
def replace_series_of_dataset(
self, dataset_code: DatasetCode, series_json_iter: Iterable[dict], deleted_series_codes: set[SeriesCode]
):
rows_iter, fieldnames = self._csv_row_builder.build_series(dataset_code, series_json_iter)
try:
rows_iter, fieldnames = self._csv_row_builder.build_series(dataset_code, series_json_iter)
except MissingDatasetMetadataError as exc:
logger.error(str(exc))
return
self._dolt_client.import_table_from_rows("series", fieldnames, TableImportMode.UPDATE, rows_iter)
if deleted_series_codes:
logger.debug(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment