.dockerignore

+*
+!/convert.py
+!/download.py
+!/requirements.txt

.gitlab-ci.yml

 include:
-  - remote: https://git.nomics.world/dbnomics/dbnomics-fetcher-pipeline/-/raw/pipeline-ng/pipelines/fetcher.yml
+  - project: dbnomics/dbnomics-fetcher-pipeline
+    ref: main
+    file: pipelines/fetcher.yml

convert.py

@@ -161,6 +161,8 @@ def series_observations_from_xml_files(xml_files: List[Path]):
 
     # Use pandas to resample extracted observations to daily data
     data = [i[0:2] for i in list(iter_observations(xml_files))]
+    if len(data) == 0:
+        return None
     frequency = list(iter_observations(xml_files))[0][2]
     current_year = list(iter_observations(xml_files))[0][3]
     df = pd.DataFrame(data=data, columns=["period", "value"])
@@ -229,17 +231,18 @@ def convert_agpt_dataset(source_dir: Path, target_dir: Path):
         dim_acc["type"][psr_type] = PSRTYPE_MAPPINGS.get(psr_type, psr_type)
 
         observations = series_observations_from_xml_files(xml_files)
-        series_info_list.append(
-            {
-                "code": f"{country_code}.{psr_type}.D",
-                "dimensions": {
-                    "country": country_code,
-                    "type": psr_type,
-                    "frequency": "D",
-                },
-                "observations": [("PERIOD", "VALUE")] + observations,
-            }
-        )
+        if observations:
+            series_info_list.append(
+                {
+                    "code": f"{country_code}.{psr_type}.D",
+                    "dimensions": {
+                        "country": country_code,
+                        "type": psr_type,
+                        "frequency": "D",
+                    },
+                    "observations": [("PERIOD", "VALUE")] + observations,
+                }
+            )
 
     # dataset.json
     write_json_file(target_dir / "dataset.json", dataset_json_data)
@@ -283,15 +286,16 @@ def convert_atl_dataset(source_dir: Path, target_dir: Path):
         dim_acc["country"][country_code] = country_name
 
         observations = series_observations_from_xml_files(xml_files)
-        series_info_list.append(
-            {
-                "code": f"{country_code}.D",
-                "dimensions": {
-                    "country": country_code,
-                    "frequency": "D",
-                },
-                "observations": [("PERIOD", "VALUE")] + observations,
-            }
+        if observations:
+            series_info_list.append(
+                {
+                    "code": f"{country_code}.D",
+                    "dimensions": {
+                        "country": country_code,
+                        "frequency": "D",
+                    },
+                    "observations": [("PERIOD", "VALUE")] + observations,
+                }
         )
 
     # dataset.json

download.py

@@ -43,7 +43,7 @@ from requests.models import PreparedRequest
 from entsoe import EntsoeRawClient
 from entsoe.mappings import lookup_area, Area, PSRTYPE_MAPPINGS
 
-ENTSOE_API_BASE_URL = "https://transparency.entsoe.eu/api"
+ENTSOE_API_BASE_URL = "https://web-api.tp.entsoe.eu/api"
 API_KEY_NAME = "WEB_SECURITY_TOKEN"
 
 log = logging.getLogger(__name__)
@@ -92,7 +92,7 @@ def download_stream(url: str, filepath: Path, cache=CACHE):
         return
     log.info("downloading %r", filepath.name)
     try:
-        with requests.get(url, stream=True) as req:
+        with requests.get(url, stream=True, timeout=300) as req:
             req.raise_for_status()
             with filepath.open("wb") as f:
                 for chunk in req.iter_content(chunk_size=8192):