Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • dbnomics-fetchers/entsoe-fetcher
  • Isaac_AFAMBO/entsoe-fetcher
2 results
Show changes
Commits on Source (9)
*
!/convert.py
!/download.py
!/requirements.txt
include: include:
- remote: https://git.nomics.world/dbnomics/dbnomics-fetcher-pipeline/-/raw/pipeline-ng/pipelines/fetcher.yml - project: dbnomics/dbnomics-fetcher-pipeline
ref: main
file: pipelines/fetcher.yml
...@@ -161,6 +161,8 @@ def series_observations_from_xml_files(xml_files: List[Path]): ...@@ -161,6 +161,8 @@ def series_observations_from_xml_files(xml_files: List[Path]):
# Use pandas to resample extracted observations to daily data # Use pandas to resample extracted observations to daily data
data = [i[0:2] for i in list(iter_observations(xml_files))] data = [i[0:2] for i in list(iter_observations(xml_files))]
if len(data) == 0:
return None
frequency = list(iter_observations(xml_files))[0][2] frequency = list(iter_observations(xml_files))[0][2]
current_year = list(iter_observations(xml_files))[0][3] current_year = list(iter_observations(xml_files))[0][3]
df = pd.DataFrame(data=data, columns=["period", "value"]) df = pd.DataFrame(data=data, columns=["period", "value"])
...@@ -229,17 +231,18 @@ def convert_agpt_dataset(source_dir: Path, target_dir: Path): ...@@ -229,17 +231,18 @@ def convert_agpt_dataset(source_dir: Path, target_dir: Path):
dim_acc["type"][psr_type] = PSRTYPE_MAPPINGS.get(psr_type, psr_type) dim_acc["type"][psr_type] = PSRTYPE_MAPPINGS.get(psr_type, psr_type)
observations = series_observations_from_xml_files(xml_files) observations = series_observations_from_xml_files(xml_files)
series_info_list.append( if observations:
{ series_info_list.append(
"code": f"{country_code}.{psr_type}.D", {
"dimensions": { "code": f"{country_code}.{psr_type}.D",
"country": country_code, "dimensions": {
"type": psr_type, "country": country_code,
"frequency": "D", "type": psr_type,
}, "frequency": "D",
"observations": [("PERIOD", "VALUE")] + observations, },
} "observations": [("PERIOD", "VALUE")] + observations,
) }
)
# dataset.json # dataset.json
write_json_file(target_dir / "dataset.json", dataset_json_data) write_json_file(target_dir / "dataset.json", dataset_json_data)
...@@ -283,15 +286,16 @@ def convert_atl_dataset(source_dir: Path, target_dir: Path): ...@@ -283,15 +286,16 @@ def convert_atl_dataset(source_dir: Path, target_dir: Path):
dim_acc["country"][country_code] = country_name dim_acc["country"][country_code] = country_name
observations = series_observations_from_xml_files(xml_files) observations = series_observations_from_xml_files(xml_files)
series_info_list.append( if observations:
{ series_info_list.append(
"code": f"{country_code}.D", {
"dimensions": { "code": f"{country_code}.D",
"country": country_code, "dimensions": {
"frequency": "D", "country": country_code,
}, "frequency": "D",
"observations": [("PERIOD", "VALUE")] + observations, },
} "observations": [("PERIOD", "VALUE")] + observations,
}
) )
# dataset.json # dataset.json
......
...@@ -43,7 +43,7 @@ from requests.models import PreparedRequest ...@@ -43,7 +43,7 @@ from requests.models import PreparedRequest
from entsoe import EntsoeRawClient from entsoe import EntsoeRawClient
from entsoe.mappings import lookup_area, Area, PSRTYPE_MAPPINGS from entsoe.mappings import lookup_area, Area, PSRTYPE_MAPPINGS
ENTSOE_API_BASE_URL = "https://transparency.entsoe.eu/api" ENTSOE_API_BASE_URL = "https://web-api.tp.entsoe.eu/api"
API_KEY_NAME = "WEB_SECURITY_TOKEN" API_KEY_NAME = "WEB_SECURITY_TOKEN"
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
...@@ -92,7 +92,7 @@ def download_stream(url: str, filepath: Path, cache=CACHE): ...@@ -92,7 +92,7 @@ def download_stream(url: str, filepath: Path, cache=CACHE):
return return
log.info("downloading %r", filepath.name) log.info("downloading %r", filepath.name)
try: try:
with requests.get(url, stream=True) as req: with requests.get(url, stream=True, timeout=300) as req:
req.raise_for_status() req.raise_for_status()
with filepath.open("wb") as f: with filepath.open("wb") as f:
for chunk in req.iter_content(chunk_size=8192): for chunk in req.iter_content(chunk_size=8192):
......