Skip to content
Snippets Groups Projects
Commit 6ad5087a authored by Pierre Dittgen's avatar Pierre Dittgen
Browse files

Implements download cache for structure files

parent c85f0b8b
No related branches found
No related tags found
No related merge requests found
......@@ -93,7 +93,7 @@ def download_topics(topics_url, topics_codes, target_dir: Path):
zip_data.extractall(str(topic_dir))
def download_structures(structure_url, ds_codes, target_dir: Path):
def download_structures(structure_url, ds_codes, target_dir: Path, cache=True):
""" Downloads each structure information structures folder """
structure_dir = target_dir / 'structures'
if not structure_dir.exists():
......@@ -101,9 +101,13 @@ def download_structures(structure_url, ds_codes, target_dir: Path):
log.info('Downloading structures...')
for code in ds_codes:
structure_filepath = structure_dir / '{}.xml'.format(code)
if cache and structure_filepath.exists():
log.info('already downloaded.')
continue
log.info('Downloading structure [{}]...'.format(code))
r = requests.get(structure_url + code)
structure_filepath = structure_dir / '{}.xml'.format(code)
with structure_filepath.open('wb') as fd:
fd.write(r.content)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment