Commit 7d68802f authored by Pierre Dittgen's avatar Pierre Dittgen
Browse files

Retry if bad zip archive

parent c542be2d
Pipeline #33432 passed with stage
in 20 seconds
......@@ -24,14 +24,16 @@
"""
import argparse
import http.client
import logging
import shutil
import sys
import zipfile
from pathlib import Path
import http.client
import requests
MAX_RETRY = 5
log = logging.Logger(__name__)
......@@ -59,27 +61,53 @@ def die(errmsg):
sys.exit(1)
def get_verified_zip_archive(zip_filepath: Path):
"""Return opened zip_archive or None if an error occurred."""
try:
zip_archive = zipfile.ZipFile(str(zip_filepath))
except zipfile.BadZipFile:
return None
if zip_archive.testzip() is not None:
return None
return zip_archive
def download_and_extract(url, target_dir: Path, dir_name, cache=False):
""" Downloads zip archive and extracts it in a folder """
# Downloads zip
zip_filepath = target_dir / '{}.zip'.format(dir_name)
download_binary_file(url, zip_filepath, cache=cache)
assert zip_filepath.exists()
# Bad zip :-(
zip_archive = zipfile.ZipFile(str(zip_filepath))
if zip_archive.testzip() is not None:
die('Bad zip file: [{}]'.format(str(zip_filepath)))
# Extracts all CSV
csv_dir = target_dir / dir_name
csv_dir.mkdir()
zip_archive.extractall(str(csv_dir))
# And removes zip archive
if not cache:
zip_filepath.unlink()
nb_retry = 0
ok = False
while nb_retry < MAX_RETRY:
nb_retry += 1
# Downloads zip
download_binary_file(url, zip_filepath, cache=cache)
assert zip_filepath.exists()
# Gets zip_archive
zip_archive = get_verified_zip_archive(zip_filepath)
if not zip_archive:
log.error('Bad zip file: [%s], retrying (%d)...', str(zip_filepath), nb_retry)
zip_filepath.unlink()
continue
# Extracts all CSV
csv_dir = target_dir / dir_name
csv_dir.mkdir()
zip_archive.extractall(str(csv_dir))
log.info('Zip [%s] extracted.', str(zip_filepath))
ok = True
# And removes zip archive
if not cache:
zip_filepath.unlink()
break
if not ok:
log.error("Max retry exceeded for %s", str(zip_filepath))
def main():
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment