Commit d80a8b70 authored by Christophe Benz's avatar Christophe Benz

Keep ZIP files as artifacts

parent 74ceb46c
Pipeline #102060 passed with stage
in 23 seconds
......@@ -43,7 +43,8 @@ job:
cd ${PROVIDER_SLUG}-source-data
time find -not -path "./.git/*" -not -name ".git" -delete
cd ..
time python3 download.py ${PROVIDER_SLUG}-source-data
mkdir zip_dir
time python3 download.py ${PROVIDER_SLUG}-source-data --cache-dir zip_dir
cd ${PROVIDER_SLUG}-source-data
time git add --ignore-removal .
time git commit -m "New download" --quiet || true
......@@ -62,3 +63,6 @@ job:
time git commit -m "New conversion" --quiet || true
time git push
fi
artifacts:
paths:
- zip_dir/*
......@@ -72,10 +72,10 @@ def get_verified_zip_archive(zip_filepath: Path):
return zip_archive
def download_and_extract(url, target_dir: Path, dir_name, cache=False):
def download_and_extract(url, target_dir: Path, dir_name, cache_dir: Path = None):
""" Downloads zip archive and extracts it in a folder """
zip_filepath = target_dir / '{}.zip'.format(dir_name)
zip_filepath = (cache_dir or target_dir) / '{}.zip'.format(dir_name)
nb_retry = 0
ok = False
......@@ -83,7 +83,7 @@ def download_and_extract(url, target_dir: Path, dir_name, cache=False):
nb_retry += 1
# Downloads zip
download_binary_file(url, zip_filepath, cache=cache)
download_binary_file(url, zip_filepath, cache=cache_dir is not None)
assert zip_filepath.exists()
# Gets zip_archive
......@@ -101,7 +101,7 @@ def download_and_extract(url, target_dir: Path, dir_name, cache=False):
ok = True
# And removes zip archive
if not cache:
if cache_dir is None:
zip_filepath.unlink()
break
......@@ -115,6 +115,7 @@ def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('target_dir', type=Path, help='path of target directory')
parser.add_argument('--cache-dir', type=Path, help='path of cache directory')
parser.add_argument('--debug-http', action='store_true', help='display http.client debug messages')
parser.add_argument('--log', default='WARNING', help='level of logging messages')
args = parser.parse_args()
......@@ -138,9 +139,9 @@ def main():
log.info('Downloading meti data')
download_and_extract('http://www.meti.go.jp/english/statistics/tyo/iip/csv/b2010_g1e.zip',
target_dir, 'b2010_g1e')
target_dir, 'b2010_g1e', cache_dir=args.cache_dir)
download_and_extract('http://www.meti.go.jp/english/statistics/tyo/sanzi/csv/b2010_ke.zip',
target_dir, 'b2010_ke')
target_dir, 'b2010_ke', cache_dir=args.cache_dir)
if __name__ == '__main__':
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment