Commit 7c862b0c authored by Christophe Benz's avatar Christophe Benz
Browse files

Remove cache

parent 27114dd4
Pipeline #111984 failed with stage
in 13 seconds
......@@ -43,8 +43,7 @@ job:
cd ${PROVIDER_SLUG}-source-data
time find -not -path "./.git/*" -not -name ".git" -delete
cd ..
mkdir zip_dir
time python3 download.py ${PROVIDER_SLUG}-source-data --cache-dir zip_dir
time python3 download.py ${PROVIDER_SLUG}-source-data
cd ${PROVIDER_SLUG}-source-data
time git add --ignore-removal .
time git commit -m "New download" --quiet || true
......@@ -63,6 +62,3 @@ job:
time git commit -m "New conversion" --quiet || true
time git push
fi
artifacts:
paths:
- zip_dir/*
......@@ -36,13 +36,10 @@ import requests
log = logging.Logger(__name__)
def download_binary_file(url, file_path: Path, cache=False):
def download_binary_file(url, file_path: Path):
""" Download url into binary file """
log.debug("Downloading %s... ", file_path.name)
if cache and file_path.exists():
log.debug('-> cached.')
return
req = requests.get(url, stream=True)
with file_path.open('wb') as fout:
......@@ -51,11 +48,11 @@ def download_binary_file(url, file_path: Path, cache=False):
log.debug('-> done.')
def download_and_extract(url, target_dir: Path, dir_name, cache_dir: Path = None):
def download_and_extract(url, target_dir: Path, dir_name):
""" Downloads zip archive and extracts it in a folder """
zip_filepath = (cache_dir or target_dir) / '{}.zip'.format(dir_name)
zip_filepath = target_dir / '{}.zip'.format(dir_name)
download_binary_file(url, zip_filepath, cache=cache_dir is not None)
download_binary_file(url, zip_filepath)
# Extracts all CSV
csv_dir = target_dir / dir_name
......@@ -65,8 +62,7 @@ def download_and_extract(url, target_dir: Path, dir_name, cache_dir: Path = None
log.info('Zip [%s] extracted.', str(zip_filepath))
# And removes zip archive
if cache_dir is None:
zip_filepath.unlink()
zip_filepath.unlink()
def main():
......@@ -74,7 +70,6 @@ def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('target_dir', type=Path, help='path of target directory')
parser.add_argument('--cache-dir', type=Path, help='path of cache directory')
parser.add_argument('--debug-http', action='store_true', help='display http.client debug messages')
parser.add_argument('--log', default='WARNING', help='level of logging messages')
args = parser.parse_args()
......@@ -98,9 +93,9 @@ def main():
log.info('Downloading meti data')
download_and_extract('http://www.meti.go.jp/english/statistics/tyo/iip/csv/b2015_g1e.zip',
target_dir, 'b2015_g1e', cache_dir=args.cache_dir)
target_dir, 'b2015_g1e')
download_and_extract('http://www.meti.go.jp/english/statistics/tyo/sanzi/csv/b2010_ke.zip',
target_dir, 'b2010_ke', cache_dir=args.cache_dir)
target_dir, 'b2010_ke')
if __name__ == '__main__':
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment