Commit 08591112 authored by Christophe Benz's avatar Christophe Benz

Update GitLab CI config

parent 533ed7a9
Pipeline #1691 failed with stage
in 49 seconds
image: debian:latest
before_script:
- apt update --yes
- apt install --yes git
- apt install --yes python3
- apt install --yes python3-lxml
- apt install --yes python3-pymongo
- apt install --yes python3-pip
- apt install --yes python3-slugify
- export DEBIAN_FRONTEND=noninteractive
- apt -qq --yes update
- apt -qq --yes install git
- apt -qq --yes install python3 python3-pip
# Install ssh-agent if not already installed, it is required by Docker.
- 'which ssh-agent || apt install openssh-client --yes'
- 'which ssh-agent || apt -qq --yes install openssh-client'
# Run ssh-agent (inside the build environment)
# Run ssh-agent (inside the build environment).
- eval $(ssh-agent -s)
# Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
# Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store.
- ssh-add <(echo "$SSH_PRIVATE_KEY")
# For Docker builds disable host key checking. Be aware that by adding that
# Add the SSH keys of remote Git servers, to disable host key checking questions.
- mkdir -p ~/.ssh
- ssh-keyscan -t rsa git.nomics.world >> ~/.ssh/known_hosts
# You can optionally disable host key checking. Be aware that by adding that
# you are suspectible to man-in-the-middle attacks.
# WARNING: Use this only with the Docker executor, if you use it with shell
# you will overwrite your user's SSH config.
- mkdir -p ~/.ssh
- '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
- git config --global push.default simple
- git config --global user.email "ameco-fetcher@db.nomics.world"
- git config --global user.name "Ameco Fetcher"
image: debian:latest
- git config --global user.name "ameco fetcher"
# Display current date to ease post-mortem debugging.
- date
variables:
# Can be "download" or "convert"
JOB: convert
job:
stage: build
only:
- master
tags:
- docker
script:
- if [ "${JOB}" == "download" ]; then ./ameco_fetcher.py --push ameco-source-data; fi
- set -x
- |
if [ "${JOB}" == "download" ]; then
time git clone --quiet --depth=1 git@git.nomics.world:dbnomics-source-data/ameco-source-data.git
cd ameco-source-data
time find -not -path "./.git/*" -not -name ".git" -delete
cd ..
time python3 download.py ameco-source-data
cd ameco-source-data
time git add -A
time git commit -m "New fetch" --quiet || echo "Nothing to commit"
time git push origin master || echo "Nothing to push"
fi
- |
if [ "${JOB}" == "convert" ]; then
git clone https://git.nomics.world/dbnomics/dbnomics-data-model.git
cd dbnomics-data-model
pip3 install -e .
time git clone --quiet --depth=1 https://git.nomics.world/dbnomics-source-data/ameco-source-data.git
time git clone --quiet --depth=1 git@git.nomics.world:dbnomics-json-data/ameco-json-data.git
cd ameco-json-data
time find -not -path "./.git/*" -not -name ".git" -delete
cd ..
./ameco_to_dbnomics.py --push ameco-source-data ameco-json-data
time python3 convert.py ameco-source-data ameco-json-data
cd ameco-json-data
time git add -A
time git commit -m "New conversion" --quiet || echo "Nothing to commit"
time git push origin master || echo "Nothing to push"
fi
stage: build
tags:
- docker
......@@ -54,34 +54,8 @@ def main():
action='store_true',
help='fetch the whole history of the database instead of only the latest version',
)
parser.add_argument(
'-p',
'--push',
action='store_true',
help='push commits to remote Git server',
)
args = parser.parse_args()
if not os.path.exists(args.target_dir):
if args.full:
os.makedirs(args.target_dir)
else:
subprocess.check_call(
['git', 'clone', data_repository_url, args.target_dir],
)
target_git_dir = os.path.join(args.target_dir, '.git')
if os.path.exists(target_git_dir):
assert not args.full
else:
subprocess.check_call(
['git', 'init'],
cwd=args.target_dir,
)
subprocess.check_call(
['git', 'remote', 'add', 'origin', data_repository_url],
cwd=args.target_dir,
)
if args.full:
page_url = 'http://ec.europa.eu/economy_finance/db_indicators/ameco/archive_en.htm'
response = urllib.request.urlopen(page_url)
......@@ -100,15 +74,6 @@ def main():
os.remove(os.path.join(args.target_dir, filename))
with zipfile.ZipFile(io.BytesIO(response.read())) as database_zip:
database_zip.extractall(args.target_dir)
subprocess.check_call(
['git', 'add', '.'],
cwd=args.target_dir,
)
# Note: Don't fail when there is nothing to commit.
subprocess.call(
['git', 'commit', '-m', name],
cwd=args.target_dir,
)
# From https://ec.europa.eu/info/business-economy-euro/indicators-statistics/economic-databases \
# /macro-economic-database-ameco/download-annual-data-set-macro-economic-database-ameco_en
......@@ -118,27 +83,6 @@ def main():
response = urllib.request.urlopen(database_url)
with zipfile.ZipFile(io.BytesIO(response.read())) as database_zip:
database_zip.extractall(args.target_dir)
subprocess.check_call(
['git', 'add', '.'],
cwd=args.target_dir,
)
# Note: Don't fail when there is nothing to commit.
subprocess.call(
['git', 'commit', '-m', 'New fetch'],
cwd=args.target_dir,
)
if args.push:
# Note: Don't fail when there is nothing to push.
if args.full:
subprocess.call(
['git', 'push', '-f', 'origin', 'master'],
cwd=args.target_dir,
)
else:
subprocess.call(
['git', 'push', 'origin', 'master'],
cwd=args.target_dir,
)
return 0
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment