diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1940b93ce7fac62c272e9f6edd9e181cf074fb18..e249390e7fbd72a69075c36d4b1a90da3b1280e4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ variables: # Can be "download" or "convert" JOB: convert PROVIDER_SLUG: eurostat + FETCHER_ENV_DIR: ~/fetchers-envs/${PROVIDER_SLUG} before_script: # Display info about environment. @@ -12,7 +13,8 @@ before_script: - locale - echo "Running job ${JOB} for provider ${PROVIDER_SLUG}" - # Install fetcher dependencies. + # Enter virtualenv and install fetcher dependencies. + - source ~/${FETCHER_ENV_DIR}/venv/bin/activate - '[ -f requirements.txt ] && pip3 install --requirement requirements.txt' # Run ssh-agent (inside the build environment). @@ -38,34 +40,22 @@ job: - eurostat script: - set -x - - cd ~/jobs/${PROVIDER_SLUG} - | if [ "${JOB}" == "download" ]; then - time git clone --quiet --depth=1 git@git.nomics.world:dbnomics-source-data/${PROVIDER_SLUG}-source-data.git - cd ${PROVIDER_SLUG}-source-data - time find -not -path "./.git/*" -not -name ".git" -delete - cd .. - # Don't trust dates given by Eurostat in table_of_contents.xml - time python3 download.py --full ${PROVIDER_SLUG}-source-data - cd ${PROVIDER_SLUG}-source-data + cd ${FETCHER_ENV_DIR}/${PROVIDER_SLUG}-source-data + git pull + time python3 ${CI_PROJECT_DIR}/download.py . time git add -A time git commit -m "New download" --quiet || true time git push fi - | if [ "${JOB}" == "convert" ]; then - # Test SSH connection - ssh gitlab-runner@dolos.nomics.world "ls" - - time git clone --quiet --depth=1 https://git.nomics.world/dbnomics-source-data/${PROVIDER_SLUG}-source-data.git - time git clone --quiet --depth=1 git@git.nomics.world:dbnomics-json-data/${PROVIDER_SLUG}-json-data.git - - cd ${PROVIDER_SLUG}-json-data - time find -not -path "./.git/*" -not -name ".git" -delete + cd ${FETCHER_ENV_DIR}/${PROVIDER_SLUG}-json-data + git pull cd .. - mkdir sqlite - time python3 convert.py ${PROVIDER_SLUG}-source-data ${PROVIDER_SLUG}-json-data --sqlite-dir sqlite + time python3 ${CI_PROJECT_DIR}/convert.py ${PROVIDER_SLUG}-source-data ${PROVIDER_SLUG}-json-data --sqlite-dir sqlite cd ${PROVIDER_SLUG}-json-data time git add -A @@ -73,5 +63,5 @@ job: time git push cd .. - scp -r sqlite/* gitlab-runner@dolos.nomics.world:sqlite/${PROVIDER_SLUG} + scp sqlite/* gitlab-runner@dolos.nomics.world:sqlite/${PROVIDER_SLUG} fi