Commit e50ac397 authored by Bruno Duyé's avatar Bruno Duyé

.gitlab-ci.yml update

parent 634738d0
Pipeline #55083 passed with stage
in 72 minutes and 51 seconds
......@@ -11,8 +11,6 @@ before_script:
- locale
- echo "Running job ${JOB} for provider ${PROVIDER_SLUG}"
- apt update
# Install fetcher dependencies.
- '[ -f requirements.txt ] && pip3 install --requirement requirements.txt'
......@@ -40,7 +38,6 @@ job:
- set -x
- |
if [ "${JOB}" == "download" ]; then
apt install chromium --yes
time git clone --quiet --depth=1 git@git.nomics.world:dbnomics-source-data/${PROVIDER_SLUG}-source-data.git
cd ${PROVIDER_SLUG}-source-data
time find -not -path "./.git/*" -not -name ".git" -delete
......@@ -50,17 +47,28 @@ job:
time git add -A
time git commit -m "New download" --quiet || true
time git push
# Move errors.json file, if present; to be kept as artifact
[ -f errors.json ] && mv errors.json .. || true
fi
- |
if [ "${JOB}" == "convert" ]; then
time git clone --quiet --depth=1 https://git.nomics.world/dbnomics-source-data/${PROVIDER_SLUG}-source-data.git
time git clone --quiet --depth=1 git@git.nomics.world:dbnomics-json-data/${PROVIDER_SLUG}-json-data.git
cd ${PROVIDER_SLUG}-json-data
# If not on master, checkout corresponding branch on json-data
git checkout -B ${CI_COMMIT_REF_NAME}
# Delete all existing files
time find -not -path "./.git/*" -not -name ".git" -delete
cd ..
time python3 convert.py ${PROVIDER_SLUG}-source-data ${PROVIDER_SLUG}-json-data
cd ${PROVIDER_SLUG}-json-data
time git add -A
time git commit -m "New conversion" --quiet || true
time git push
fi
\ No newline at end of file
# Commit conversion result, ignoring absent datasets
time git add --ignore-removal .
time git commit -m "New conversion..." -m "from source-data $(git -C ../${PROVIDER_SLUG}-source-data rev-parse HEAD)" --quiet || true
time git push origin ${CI_COMMIT_REF_NAME}
# Move errors.json file, if present; to be kept as artifact
[ -f errors.json ] && mv errors.json .. || true
fi
artifacts:
paths:
- errors.json
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment