Commit 69b3911c authored by Christophe Benz's avatar Christophe Benz

Show pipeline variables, job dotenv report, skipped jobs because of no commit

parent d576692d
Pipeline #166137 passed with stage
in 2 minutes and 42 seconds
......@@ -27,6 +27,7 @@ See https://git.nomics.world/dbnomics-fetchers/documentation/wikis/Setup-CI-jobs
"""
import argparse
import gzip
import io
import logging
import os
......@@ -53,6 +54,7 @@ from gitlab.v4.objects import (
Project,
ProjectJob,
ProjectPipeline,
ProjectPipelineJob,
ProjectPipelineSchedule,
)
from jinja2 import Environment, FileSystemLoader, select_autoescape
......@@ -212,21 +214,24 @@ class Job:
"""A GitLab CI job."""
gitlab_job: ProjectJob
dotenv_report: Dict[str, str] = field(default_factory=dict)
is_fetcher_job = False
def get_job_icon_classes(self):
# Class names are from https://fontawesome.com/icons
job_status = self.get_status()
if job_status == "success":
return "fa-circle text-success"
if job_status == "no_commit":
return "far fa-circle text-success" # empty circle
elif job_status == "success":
return "fas fa-circle text-success" # full circle
elif job_status == "running":
return "fa-clock text-info"
return "fas fa-clock text-info"
elif job_status == "failed":
return "fa-exclamation-circle text-danger"
return "fas fa-exclamation-circle text-danger"
elif job_status == "stuck":
return "fa-exclamation-circle text-dark"
return "fas fa-exclamation-circle text-dark"
elif job_status == "canceled":
return "fa-minus-circle text-dark"
return "fas fa-minus-circle text-dark"
logger.warning("Unsupported job status: %r", job_status)
return ""
......@@ -235,6 +240,8 @@ class Job:
"""Introduce a new status for jobs that never started."""
if not self.gitlab_job.started_at:
return "stuck"
elif self.dotenv_report.get("NO_COMMIT_IN_PREVIOUS_JOB") == "1":
return "no_commit"
return self.gitlab_job.status
......@@ -244,10 +251,12 @@ class FetcherJob(Job):
parsing_error: bool = False
is_fetcher_job = True
git_push: Optional[bool] = None
success_rate: Optional[int] = None
errors_description: Optional[str] = None
# legacy
git_push: Optional[bool] = None
def __post_init__(self):
try:
errors_json = self.get_errors()
......@@ -304,10 +313,10 @@ class FetcherJob(Job):
# and it wouldn't use `fa-*` classes.
assert self.success_rate == 100, self.success_rate
# When data is 100% converted, add a check mark to the circle.
return "fa-check-circle text-success"
return "fas fa-check-circle text-success"
elif self.parsing_error:
# error parsing errors.json file
return "fa-exclamation-triangle"
return "fas fa-exclamation-triangle"
return super().get_job_icon_classes()
......@@ -529,16 +538,43 @@ class DashboardGenerator:
def fetch_pipelines_by_slug(self):
args = self.args
def fetch_dotenv_report(job: ProjectPipelineJob) -> Dict[str, str]:
# Cf https://gitlab.com/gitlab-org/gitlab-ce/issues/49265
job_report_artifact = list(
filter(
lambda artifact: artifact["file_type"] == "dotenv", job.artifacts
)
)
if not job_report_artifact:
return {}
report_url = f"{job.web_url}/artifacts/download?file_type=dotenv"
response = requests.get(report_url, stream=True)
with gzip.open(response.raw) as gzip_f:
report_text = gzip_f.read().decode("utf-8")
report = {}
for line in report_text.splitlines():
variable_name, value = line.split("=", 1)
report[variable_name] = value
return report
def iter_project_pipelines(project):
for pipeline_light in take(
args.max_nb_pipelines_fetched, project.pipelines.list(as_list=False)
):
pipeline = project.pipelines.get(pipeline_light.id)
pipeline_jobs = pipeline.jobs.list()
# Legacy pipelines have only one job named "job".
is_new_pipeline = len(pipeline_jobs) > 1
if is_new_pipeline and (pipeline.ref == MASTER or args.all_branches):
yield [pipeline, pipeline_jobs]
yield (
pipeline,
[
(pipeline_job, fetch_dotenv_report(pipeline_job))
for pipeline_job in pipeline_jobs
],
)
logger.info("Fetch GitLab CI pipelines of all fetchers...")
pipelines_by_slug = {}
......@@ -560,7 +596,7 @@ class DashboardGenerator:
"legacy_pipeline", False
)
if self.args.enable_legacy_pipeline and legacy_pipeline:
jobs_by_slug[provider_slug] = get_fetcher_jobs_by_type(
jobs_by_slug[provider_slug] = legacy_get_fetcher_jobs_by_type(
project, all_branches=self.args.all_branches
)
return jobs_by_slug
......@@ -631,18 +667,18 @@ class DashboardGenerator:
return fetcher_metadata_by_slug
def render(self, jinja2_env):
def job_from_pipeline_job(pipeline_job):
def job_from_pipeline_job(pipeline_job, dotenv_report):
return (
FetcherJob(gitlab_job=pipeline_job)
FetcherJob(gitlab_job=pipeline_job, dotenv_report=dotenv_report)
if pipeline_job.stage in {DOWNLOAD, CONVERT}
else Job(gitlab_job=pipeline_job)
else Job(gitlab_job=pipeline_job, dotenv_report=dotenv_report)
)
def by_stage(pipeline_jobs):
jobs_by_stage = {}
for pipeline_job in pipeline_jobs:
for pipeline_job, dotenv_report in pipeline_jobs:
jobs_by_stage.setdefault(pipeline_job.stage, []).append(
job_from_pipeline_job(pipeline_job)
job_from_pipeline_job(pipeline_job, dotenv_report)
)
return jobs_by_stage
......@@ -702,7 +738,7 @@ class DashboardGenerator:
pickle.dump(self.cache, f)
def extract_fetcher_job_variables(job: ProjectJob):
def legacy_extract_fetcher_job_variables(job: ProjectJob):
"""Return job variables specific to `<provider_slug>-fetcher` projects.
Getting job variables is not supported by GitLab API v4.
......@@ -750,13 +786,13 @@ def extract_provider_slug_from_trace(job, regex):
return {"PROVIDER_SLUG": provider_slug}
def get_fetcher_jobs_by_type(project, all_branches: bool):
def legacy_get_fetcher_jobs_by_type(project, all_branches: bool):
logger.info("Fetch GitLab CI jobs of project %r...", project.name)
fetcher_jobs_by_type: Dict[str, List[FetcherJob]] = {}
for gitlab_job in project.jobs.list():
if not all_branches and gitlab_job.ref != MASTER:
continue
job_variables = extract_fetcher_job_variables(gitlab_job) or {}
job_variables = legacy_extract_fetcher_job_variables(gitlab_job) or {}
job_type = job_variables.get(JOB)
if job_type is None or job_type not in {DOWNLOAD, CONVERT}:
continue
......
......@@ -41,7 +41,7 @@
</a>
{% else %}
<a href="{{ job.gitlab_job.web_url }}" class="mr-1" data-toggle="tooltip" data-html="true" data-placement="auto" title="{{ legacy_job_tooltip(job) }}" style="{{ style }}">
<i class="fas {{ job.get_job_icon_classes() }}"></i>
<i class="{{ job.get_job_icon_classes() }}"></i>
</a>
{% endif %}
{% endmacro %}
......
......@@ -38,9 +38,13 @@
<br>
finished: {{ local_time(job.gitlab_job.finished_at) if job.gitlab_job.finished_at else "?" }}
<br>
{% if job.git_push is not none %}
git push: {{ str(job.git_push).lower() }}
<br>
{% if job.dotenv_report %}
variables:
<ul>
{% for k, v in job.dotenv_report.items() %}
<li>{{ k }}: {{ v }}</li>
{% endfor %}
</ul>
{% endif %}
{% endmacro %}
......@@ -58,6 +62,15 @@
updated: {{ local_time(pipeline.updated_at) if pipeline.updated_at else "?" }}
<br>
duration: {{ humanfriendly.format_timespan(pipeline.duration) if pipeline.duration else "?" }}
{% set variables = pipeline.variables.list() %}
{%if variables %}
variables:
<ul>
{% for variable in variables %}
<li>{{ variable.key }}: {{ variable.value }}</li>
{% endfor %}
</ul>
{% endif %}
{% endmacro %}
{% macro job_icon(job) %}
......@@ -70,7 +83,7 @@
</a>
{% else %}
<a href="{{ job.gitlab_job.web_url }}" class="mr-1" data-toggle="tooltip" data-html="true" data-placement="auto" title="{{ job_tooltip(job) }}">
<i class="fas {{ job.get_job_icon_classes() }}"></i>
<i class="{{ job.get_job_icon_classes() }}"></i>
</a>
{% endif %}
{% endmacro %}
......@@ -85,7 +98,7 @@
<a href="{{ args.grafana_dashboard_url }}?from={{ ts_ms_pair[0] }}&to={{ ts_ms_pair[1] or "" }}" class="mr-4" title="View pipeline metrics">
<i class="fas fa-chart-area"></i>
</a>
<div>
<div class="d-flex">
{% for stage, jobs in jobs_by_stage.items() %}
<span class="mr-4">
<span class="mr-2">{{ stage | capitalize }}</span>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment