From a36a9d7cd5b116811a9b60b29df1f70a1c498030 Mon Sep 17 00:00:00 2001 From: Zack Galbreath Date: Tue, 6 Sep 2022 14:47:12 -0400 Subject: [PATCH] Mitigation for skipped pipelines (issue #316) --- .github/workflows/custom_docker_builds.yml | 3 + images/gitlab-skipped-pipelines/Dockerfile | 8 ++ images/gitlab-skipped-pipelines/README.md | 22 +++++ .../gitlab-skipped-pipelines/requirements.txt | 1 + .../skipped_pipelines.py | 89 +++++++++++++++++++ k8s/custom/skipped-pipelines/cron-jobs.yaml | 24 +++++ 6 files changed, 147 insertions(+) create mode 100644 images/gitlab-skipped-pipelines/Dockerfile create mode 100644 images/gitlab-skipped-pipelines/README.md create mode 100644 images/gitlab-skipped-pipelines/requirements.txt create mode 100644 images/gitlab-skipped-pipelines/skipped_pipelines.py create mode 100644 k8s/custom/skipped-pipelines/cron-jobs.yaml diff --git a/.github/workflows/custom_docker_builds.yml b/.github/workflows/custom_docker_builds.yml index 565f260a1..9d3795c22 100644 --- a/.github/workflows/custom_docker_builds.yml +++ b/.github/workflows/custom_docker_builds.yml @@ -21,6 +21,7 @@ jobs: - gitops - gitlab-stuckpods - gitlab-clear-pipelines + - gitlab-skipped-pipelines - notary - python-aws-bash include: @@ -44,6 +45,8 @@ jobs: image-tags: ghcr.io/spack/stuckpods:0.0.1 - docker-image: gitlab-clear-pipelines image-tags: ghcr.io/spack/gitlab-clear-pipelines:0.0.1 + - docker-image: gitlab-skipped-pipelines + image-tags: ghcr.io/spack/gitlab-skipped-pipelines:0.0.1 - docker-image: notary image-tags: ghcr.io/spack/notary:latest - docker-image: python-aws-bash diff --git a/images/gitlab-skipped-pipelines/Dockerfile b/images/gitlab-skipped-pipelines/Dockerfile new file mode 100644 index 000000000..48b50733a --- /dev/null +++ b/images/gitlab-skipped-pipelines/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3 + +WORKDIR /scripts/ +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +COPY skipped_pipelines.py ./ + +ENTRYPOINT [ "python", "./skipped_pipelines.py"] diff --git a/images/gitlab-skipped-pipelines/README.md b/images/gitlab-skipped-pipelines/README.md new file mode 100644 index 000000000..f47edf518 --- /dev/null +++ b/images/gitlab-skipped-pipelines/README.md @@ -0,0 +1,22 @@ +# Purpose + +This script searches GitLab for branches that did not have a pipeline run for the most recent commit. After identifying such a commit, this script also schedules a pipeline to run on the affected branch. + +## Background + +This [issue](https://github.com/spack/spack-infrastructure/issues/316) describes the problem. + +## Cause + +Unknown + +## Mitigation + +Install a cron job to run a Python script that implements the following logic: + +``` +for each branch: + get HEAD commit + check if there is a pipeline for this commit: + if not, run one +``` diff --git a/images/gitlab-skipped-pipelines/requirements.txt b/images/gitlab-skipped-pipelines/requirements.txt new file mode 100644 index 000000000..f2293605c --- /dev/null +++ b/images/gitlab-skipped-pipelines/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/images/gitlab-skipped-pipelines/skipped_pipelines.py b/images/gitlab-skipped-pipelines/skipped_pipelines.py new file mode 100644 index 000000000..18ed0eb16 --- /dev/null +++ b/images/gitlab-skipped-pipelines/skipped_pipelines.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +import json +import os +import re +import urllib.parse + +import requests + + +GITLAB_API_URL = "https://gitlab.spack.io/api/v4/projects/2" +AUTH_HEADER = { + "PRIVATE-TOKEN": os.environ.get("GITLAB_TOKEN", None) +} + + +def paginate(query_url): + """Helper method to get all pages of paginated query results""" + results = [] + + while query_url: + resp = requests.get(query_url, headers=AUTH_HEADER) + + if resp.status_code == 401: + print(" !!! Unauthorized to make request, check GITLAB_TOKEN !!!") + return [] + + next_batch = json.loads(resp.content) + + for result in next_batch: + results.append(result) + + if "next" in resp.links: + query_url = resp.links["next"]["url"] + else: + query_url = None + + return results + + +def print_response(resp, padding=''): + """Helper method to print response status code and content""" + print(f"{padding}response code: {resp.status_code}") + print(f"{padding}response value: {resp.text}") + + +def run_new_pipeline(pipeline_ref): + """Given a ref (branch name), run a new pipeline for that ref. If + the branch has already been deleted from gitlab, this will generate + an error and a 400 response, but we probably don't care.""" + enc_ref = urllib.parse.quote_plus(pipeline_ref) + run_url = f"{GITLAB_API_URL}/pipeline?ref={enc_ref}" + print(f" !!!! running new pipeline for {pipeline_ref}") + print_response(requests.post(run_url, headers=AUTH_HEADER), " ") + + +def find_and_run_skipped_pipelines(): + """Query gitlab for all branches. Start a pipeline for any branch whose + HEAD commit does not already have one. + """ + print(f"Attempting to find & fix skipped pipelines") + branches_url = f"{GITLAB_API_URL}/repository/branches" + branches = paginate(branches_url) + print(f"Found {len(branches)} branches") + + regexp = re.compile("pr([0-9]+)") + for branch in branches: + branch_name = branch["name"] + m = regexp.search(branch_name) + if not m: + print(f"Not a PR branch: {branch_name}") + continue + branch_commit = branch["commit"]["id"] + pipelines_url = f"{GITLAB_API_URL}/pipelines?sha={branch_commit}" + pipelines = paginate(pipelines_url) + if len(pipelines) == 0: + run_new_pipeline(branch_name) + else: + print(f"no need to run a new pipeline for {branch_name}") + + +if __name__ == "__main__": + if "GITLAB_TOKEN" not in os.environ: + raise Exception("GITLAB_TOKEN environment is not set") + try: + find_and_run_skipped_pipelines() + except Exception as inst: + print("Caught unhandled exception:") + print(inst) diff --git a/k8s/custom/skipped-pipelines/cron-jobs.yaml b/k8s/custom/skipped-pipelines/cron-jobs.yaml new file mode 100644 index 000000000..98f32c9f0 --- /dev/null +++ b/k8s/custom/skipped-pipelines/cron-jobs.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: skipped-pipelines + namespace: custom +spec: + schedule: "*/30 * * * *" + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: skipped-pipelines + image: ghcr.io/spack/gitlab-skipped-pipelines:0.0.1 + imagePullPolicy: IfNotPresent + env: + - name: GITLAB_TOKEN + valueFrom: + secretKeyRef: + name: gitlab-clear-pipelines + key: gitlab-token + nodeSelector: + spack.io/node-pool: base