Skip to content

Commit

Permalink
Mitigation for skipped pipelines (issue #316)
Browse files Browse the repository at this point in the history
zackgalbreath committed Dec 6, 2022

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent b7757b2 commit a36a9d7
Showing 6 changed files with 147 additions and 0 deletions.
3 changes: 3 additions & 0 deletions .github/workflows/custom_docker_builds.yml
Original file line number Diff line number Diff line change
@@ -21,6 +21,7 @@ jobs:
- gitops
- gitlab-stuckpods
- gitlab-clear-pipelines
- gitlab-skipped-pipelines
- notary
- python-aws-bash
include:
@@ -44,6 +45,8 @@ jobs:
image-tags: ghcr.io/spack/stuckpods:0.0.1
- docker-image: gitlab-clear-pipelines
image-tags: ghcr.io/spack/gitlab-clear-pipelines:0.0.1
- docker-image: gitlab-skipped-pipelines
image-tags: ghcr.io/spack/gitlab-skipped-pipelines:0.0.1
- docker-image: notary
image-tags: ghcr.io/spack/notary:latest
- docker-image: python-aws-bash
8 changes: 8 additions & 0 deletions images/gitlab-skipped-pipelines/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
FROM python:3

WORKDIR /scripts/
COPY requirements.txt ./
RUN pip install --no-cache-dir -r requirements.txt
COPY skipped_pipelines.py ./

ENTRYPOINT [ "python", "./skipped_pipelines.py"]
22 changes: 22 additions & 0 deletions images/gitlab-skipped-pipelines/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Purpose

This script searches GitLab for branches that did not have a pipeline run for the most recent commit. After identifying such a commit, this script also schedules a pipeline to run on the affected branch.

## Background

This [issue](https://github.com/spack/spack-infrastructure/issues/316) describes the problem.

## Cause

Unknown

## Mitigation

Install a cron job to run a Python script that implements the following logic:

```
for each branch:
get HEAD commit
check if there is a pipeline for this commit:
if not, run one
```
1 change: 1 addition & 0 deletions images/gitlab-skipped-pipelines/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
requests
89 changes: 89 additions & 0 deletions images/gitlab-skipped-pipelines/skipped_pipelines.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
#!/usr/bin/env python

import json
import os
import re
import urllib.parse

import requests


GITLAB_API_URL = "https://gitlab.spack.io/api/v4/projects/2"
AUTH_HEADER = {
"PRIVATE-TOKEN": os.environ.get("GITLAB_TOKEN", None)
}


def paginate(query_url):
"""Helper method to get all pages of paginated query results"""
results = []

while query_url:
resp = requests.get(query_url, headers=AUTH_HEADER)

if resp.status_code == 401:
print(" !!! Unauthorized to make request, check GITLAB_TOKEN !!!")
return []

next_batch = json.loads(resp.content)

for result in next_batch:
results.append(result)

if "next" in resp.links:
query_url = resp.links["next"]["url"]
else:
query_url = None

return results


def print_response(resp, padding=''):
"""Helper method to print response status code and content"""
print(f"{padding}response code: {resp.status_code}")
print(f"{padding}response value: {resp.text}")


def run_new_pipeline(pipeline_ref):
"""Given a ref (branch name), run a new pipeline for that ref. If
the branch has already been deleted from gitlab, this will generate
an error and a 400 response, but we probably don't care."""
enc_ref = urllib.parse.quote_plus(pipeline_ref)
run_url = f"{GITLAB_API_URL}/pipeline?ref={enc_ref}"
print(f" !!!! running new pipeline for {pipeline_ref}")
print_response(requests.post(run_url, headers=AUTH_HEADER), " ")


def find_and_run_skipped_pipelines():
"""Query gitlab for all branches. Start a pipeline for any branch whose
HEAD commit does not already have one.
"""
print(f"Attempting to find & fix skipped pipelines")
branches_url = f"{GITLAB_API_URL}/repository/branches"
branches = paginate(branches_url)
print(f"Found {len(branches)} branches")

regexp = re.compile("pr([0-9]+)")
for branch in branches:
branch_name = branch["name"]
m = regexp.search(branch_name)
if not m:
print(f"Not a PR branch: {branch_name}")
continue
branch_commit = branch["commit"]["id"]
pipelines_url = f"{GITLAB_API_URL}/pipelines?sha={branch_commit}"
pipelines = paginate(pipelines_url)
if len(pipelines) == 0:
run_new_pipeline(branch_name)
else:
print(f"no need to run a new pipeline for {branch_name}")


if __name__ == "__main__":
if "GITLAB_TOKEN" not in os.environ:
raise Exception("GITLAB_TOKEN environment is not set")
try:
find_and_run_skipped_pipelines()
except Exception as inst:
print("Caught unhandled exception:")
print(inst)
24 changes: 24 additions & 0 deletions k8s/custom/skipped-pipelines/cron-jobs.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
apiVersion: batch/v1
kind: CronJob
metadata:
name: skipped-pipelines
namespace: custom
spec:
schedule: "*/30 * * * *"
jobTemplate:
spec:
template:
spec:
restartPolicy: Never
containers:
- name: skipped-pipelines
image: ghcr.io/spack/gitlab-skipped-pipelines:0.0.1
imagePullPolicy: IfNotPresent
env:
- name: GITLAB_TOKEN
valueFrom:
secretKeyRef:
name: gitlab-clear-pipelines
key: gitlab-token
nodeSelector:
spack.io/node-pool: base

0 comments on commit a36a9d7

Please sign in to comment.