-
Notifications
You must be signed in to change notification settings - Fork 20
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Mitigation for skipped pipelines (issue #316)
1 parent
b7757b2
commit a36a9d7
Showing
6 changed files
with
147 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
FROM python:3 | ||
|
||
WORKDIR /scripts/ | ||
COPY requirements.txt ./ | ||
RUN pip install --no-cache-dir -r requirements.txt | ||
COPY skipped_pipelines.py ./ | ||
|
||
ENTRYPOINT [ "python", "./skipped_pipelines.py"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
# Purpose | ||
|
||
This script searches GitLab for branches that did not have a pipeline run for the most recent commit. After identifying such a commit, this script also schedules a pipeline to run on the affected branch. | ||
|
||
## Background | ||
|
||
This [issue](https://github.com/spack/spack-infrastructure/issues/316) describes the problem. | ||
|
||
## Cause | ||
|
||
Unknown | ||
|
||
## Mitigation | ||
|
||
Install a cron job to run a Python script that implements the following logic: | ||
|
||
``` | ||
for each branch: | ||
get HEAD commit | ||
check if there is a pipeline for this commit: | ||
if not, run one | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
requests |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,89 @@ | ||
#!/usr/bin/env python | ||
|
||
import json | ||
import os | ||
import re | ||
import urllib.parse | ||
|
||
import requests | ||
|
||
|
||
GITLAB_API_URL = "https://gitlab.spack.io/api/v4/projects/2" | ||
AUTH_HEADER = { | ||
"PRIVATE-TOKEN": os.environ.get("GITLAB_TOKEN", None) | ||
} | ||
|
||
|
||
def paginate(query_url): | ||
"""Helper method to get all pages of paginated query results""" | ||
results = [] | ||
|
||
while query_url: | ||
resp = requests.get(query_url, headers=AUTH_HEADER) | ||
|
||
if resp.status_code == 401: | ||
print(" !!! Unauthorized to make request, check GITLAB_TOKEN !!!") | ||
return [] | ||
|
||
next_batch = json.loads(resp.content) | ||
|
||
for result in next_batch: | ||
results.append(result) | ||
|
||
if "next" in resp.links: | ||
query_url = resp.links["next"]["url"] | ||
else: | ||
query_url = None | ||
|
||
return results | ||
|
||
|
||
def print_response(resp, padding=''): | ||
"""Helper method to print response status code and content""" | ||
print(f"{padding}response code: {resp.status_code}") | ||
print(f"{padding}response value: {resp.text}") | ||
|
||
|
||
def run_new_pipeline(pipeline_ref): | ||
"""Given a ref (branch name), run a new pipeline for that ref. If | ||
the branch has already been deleted from gitlab, this will generate | ||
an error and a 400 response, but we probably don't care.""" | ||
enc_ref = urllib.parse.quote_plus(pipeline_ref) | ||
run_url = f"{GITLAB_API_URL}/pipeline?ref={enc_ref}" | ||
print(f" !!!! running new pipeline for {pipeline_ref}") | ||
print_response(requests.post(run_url, headers=AUTH_HEADER), " ") | ||
|
||
|
||
def find_and_run_skipped_pipelines(): | ||
"""Query gitlab for all branches. Start a pipeline for any branch whose | ||
HEAD commit does not already have one. | ||
""" | ||
print(f"Attempting to find & fix skipped pipelines") | ||
branches_url = f"{GITLAB_API_URL}/repository/branches" | ||
branches = paginate(branches_url) | ||
print(f"Found {len(branches)} branches") | ||
|
||
regexp = re.compile("pr([0-9]+)") | ||
for branch in branches: | ||
branch_name = branch["name"] | ||
m = regexp.search(branch_name) | ||
if not m: | ||
print(f"Not a PR branch: {branch_name}") | ||
continue | ||
branch_commit = branch["commit"]["id"] | ||
pipelines_url = f"{GITLAB_API_URL}/pipelines?sha={branch_commit}" | ||
pipelines = paginate(pipelines_url) | ||
if len(pipelines) == 0: | ||
run_new_pipeline(branch_name) | ||
else: | ||
print(f"no need to run a new pipeline for {branch_name}") | ||
|
||
|
||
if __name__ == "__main__": | ||
if "GITLAB_TOKEN" not in os.environ: | ||
raise Exception("GITLAB_TOKEN environment is not set") | ||
try: | ||
find_and_run_skipped_pipelines() | ||
except Exception as inst: | ||
print("Caught unhandled exception:") | ||
print(inst) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
apiVersion: batch/v1 | ||
kind: CronJob | ||
metadata: | ||
name: skipped-pipelines | ||
namespace: custom | ||
spec: | ||
schedule: "*/30 * * * *" | ||
jobTemplate: | ||
spec: | ||
template: | ||
spec: | ||
restartPolicy: Never | ||
containers: | ||
- name: skipped-pipelines | ||
image: ghcr.io/spack/gitlab-skipped-pipelines:0.0.1 | ||
imagePullPolicy: IfNotPresent | ||
env: | ||
- name: GITLAB_TOKEN | ||
valueFrom: | ||
secretKeyRef: | ||
name: gitlab-clear-pipelines | ||
key: gitlab-token | ||
nodeSelector: | ||
spack.io/node-pool: base |