From 66a6ffc98312a3876eb258285aaf7c6a54e87e61 Mon Sep 17 00:00:00 2001 From: Zoran Regvart Date: Thu, 31 Oct 2024 14:18:49 +0100 Subject: [PATCH] Guard Task execution via changed files The new Task in `.tekton/tasks/task-switchboard.yaml` produces a list of bindings that evaluate to true for a particular pull request as an array result. Given this result guards can be done, e.g. using when expressions to limit when a Task on the Pipeline needs to run. This way we can skip expensive Tasks that are unrelated to the change done in the pull request. Similar to work in #1188 and #524, with the distinction that the PipelineRun is executed, only potentially not in full. --- .tekton/pull-request.yaml | 47 +++++++++++++++++++++++-- .tekton/tasks/task-switchboard.yaml | 53 +++++++++++++++++++++++++++++ appstudio-utils/Dockerfile | 1 + 3 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 .tekton/tasks/task-switchboard.yaml diff --git a/.tekton/pull-request.yaml b/.tekton/pull-request.yaml index dc87d4f1a..af821cac2 100644 --- a/.tekton/pull-request.yaml +++ b/.tekton/pull-request.yaml @@ -5,7 +5,7 @@ metadata: name: build-definitions-pull-request annotations: pipelinesascode.tekton.dev/on-cel-expression: (event == "pull_request" && target_branch == "main" && ( !has(body.pull_request) || !body.pull_request.draft) ) || (event == "push" && target_branch.startsWith("gh-readonly-queue/main/")) - pipelinesascode.tekton.dev/task: "[task/git-clone/0.1/git-clone.yaml, .tekton/tasks/buildah.yaml, .tekton/tasks/task-lint.yaml, .tekton/tasks/e2e-test.yaml, task/sast-snyk-check/0.2/sast-snyk-check.yaml, task/sast-unicode-check/0.1/sast-unicode-check.yaml]" + pipelinesascode.tekton.dev/task: "[task/git-clone/0.1/git-clone.yaml, .tekton/tasks/buildah.yaml, .tekton/tasks/task-lint.yaml, .tekton/tasks/e2e-test.yaml, task/sast-snyk-check/0.2/sast-snyk-check.yaml, task/sast-unicode-check/0.1/sast-unicode-check.yaml, .tekton/tasks/task-switchboard.yaml]" pipelinesascode.tekton.dev/max-keep-runs: "5" spec: params: @@ -36,7 +36,26 @@ spec: value: $(params.revision) - name: depth value: "0" + - name: task-switchboard + taskRef: + name: task-switchboard + params: + - name: pr_number + value: "{{ pull_request_number }}" + - name: utils_image + value: quay.io/konflux-ci/pull-request-builds:appstudio-utils-{{revision}} + - expressions: + - tasks := strings.any_prefix_match(input, "task/") + - tasks_pipelines := strings.any_prefix_match(input, ["task/", "pipelines/"]) + - e2e_tests := strings.any_prefix_match(input, ["task/", "pipelines/"]) + - check_partner_tasks := strings.any_prefix_match(input, "partners/") + runAfter: + - build-appstudio-utils - name: task-lint-check + when: + - input: "tasks" + operator: "in" + values: ["$(tasks.task-switchboard.results.bindings[*])"] runAfter: - fetch-repository taskRef: @@ -74,6 +93,10 @@ spec: - name: source workspace: workspace - name: check-partner-tasks + when: + - input: "check_partner_tasks" + operator: "in" + values: ["$(tasks.task-switchboard.results.bindings[*])"] runAfter: - build-appstudio-utils taskSpec: @@ -125,13 +148,15 @@ spec: - name: source workspace: workspace - name: build-bundles + when: + - input: "tasks_pipelines" + operator: "in" + values: ["$(tasks.task-switchboard.results.bindings[*])"] params: - name: revision value: "{{ revision }}" - name: e2e_test_namespace value: $(params.e2e_test_namespace) - runAfter: - - build-appstudio-utils workspaces: - name: source workspace: workspace @@ -174,6 +199,10 @@ spec: workspaces: - name: source - name: e2e-tests + when: + - input: "e2e_tests" + operator: "in" + values: ["$(tasks.task-switchboard.results.bindings[*])"] params: - name: e2e_test_namespace value: $(params.e2e_test_namespace) @@ -198,6 +227,10 @@ spec: # Added a timeout due to https://issues.redhat.com/browse/STONEBLD-2265 timeout: "2h" - name: check-task-pipeline-repo-existence + when: + - input: "tasks_pipelines" + operator: "in" + values: ["$(tasks.task-switchboard.results.bindings[*])"] runAfter: - build-bundles taskSpec: @@ -214,6 +247,10 @@ spec: - name: source workspace: workspace - name: ec-task-checks + when: + - input: "tasks_pipelines" + operator: "in" + values: ["$(tasks.task-switchboard.results.bindings[*])"] runAfter: - fetch-repository taskRef: @@ -222,6 +259,10 @@ spec: - name: source workspace: workspace - name: check-task-migration-md + when: + - input: "tasks" + operator: "in" + values: ["$(tasks.task-switchboard.results.bindings[*])"] runAfter: - fetch-repository taskSpec: diff --git a/.tekton/tasks/task-switchboard.yaml b/.tekton/tasks/task-switchboard.yaml new file mode 100644 index 000000000..457d13c16 --- /dev/null +++ b/.tekton/tasks/task-switchboard.yaml @@ -0,0 +1,53 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: task-switchboard + labels: + app.kubernetes.io/version: "0.1" + annotations: + tekton.dev/pipelines.minVersion: "0.12.1" + tekton.dev/displayName: Task Switchboard + tekton.dev/platforms: "linux/amd64" +spec: + description: "Computes a set of expressions based on the changed files in the + pipeline, used to determine which tasks to run" + params: + - name: pr_number + type: string + - name: utils_image + type: string + - name: expressions + type: array + results: + - name: bindings + type: array + steps: + - name: list-changed-files + image: $(params.utils_image) + env: + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: "{{ git_auth_secret }}" + key: "git-provider-token" + args: + - "$(params.expressions[*])" + script: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + rules="$(mktemp -d)" + trap 'rm -rf "${rules}"' EXIT + for ((i=1; i<=$#; ++i)); do + printf "package rule\n%s" "${!i}" > "${rules}/$i.rego" + done + + ec opa check --v1-compatible "${rules}" + + printf '%s\n' "$@" \ + | ec opa eval --v1-compatible --stdin --format bindings --input \ + <(gh pr view "https://github.com/konflux-ci/build-definitions/pull/$(params.pr_number)" --json files --jq '.files.[].path') \ + | jq '[to_entries | .[] | select(.value == true) | .key]' \ + | tee "$(results.bindings.path)" \ No newline at end of file diff --git a/appstudio-utils/Dockerfile b/appstudio-utils/Dockerfile index 9d7d9626f..474997957 100755 --- a/appstudio-utils/Dockerfile +++ b/appstudio-utils/Dockerfile @@ -8,6 +8,7 @@ RUN curl -L https://github.com/tektoncd/cli/releases/download/v0.32.2/tkn_0.32.2 RUN curl -L https://github.com/sigstore/rekor/releases/download/v0.5.0/rekor-cli-linux-amd64 -o /usr/bin/rekor-cli && chmod +x /usr/bin/rekor-cli RUN curl -L https://github.com/open-policy-agent/conftest/releases/download/v0.32.0/conftest_0.32.0_Linux_x86_64.tar.gz | tar -xz --no-same-owner -C /usr/bin RUN curl -L https://github.com/enterprise-contract/ec-cli/releases/download/snapshot/ec_linux_amd64 -o /usr/bin/ec && chmod +x /usr/bin/ec && ec version +RUN curl -L https://github.com/cli/cli/releases/download/v2.60.1/gh_2.60.1_linux_amd64.tar.gz | tar -xz -C /usr/bin --wildcards "gh_*/bin/gh" --strip-components=2 --no-same-owner RUN dnf -y --setopt=tsflags=nodocs install \ git \