From 6ddbf395b9626702441d6aad0436fc3c0f5206b1 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Mon, 22 Jul 2024 16:01:59 +0100 Subject: [PATCH] ROX-25321: convert konflux builds to OCI artifacts (#1573) --- .tekton/determine-image-tag-oci-ta.yaml | 44 +++++++ .tekton/fetch-scanner-data-oci-ta-task.yaml | 71 +++++++++++ .tekton/scanner-build.yaml | 10 -- .tekton/scanner-component-pipeline.yaml | 132 ++++++++------------ .tekton/scanner-db-build.yaml | 10 -- .tekton/scanner-db-slim-build.yaml | 10 -- .tekton/scanner-slim-build.yaml | 10 -- 7 files changed, 164 insertions(+), 123 deletions(-) create mode 100644 .tekton/determine-image-tag-oci-ta.yaml create mode 100644 .tekton/fetch-scanner-data-oci-ta-task.yaml diff --git a/.tekton/determine-image-tag-oci-ta.yaml b/.tekton/determine-image-tag-oci-ta.yaml new file mode 100644 index 000000000..a9799a3b0 --- /dev/null +++ b/.tekton/determine-image-tag-oci-ta.yaml @@ -0,0 +1,44 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: determine-image-tag-oci-ta + namespace: rh-acs-tenant +# TODO(ROX-23812): Refactor to a task bundle +spec: + description: Determines the tag for the output image using the StackRox convention from 'make tag' output. + params: + - name: TAG_SUFFIX + description: Suffix to append to generated image tag. + type: string + - name: SOURCE_ARTIFACT + description: The Trusted Artifact URI pointing to the artifact with + the application source code. This should be the result of the git-clone task, + results from other tasks might fail as dirty. + type: string + results: + - name: IMAGE_TAG + description: Image Tag determined by custom logic. + volumes: + - name: workdir + emptyDir: {} + stepTemplate: + volumeMounts: + - mountPath: /var/workdir + name: workdir + steps: + - name: use-trusted-artifact + image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:4e39fb97f4444c2946944482df47b39c5bbc195c54c6560b0647635f553ab23d + args: + - use + - $(params.SOURCE_ARTIFACT)=/var/workdir/source + - name: determine-image-tag + image: registry.access.redhat.com/ubi8:latest + workingDir: /var/workdir/source + script: | + #!/usr/bin/env bash + set -euo pipefail + dnf -y upgrade --nobest + dnf -y install git make + + scripts/konflux/fail-build-if-git-is-dirty.sh + echo -n "$(make --quiet --no-print-directory tag)$(params.TAG_SUFFIX)" | tee "$(results.IMAGE_TAG.path)" diff --git a/.tekton/fetch-scanner-data-oci-ta-task.yaml b/.tekton/fetch-scanner-data-oci-ta-task.yaml new file mode 100644 index 000000000..bc4ac602f --- /dev/null +++ b/.tekton/fetch-scanner-data-oci-ta-task.yaml @@ -0,0 +1,71 @@ +apiVersion: tekton.dev/v1 +kind: Task +metadata: + name: fetch-scanner-data-oci-ta + namespace: rh-acs-tenant +# TODO(ROX-22196): Make the task EC-compliant (e.g. move to a bundle) +spec: + description: Downloads blobs from definitions.stackrox.io GCloud bucket to be included in Scanner container builds. + params: + - name: blobs-to-fetch + description: | + List of scanner-data file names to fetch to include in the container build. + An empty list is allowed which results in no-op. + type: array + - name: target-dir + description: Target directory relative to workspace where to save downloaded blobs. + type: string + default: "." + - name: SOURCE_ARTIFACT + description: The Trusted Artifact URI pointing to the artifact with + the application source code. This should be the result of the git-clone task, + results from other tasks might fail as dirty. + type: string + - name: ociStorage + description: The OCI repository where the Trusted Artifacts are stored. + type: string + - name: ociArtifactExpiresAfter + description: Expiration date for the trusted artifacts created in the + OCI repository. + type: string + results: + - name: SOURCE_ARTIFACT + description: The Trusted Artifact URI pointing to the artifact with the application source code + and additional smuggled activation key. + type: string + volumes: + - name: workdir + emptyDir: {} + stepTemplate: + volumeMounts: + - mountPath: /var/workdir + name: workdir + steps: + - name: use-trusted-artifact + image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:4e39fb97f4444c2946944482df47b39c5bbc195c54c6560b0647635f553ab23d + args: + - use + - $(params.SOURCE_ARTIFACT)=/var/workdir/source + - name: fetch-scanner-data + image: registry.access.redhat.com/ubi8/ubi-minimal:latest + # The only functioning way to pass array parameter that I found is through args array. + # Array params have weird limitations, see https://github.com/tektoncd/pipeline/blob/main/docs/tasks.md#substituting-array-parameters + # Attempts to pass this in other places result in webhook errors and pipeline not starting. + args: [ "$(params.blobs-to-fetch[*])" ] + workingDir: /var/workdir/source + script: | + #!/usr/bin/env bash + set -euo pipefail + scripts/konflux/fetch-scanner-data.sh "$(params.target-dir)" "$@" + # It should not take long to download blobs otherwise there's something odd going on. + timeout: 10m + - name: create-trusted-artifact + image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:4e39fb97f4444c2946944482df47b39c5bbc195c54c6560b0647635f553ab23d + args: + - create + - --store + - $(params.ociStorage) + - $(results.SOURCE_ARTIFACT.path)=/var/workdir/source + env: + - name: IMAGE_EXPIRES_AFTER + value: $(params.ociArtifactExpiresAfter) diff --git a/.tekton/scanner-build.yaml b/.tekton/scanner-build.yaml index c0008f32c..9dcf1b503 100644 --- a/.tekton/scanner-build.yaml +++ b/.tekton/scanner-build.yaml @@ -53,16 +53,6 @@ spec: value: [ 'nvd-definitions.zip', 'k8s-definitions.zip', 'repo2cpe.zip', 'genesis_manifests.json' ] workspaces: - - name: workspace - volumeClaimTemplate: - metadata: - creationTimestamp: null - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - name: git-auth secret: secretName: '{{ git_auth_secret }}' diff --git a/.tekton/scanner-component-pipeline.yaml b/.tekton/scanner-component-pipeline.yaml index 34be73427..2c795d2ca 100644 --- a/.tekton/scanner-component-pipeline.yaml +++ b/.tekton/scanner-component-pipeline.yaml @@ -19,28 +19,6 @@ spec: - name: kind value: task resolver: bundles - - name: show-summary - params: - - name: pipelinerun-name - value: $(context.pipelineRun.name) - - name: git-url - value: $(tasks.clone-repository.results.url)?rev=$(tasks.clone-repository.results.commit) - - name: image-url - value: $(tasks.build-container.results.IMAGE_URL) - - name: build-task-status - value: $(tasks.build-container.status) - workspaces: - - name: workspace - workspace: workspace - taskRef: - params: - - name: name - value: summary - - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-summary:0.2@sha256:51d5aaa4e13e9fb4303f667e38d07e758820040032ed9fb3ab5f6afaaffc60d8 - - name: kind - value: task - resolver: bundles params: - description: Source Repository URL @@ -112,6 +90,10 @@ spec: description: List of scanner-data file names to fetch to include in the container build. name: blobs-to-fetch type: array + - default: "1d" + description: This sets the expiration time for intermediate OCI artifacts produced and used during builds after which they can be garbage collected. + name: oci-artifact-expires-after + type: string results: - description: "" @@ -131,7 +113,6 @@ spec: value: $(tasks.build-container.results.JAVA_COMMUNITY_DEPENDENCIES) workspaces: - - name: workspace - name: git-auth tasks: @@ -167,14 +148,16 @@ spec: value: $(params.clone-depth) - name: fetchTags value: $(params.clone-fetch-tags) - runAfter: - - init + - name: ociStorage + value: $(params.output-image-repo):konflux-$(params.revision).git + - name: ociArtifactExpiresAfter + value: $(params.oci-artifact-expires-after) taskRef: params: - name: name - value: git-clone + value: git-clone-oci-ta - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-git-clone:0.1@sha256:ae1249aa49e82da5f99cc23b256172dce8f7c7951ece68ca0419240c4ecb52e2 + value: quay.io/redhat-appstudio-tekton-catalog/task-git-clone-oci-ta:0.1@sha256:f1e58dcdb32efa9ca5b6f44e3600814624b9a8cfd59a1701379c789eeb8eef5b - name: kind value: task resolver: bundles @@ -183,59 +166,55 @@ spec: operator: in values: [ "true" ] workspaces: - - name: output - workspace: workspace - name: basic-auth workspace: git-auth - name: determine-image-tag params: - - name: tag-suffix + - name: TAG_SUFFIX value: $(params.output-tag-suffix) - runAfter: - # This task must run on a freshly cloned repository to prevent seeing any changes from other tasks. - - clone-repository + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) taskRef: - name: determine-image-tag - workspaces: - - name: source - workspace: workspace + name: determine-image-tag-oci-ta - name: prefetch-dependencies params: - name: input value: $(params.prefetch-input) - runAfter: - - determine-image-tag + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + - name: ociStorage + value: $(params.output-image-repo):konflux-$(params.revision).prefetch + - name: ociArtifactExpiresAfter + value: $(params.oci-artifact-expires-after) taskRef: params: - name: name - value: prefetch-dependencies + value: prefetch-dependencies-oci-ta - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-prefetch-dependencies:0.1@sha256:9aec3ae9f0f50a05abdc739faf4cbc82832cff16c77ac74e1d54072a882c0503 + value: quay.io/redhat-appstudio-tekton-catalog/task-prefetch-dependencies-oci-ta:0.1@sha256:a977d1a1b8cb79f29056e20a2e225b1018d84daad8b546de82f30b6326afee22 - name: kind value: task resolver: bundles - workspaces: - - name: source - workspace: workspace - name: fetch-scanner-data params: - name: blobs-to-fetch value: [ "$(params.blobs-to-fetch[*])" ] - runAfter: - - determine-image-tag + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: ociStorage + value: $(params.output-image-repo):konflux-$(params.revision).prefetch + - name: ociArtifactExpiresAfter + value: $(params.oci-artifact-expires-after) taskRef: - name: fetch-scanner-data - workspaces: - - name: source - workspace: workspace + name: fetch-scanner-data-oci-ta - name: build-container params: - name: IMAGE - value: $(params.output-image-repo):$(tasks.determine-image-tag.results.image-tag) + value: $(params.output-image-repo):$(tasks.determine-image-tag.results.IMAGE_TAG) - name: DOCKERFILE value: $(params.dockerfile) - name: CONTEXT @@ -252,16 +231,17 @@ spec: value: $(params.build-target-stage) - name: BUILD_ARGS value: - - SCANNER_TAG=$(tasks.determine-image-tag.results.image-tag) - runAfter: - - prefetch-dependencies - - fetch-scanner-data + - SCANNER_TAG=$(tasks.determine-image-tag.results.IMAGE_TAG) + - name: SOURCE_ARTIFACT + value: $(tasks.fetch-scanner-data.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) taskRef: params: - name: name - value: buildah + value: buildah-oci-ta - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-buildah:0.1@sha256:102500165339bc08791775cf2c4dcae3dd4bde557a9009d44dc590ef66dde384 + value: quay.io/redhat-appstudio-tekton-catalog/task-buildah-oci-ta:0.1@sha256:2a42822363c83b95a84c2f6a10c4d957835431d812b1e4a045b51fab1cca9769 - name: kind value: task resolver: bundles @@ -269,9 +249,6 @@ spec: - input: $(tasks.init.results.build) operator: in values: [ "true" ] - workspaces: - - name: source - workspace: workspace - name: apply-tags params: @@ -298,14 +275,16 @@ spec: value: $(tasks.build-container.results.IMAGE_URL) - name: BASE_IMAGES value: $(tasks.build-container.results.BASE_IMAGES_DIGESTS) - runAfter: - - build-container + - name: SOURCE_ARTIFACT + value: $(tasks.fetch-scanner-data.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) taskRef: params: - name: name - value: source-build + value: source-build-oci-ta - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-source-build:0.1@sha256:1a976a35adee9163e455d0c5aee5d9bf9cb3c6a770656ae347558f8c54977709 + value: quay.io/redhat-appstudio-tekton-catalog/task-source-build-oci-ta:0.1@sha256:eae2e52027120286bcd3c1e3a48bf3f1281d9718549b9cd4098dcf11b06b71b8 - name: kind value: task resolver: bundles @@ -316,9 +295,6 @@ spec: - input: $(params.build-source-image) operator: in values: [ "true" ] - workspaces: - - name: workspace - workspace: workspace - name: deprecated-base-image-check params: @@ -328,8 +304,6 @@ spec: value: $(tasks.build-container.results.IMAGE_URL) - name: IMAGE_DIGEST value: $(tasks.build-container.results.IMAGE_DIGEST) - runAfter: - - build-container taskRef: params: - name: name @@ -350,8 +324,6 @@ spec: value: $(tasks.build-container.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-container.results.IMAGE_URL) - runAfter: - - build-container taskRef: params: - name: name @@ -367,14 +339,15 @@ spec: values: [ "false" ] - name: sast-snyk-check - runAfter: - - clone-repository + params: + - name: SOURCE_ARTIFACT + value: $(tasks.fetch-scanner-data.results.SOURCE_ARTIFACT) taskRef: params: - name: name - value: sast-snyk-check + value: sast-snyk-check-oci-ta - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-sast-snyk-check:0.1@sha256:242acc527a06a11fac9dd6524467f62f3a086c186c5f885973e5780a04d4289c + value: quay.io/redhat-appstudio-tekton-catalog/task-sast-snyk-check-oci-ta:0.1@sha256:45f7edd80cde6c303d0bc7060ad2a98f2d84d96dc19e9973cdb6179d0e6ae7eb - name: kind value: task resolver: bundles @@ -382,9 +355,6 @@ spec: - input: $(params.skip-checks) operator: in values: [ "false" ] - workspaces: - - name: workspace - workspace: workspace - name: clamav-scan params: @@ -392,14 +362,12 @@ spec: value: $(tasks.build-container.results.IMAGE_DIGEST) - name: image-url value: $(tasks.build-container.results.IMAGE_URL) - runAfter: - - build-container taskRef: params: - name: name value: clamav-scan - name: bundle - value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:5dbe6c646c3502ddc7fbe6016b8584bed6ce3ab7028b0c405ebaabc7e6e9e64c + value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:3d175c521a65a8c00f509e67e62def03ab28911f70868399619c9804b81e38a0 - name: kind value: task resolver: bundles @@ -414,8 +382,6 @@ spec: value: $(tasks.build-container.results.IMAGE_URL) - name: IMAGE_DIGEST value: $(tasks.build-container.results.IMAGE_DIGEST) - runAfter: - - build-container taskRef: params: - name: name diff --git a/.tekton/scanner-db-build.yaml b/.tekton/scanner-db-build.yaml index 8ea49a7d7..b3e6d4a8a 100644 --- a/.tekton/scanner-db-build.yaml +++ b/.tekton/scanner-db-build.yaml @@ -54,16 +54,6 @@ spec: value: [ 'pg-definitions.sql.gz' ] workspaces: - - name: workspace - volumeClaimTemplate: - metadata: - creationTimestamp: null - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - name: git-auth secret: secretName: '{{ git_auth_secret }}' diff --git a/.tekton/scanner-db-slim-build.yaml b/.tekton/scanner-db-slim-build.yaml index 25b2c655f..d36d3c475 100644 --- a/.tekton/scanner-db-slim-build.yaml +++ b/.tekton/scanner-db-slim-build.yaml @@ -54,16 +54,6 @@ spec: value: [ ] workspaces: - - name: workspace - volumeClaimTemplate: - metadata: - creationTimestamp: null - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - name: git-auth secret: secretName: '{{ git_auth_secret }}' diff --git a/.tekton/scanner-slim-build.yaml b/.tekton/scanner-slim-build.yaml index 23f817359..b7e17d107 100644 --- a/.tekton/scanner-slim-build.yaml +++ b/.tekton/scanner-slim-build.yaml @@ -53,16 +53,6 @@ spec: value: [ 'nvd-definitions.zip', 'k8s-definitions.zip', 'repo2cpe.zip', 'genesis_manifests.json' ] workspaces: - - name: workspace - volumeClaimTemplate: - metadata: - creationTimestamp: null - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - name: git-auth secret: secretName: '{{ git_auth_secret }}'