From 16cf93d6a32b537a2078595f689b62612fbff711 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 15 Oct 2024 15:10:04 +0200 Subject: [PATCH] =?UTF-8?q?Backport=20release=20"scripts"=20changes=20from?= =?UTF-8?q?=20main=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … as we switch to ghcr.io, we need those to be updated to work as they are referered during releases. Signed-off-by: Vincent Demeester --- tekton/README.md | 1 - tekton/bugfix-release.sh | 67 +++++++++++++++++++++++++++ tekton/publish.yaml | 85 ++++++++++++++++++++++++++--------- tekton/release-cheat-sheet.md | 17 ++++--- tekton/release-pipeline.yaml | 34 ++++++++++++-- 5 files changed, 173 insertions(+), 31 deletions(-) create mode 100755 tekton/bugfix-release.sh diff --git a/tekton/README.md b/tekton/README.md index a35b3ab40f0..3645146d6aa 100644 --- a/tekton/README.md +++ b/tekton/README.md @@ -110,7 +110,6 @@ Apply the tasks from the `pipeline` repo: # Apply the Tasks and Pipelines we use from this repo kubectl apply -f tekton/publish.yaml kubectl apply -f tekton/release-pipeline.yaml -kubectl apply -f tekton/release-pipeline-nightly.yaml # Apply the resources - note that when manually releasing you'll re-apply these kubectl apply -f tekton/resources.yaml diff --git a/tekton/bugfix-release.sh b/tekton/bugfix-release.sh new file mode 100755 index 00000000000..004d7670190 --- /dev/null +++ b/tekton/bugfix-release.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +set -eu -o pipefail + +RELEASE_BRANCH=${1:-$(git rev-parse --abbrev-ref HEAD)} +shift + +echo "> Make sure our remotes are up-to-date" +git fetch -p --all + +TEKTON_RELEASE_GIT_SHA=$(git rev-parse "${RELEASE_BRANCH}") +TEKTON_OLD_VERSION=$(git describe --tags --abbrev=0 "${TEKTON_RELEASE_GIT_SHA}") +TEKTON_OLD_VERSION_COMMIT_SHA=$(git rev-list -n 1 "${TEKTON_OLD_VERSION}") +TEKTON_RELEASE_NAME=$(gh release view "${TEKTON_OLD_VERSION}" --json name | jq .name | sed -e 's/.*\\"\(.*\)\\"\"/\1/') + +if [[ "${TEKTON_RELEASE_GIT_SHA}" == "${TEKTON_OLD_VERSION_COMMIT_SHA}" ]]; then + echo "> No new commit in ${RELEASE_BRANCH} (${TEKTON_RELEASE_GIT_SHA}==${TEKTON_OLD_VERSION_COMMIT_SHA})" + exit 0 +fi + +TEKTON_VERSION=$(echo ${TEKTON_OLD_VERSION} | awk -F. -v OFS=. '{$NF += 1 ; print}') + +echo "> Old version : ${TEKTON_OLD_VERSION}" +echo "> Old version commit : ${TEKTON_OLD_VERSION_COMMIT_SHA}" +echo "> New version : ${TEKTON_VERSION}" +echo "> New version commit: ${TEKTON_RELEASE_GIT_SHA}" +echo "> Tekton Release Name: ${TEKTON_RELEASE_NAME}" + +# Might be overkill +git --no-pager diff "${TEKTON_OLD_VERSION_COMMIT_SHA}" "${TEKTON_RELEASE_GIT_SHA}" + +cat < workspace-template.yaml +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +EOF + +echo "> Starting the release pipeline" +tkn pipeline start pipeline-release \ + --serviceaccount=release-right-meow \ + --param=gitRevision="${TEKTON_RELEASE_GIT_SHA}" \ + --param=serviceAccountPath=release.json \ + --param=versionTag="${TEKTON_VERSION}" \ + --param=releaseBucket=gs://tekton-releases/pipeline \ + --param=releaseAsLatest="false" \ + --workspace name=release-secret,secret=release-secret \ + --workspace name=workarea,volumeClaimTemplateFile=workspace-template.yaml --use-param-defaults --pipeline-timeout 3h --showlog + +RELEASE_FILE=https://storage.googleapis.com/tekton-releases/pipeline/previous/${TEKTON_VERSION}/release.yaml +CONTROLLER_IMAGE_SHA=$(curl $RELEASE_FILE | egrep 'gcr.io.*controller' | cut -d'@' -f2) +REKOR_UUID=$(rekor-cli search --sha $CONTROLLER_IMAGE_SHA | grep -v Found | head -1) +echo -e "CONTROLLER_IMAGE_SHA: ${CONTROLLER_IMAGE_SHA}\nREKOR_UUID: ${REKOR_UUID}" + +echo "> Starting the release-draft pipeline" +tkn pipeline start release-draft \ + --workspace name=shared,volumeClaimTemplateFile=workspace-template.yaml \ + --workspace name=credentials,secret=release-secret \ + -p package="tektoncd/pipeline" \ + -p git-revision="${TEKTON_RELEASE_GIT_SHA}" \ + -p release-tag="${TEKTON_VERSION}" \ + -p previous-release-tag="${TEKTON_OLD_VERSION}" \ + -p release-name="${TEKTON_RELEASE_NAME}" \ + -p bucket="gs://tekton-releases/pipeline" \ + -p rekor-uuid="$REKOR_UUID" \ + --showlog diff --git a/tekton/publish.yaml b/tekton/publish.yaml index fc81ce30e22..8a3ae9bbc4b 100644 --- a/tekton/publish.yaml +++ b/tekton/publish.yaml @@ -12,6 +12,9 @@ spec: - name: images description: List of cmd/* paths to be published as images default: "controller webhook entrypoint nop workingdirinit resolvers sidecarlogresults events" + - name: koExtraArgs + description: Extra args to be passed to ko + default: "--preserve-import-paths" - name: versionTag description: The vX.Y.Z version that the artifacts should be tagged with (including `v`) - name: imageRegistry @@ -22,6 +25,9 @@ spec: - name: imageRegistryRegions description: The target image registry regions default: "us eu asia" + - name: imageRegistryUser + description: Username to be used to login to the container registry + default: "_json_key" - name: releaseAsLatest description: Whether to tag and publish this release as Pipelines' latest default: "true" @@ -46,41 +52,47 @@ spec: env: - name: "PROJECT_ROOT" value: "$(workspaces.source.path)" - - name: CONTAINER_REGISTY_CREDENTIALS + - name: CONTAINER_REGISTRY_CREDENTIALS value: "$(workspaces.release-secret.path)/$(params.serviceAccountPath)" - name: CONTAINER_REGISTRY value: "$(params.imageRegistry)/$(params.imageRegistryPath)" + - name: IMAGE_REGISTRY_PATH + value: "$(params.imageRegistryPath)" + - name: CONTAINER_REGISTRY_USER + value: "$(params.imageRegistryUser)" - name: REGIONS value: "$(params.imageRegistryRegions)" - name: OUTPUT_RELEASE_DIR value: "$(workspaces.output.path)/$(params.versionTag)" + - name: KO_EXTRA_ARGS + value: "$(params.koExtraArgs)" results: # IMAGES result is picked up by Tekton Chains to sign the release. # See https://github.com/tektoncd/plumbing/blob/main/docs/signing.md for more info. - name: IMAGES steps: - - name: container-registy-auth - image: gcr.io/go-containerregistry/crane:debug + - name: container-registry-auth + image: cgr.dev/chainguard/crane:latest-dev@sha256:b33c82b22a6cfb21e3db968fba5f426461f7540d7fa37048e2a6ffb9aaca7f19 script: | - #!/busybox/sh + #!/bin/sh set -ex # Login to the container registry - DOCKER_CONFIG=$(cat ${CONTAINER_REGISTY_CREDENTIALS} | \ - crane auth login -u _json_key --password-stdin $(params.imageRegistry) 2>&1 | \ + DOCKER_CONFIG=$(cat ${CONTAINER_REGISTRY_CREDENTIALS} | \ + crane auth login -u ${CONTAINER_REGISTRY_USER} --password-stdin $(params.imageRegistry) 2>&1 | \ sed 's,^.*logged in via \(.*\)$,\1,g') # Auth with account credentials for all regions. for region in ${REGIONS} do HOSTNAME=${region}.$(params.imageRegistry) - cat ${CONTAINER_REGISTY_CREDENTIALS} | crane auth login -u _json_key --password-stdin ${HOSTNAME} + cat ${CONTAINER_REGISTRY_CREDENTIALS} | crane auth login -u ${CONTAINER_REGISTRY_USER} --password-stdin ${HOSTNAME} done cp ${DOCKER_CONFIG} /workspace/docker-config.json - name: create-ko-yaml - image: golang:1.18.7 + image: cgr.dev/chainguard/go:latest-dev@sha256:35dc4adbb3b6fadafd60d0a004d06d706f2f6ed1511f5c24f22f92f8fe94f783 script: | #!/bin/sh set -ex @@ -93,13 +105,21 @@ spec: # Change to directory with vendor/ cd ${PROJECT_ROOT} + COMBINED_BASE_IMAGE_BASE=${CONTAINER_REGISTRY} + # If the IMAGE_REGISTRY_PATH does not already includes the package, add it + # Package looks like github.com// + # Path may look like "tekton-releases" or "tektoncd/pipeline" + if [[ ! "$(params.package)" == "github.com/${IMAGE_REGISTRY_PATH}" ]]; then + COMBINED_BASE_IMAGE_BASE=${COMBINED_BASE_IMAGE_BASE}/${IMAGE_REGISTRY_PATH} + fi + # Combine Distroless with a Windows base image, used for the entrypoint image. # Distroless is pinned to the last version based on Alpine 3.18. Newer versions are based on Alpine 3.19_alpha20230901. COMBINED_BASE_IMAGE=$(go run ./vendor/github.com/tektoncd/plumbing/cmd/combine/main.go \ cgr.dev/chainguard/static@sha256:67a1b00e0134e2b3a614c7198a26f7deed9d11b7acad4d52c79c0cfd47a2eae7 \ mcr.microsoft.com/windows/nanoserver:ltsc2019 \ mcr.microsoft.com/windows/nanoserver:ltsc2022 \ - ${CONTAINER_REGISTRY}/$(params.package)/combined-base-image:latest) + ${COMBINED_BASE_IMAGE_BASE}/combined-base-image:latest) # NOTE: Make sure this list of images to use the combined base image is in sync with what's in test/presubmit-tests.sh's 'ko_resolve' function. cat < /workspace/.ko.yaml @@ -115,7 +135,7 @@ spec: cat /workspace/.ko.yaml - name: run-ko - image: gcr.io/tekton-releases/dogfooding/ko@sha256:9471f9698c2bc1816c03ed8eefbfc2613d90a843cb936da4236c2c2b3a18b6de + image: gcr.io/tekton-releases/dogfooding/ko@sha256:bdcd596b40583f6f8316745e27d7ff327a0756fb6aee1251dfc0bdbd01b26c88 env: - name: KO_DOCKER_REPO value: $(params.imageRegistry)/$(params.imageRegistryPath) @@ -159,37 +179,62 @@ spec: # The real "tagging" will happen with the "create-release" pipeline. git tag $(params.versionTag) - ko resolve --platform=$(params.platforms) --preserve-import-paths -t $(params.versionTag) -R -f ${PROJECT_ROOT}/config/ > $OUTPUT_RELEASE_DIR/release.yaml + ko resolve \ + --image-label=org.opencontainers.image.source=https://$(params.package) \ + --platform=$(params.platforms) \ + -t $(params.versionTag) \ + -R ${KO_EXTRA_ARGS} \ + -f ${PROJECT_ROOT}/config/ > $OUTPUT_RELEASE_DIR/release.yaml # Publish images and create release.notags.yaml # This is useful if your container runtime doesn't support the `image-reference:tag@digest` notation # This is currently the case for `cri-o` (and most likely others) - ko resolve --platform=$(params.platforms) --preserve-import-paths -R -f ${PROJECT_ROOT}/config/ > $OUTPUT_RELEASE_DIR/release.notags.yaml + ko resolve \ + --image-label=org.opencontainers.image.source=https://$(params.package) \ + --platform=$(params.platforms) \ + -R ${KO_EXTRA_ARGS} \ + -f ${PROJECT_ROOT}/config/ > $OUTPUT_RELEASE_DIR/release.notags.yaml # Rewrite "devel" to params.versionTag sed -i -e 's/\(pipeline.tekton.dev\/release\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(app.kubernetes.io\/version\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(version\): "devel"/\1: "$(params.versionTag)"/g' ${OUTPUT_RELEASE_DIR}/release.yaml sed -i -e 's/\(pipeline.tekton.dev\/release\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(app.kubernetes.io\/version\): "devel"/\1: "$(params.versionTag)"/g' -e 's/\(version\): "devel"/\1: "$(params.versionTag)"/g' ${OUTPUT_RELEASE_DIR}/release.notags.yaml - name: koparse - image: gcr.io/tekton-releases/dogfooding/koparse@sha256:5945f709f5533347e2fac2f7e757a2acde2ce25418a7193489bf49027aa0497f + image: gcr.io/tekton-releases/dogfooding/koparse@sha256:6b70f2d6fc1cc7849c5e65dcf404ee153653055799ceea511935bba7a27d3c44 script: | set -ex - IMAGES_PATH=${CONTAINER_REGISTRY}/$(params.package) + # Find "--preserve-import-paths" in a list of args + function find_preserve_import_path() { + for arg in $@; do + if [[ "$arg" == "--preserve-import-paths" ]]; then + return 0 + fi + done + return 1 + } + + # If "--preserve-import-paths" is used, include "package" in the expected path + find_preserve_import_path \ + $(echo $KO_EXTRA_ARGS) && \ + PRESERVE_IMPORT_PATH="--preserve-path" || \ + PRESERVE_IMPORT_PATH="--no-preserve-path" for cmd in $(params.images) do - IMAGES="${IMAGES} ${IMAGES_PATH}/cmd/${cmd}:$(params.versionTag)" + IMAGES="${IMAGES} $(params.package)/cmd/${cmd}:$(params.versionTag)" done # Parse the built images from the release.yaml generated by ko koparse \ --path $OUTPUT_RELEASE_DIR/release.yaml \ - --base ${IMAGES_PATH} --images ${IMAGES} > /workspace/built_images - + --base $(params.package) \ + --container-registry ${CONTAINER_REGISTRY} \ + --images ${IMAGES} \ + ${PRESERVE_IMPORT_PATH} > /workspace/built_images - name: tag-images - image: gcr.io/go-containerregistry/crane:debug + image: cgr.dev/chainguard/crane:latest-dev@sha256:b33c82b22a6cfb21e3db968fba5f426461f7540d7fa37048e2a6ffb9aaca7f19 script: | - #!/busybox/sh + #!/bin/sh set -ex # Setup docker-auth @@ -197,8 +242,6 @@ spec: mkdir -p ${DOCKER_CONFIG} cp /workspace/docker-config.json ${DOCKER_CONFIG}/config.json - REGIONS="us eu asia" - # Tag the images and put them in all the regions for IMAGE in $(cat /workspace/built_images) do diff --git a/tekton/release-cheat-sheet.md b/tekton/release-cheat-sheet.md index af86c88f57d..bf8efa69143 100644 --- a/tekton/release-cheat-sheet.md +++ b/tekton/release-cheat-sheet.md @@ -60,15 +60,22 @@ the pipelines repo, a terminal window and a text editor. ```bash tkn --context dogfooding pipeline start pipeline-release \ --serviceaccount=release-right-meow \ - --param=gitRevision="${TEKTON_RELEASE_GIT_SHA}" \ - --param=serviceAccountPath=release.json \ - --param=versionTag="${TEKTON_VERSION}" \ - --param=releaseBucket=gs://tekton-releases/pipeline \ + --param package=github.com/tektoncd/pipeline \ + --param gitRevision="${TEKTON_RELEASE_GIT_SHA}" \ + --param imageRegistry=ghcr.io \ + --param imageRegistryPath=tektoncd/pipeline \ + --param imageRegistryRegions="" \ + --param imageRegistryUser=tekton-robot \ + --param serviceAccountPath=release.json \ + --param serviceAccountImagesPath=credentials \ + --param versionTag="${TEKTON_VERSION}" \ + --param releaseBucket=gs://tekton-releases/pipeline \ + --param koExtraArgs="" \ --workspace name=release-secret,secret=release-secret \ + --workspace name=release-images-secret,secret=ghcr-creds \ --workspace name=workarea,volumeClaimTemplateFile=workspace-template.yaml \ --tasks-timeout 2h \ --pipeline-timeout 3h - ``` Accept the default values of the parameters (except for "releaseAsLatest" if backporting). diff --git a/tekton/release-pipeline.yaml b/tekton/release-pipeline.yaml index 7bc5eafe12b..e1db44e2bc3 100644 --- a/tekton/release-pipeline.yaml +++ b/tekton/release-pipeline.yaml @@ -16,6 +16,12 @@ spec: - name: imageRegistryPath description: The path (project) in the image registry default: tekton-releases + - name: imageRegistryRegions + description: The target image registry regions + default: "us eu asia" + - name: imageRegistryUser + description: The user for the image registry credentials + default: _json_key - name: versionTag description: The X.Y.Z version that the artifacts should be tagged with - name: releaseBucket @@ -33,13 +39,23 @@ spec: can differ from buildPlatforms due to the fact that a windows-compatible base image is constructed for the publishing phase. default: linux/amd64,linux/arm,linux/arm64,linux/s390x,linux/ppc64le,windows/amd64 + - name: koExtraArgs + description: Extra args to be passed to ko + default: "--preserve-import-paths" - name: serviceAccountPath description: The path to the service account file within the release-secret workspace + - name: serviceAccountImagesPath + description: The path to the service account file or credentials within the release-images-secret workspace + - name: runTests + description: If set to something other than "true", skip the build and test tasks + default: "true" workspaces: - name: workarea description: The workspace where the repo will be cloned. - name: release-secret - description: The secret that contains a service account authorized to push to the imageRegistry and to the output bucket + description: The secret that contains a service account authorized to push to the output bucket + - name: release-images-secret + description: The secret that contains a service account authorized to push to the imageRegistry results: - name: commit-sha description: the sha of the commit that was released @@ -94,6 +110,8 @@ spec: subpath: git - name: unit-tests runAfter: [precheck] + when: + - cel: "'$(params.runTests)' == 'true'" taskRef: resolver: bundles params: @@ -114,6 +132,8 @@ spec: subpath: git - name: build runAfter: [precheck] + when: + - cel: "'$(params.runTests)' == 'true'" taskRef: resolver: bundles params: @@ -154,12 +174,18 @@ spec: value: $(params.imageRegistry) - name: imageRegistryPath value: $(params.imageRegistryPath) + - name: imageRegistryUser + value: $(params.imageRegistryUser) + - name: imageRegistryRegions + value: $(params.imageRegistryRegions) - name: releaseAsLatest value: $(params.releaseAsLatest) - name: serviceAccountPath - value: $(params.serviceAccountPath) + value: $(params.serviceAccountImagesPath) - name: platforms value: $(params.publishPlatforms) + - name: koExtraArgs + value: $(params.koExtraArgs) workspaces: - name: source workspace: workarea @@ -168,7 +194,7 @@ spec: workspace: workarea subpath: bucket - name: release-secret - workspace: release-secret + workspace: release-images-secret timeout: 2h - name: publish-to-bucket runAfter: [publish-images] @@ -242,7 +268,7 @@ spec: description: The full URL of the release file (no tag) in the bucket steps: - name: create-results - image: alpine + image: docker.io/library/alpine:3.20.3@sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d env: - name: RELEASE_BUCKET value: $(params.releaseBucket)