From f66419be19ff36385573d5557d8a7d18101bbe3f Mon Sep 17 00:00:00 2001 From: Terry Howe Date: Wed, 11 Dec 2024 06:44:57 -0700 Subject: [PATCH] Squashed 'release-tools/' changes from 49676850..406a79ac 406a79ac Merge pull request #267 from huww98/gomodcache 9cec273d Set GOMODCACHE to avoid re-download toolchain 98f23071 Merge pull request #260 from TerryHowe/update-csi-driver-version e9d8712d Merge pull request #259 from stmcginnis/deprecated-kind-kube-root faf79ff6 Remove --kube-root deprecated kind argument 734c2b95 Merge pull request #265 from Rakshith-R/consider-main-branch f95c855b Merge pull request #262 from huww98/golang-toolchain 3c8d966f Treat main branch as equivalent to master branch e31de525 Merge pull request #261 from huww98/golang fd153a9e Bump golang to 1.23.1 a8b3d050 pull-test.sh: fix "git subtree pull" errors 6b05f0fc use new GOTOOLCHAIN env to manage go version 18b6ac6d chore: update CSI driver version to 1.15 227577e0 Merge pull request #258 from gnufied/enable-race-detection e1ceee28 Always enable race detection while running tests 988496a1 Merge pull request #257 from jakobmoellerdev/csi-prow-sidecar-e2e-path 028f8c69 chore: bump to Go 1.22.5 69bd71e8 chore: add CSI_PROW_SIDECAR_E2E_PATH f40f0ccd Merge pull request #256 from solumath/master cfa92106 Instruction update 379a1bb9 Merge pull request #255 from humblec/sidecar-md a5667bbb fix typo in sidecar release process git-subtree-dir: release-tools git-subtree-split: 406a79acf021b5564108afebeea7d0ed44648d3f --- SIDECAR_RELEASE_PROCESS.md | 2 +- build.make | 13 +++++++------ generate-patch-release-notes.sh | 16 ++++++++++------ prow.sh | 32 ++++++++++++++++++-------------- pull-test.sh | 5 +++++ 5 files changed, 41 insertions(+), 27 deletions(-) diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md index 647d2342b..aab8d6e21 100644 --- a/SIDECAR_RELEASE_PROCESS.md +++ b/SIDECAR_RELEASE_PROCESS.md @@ -84,7 +84,7 @@ naming convention `-on-`. 1. Compare the generated output to the new commits for the release to check if any notable change missed a release note. 1. Reword release notes as needed, ideally in the original PRs so that the - release notes can be regnerated. Make sure to check notes for breaking + release notes can be regenerated. Make sure to check notes for breaking changes and deprecations. 1. If release is a new major/minor version, create a new `CHANGELOG-..md` file. diff --git a/build.make b/build.make index fe120c52b..39a34777d 100644 --- a/build.make +++ b/build.make @@ -45,9 +45,10 @@ REV=$(shell git describe --long --tags --match='v*' --dirty 2>/dev/null || git r # Determined dynamically. IMAGE_TAGS= -# A "canary" image gets built if the current commit is the head of the remote "master" branch. +# A "canary" image gets built if the current commit is the head of the remote "master" or "main" branch. # That branch does not exist when building some other branch in TravisCI. IMAGE_TAGS+=$(shell if [ "$$(git rev-list -n1 HEAD)" = "$$(git rev-list -n1 origin/master 2>/dev/null)" ]; then echo "canary"; fi) +IMAGE_TAGS+=$(shell if [ "$$(git rev-list -n1 HEAD)" = "$$(git rev-list -n1 origin/main 2>/dev/null)" ]; then echo "canary"; fi) # A "X.Y.Z-canary" image gets built if the current commit is the head of a "origin/release-X.Y.Z" branch. # The actual suffix does not matter, only the "release-" prefix is checked. @@ -62,9 +63,9 @@ IMAGE_NAME=$(REGISTRY_NAME)/$* ifdef V # Adding "-alsologtostderr" assumes that all test binaries contain glog. This is not guaranteed. -TESTARGS = -v -args -alsologtostderr -v 5 +TESTARGS = -race -v -args -alsologtostderr -v 5 else -TESTARGS = +TESTARGS = -race endif # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables @@ -143,7 +144,7 @@ DOCKER_BUILDX_CREATE_ARGS ?= # Windows binaries can be built before adding a Dockerfile for it. # # BUILD_PLATFORMS determines which individual images are included in the multiarch image. -# PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines +# PULL_BASE_REF must be set to 'master', 'main', 'release-x.y', or a tag name, and determines # the tag for the resulting multiarch image. $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% set -ex; \ @@ -191,7 +192,7 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% done; \ docker manifest push -p $(IMAGE_NAME):$$tag; \ }; \ - if [ $(PULL_BASE_REF) = "master" ]; then \ + if [ $(PULL_BASE_REF) = "master" ] || [ $(PULL_BASE_REF) = "main" ]; then \ : "creating or overwriting canary image"; \ pushMultiArch canary; \ elif echo $(PULL_BASE_REF) | grep -q -e 'release-*' ; then \ @@ -209,7 +210,7 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% .PHONY: check-pull-base-ref check-pull-base-ref: if ! [ "$(PULL_BASE_REF)" ]; then \ - echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name."; \ + echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'main', 'release-x.y', or a tag name."; \ exit 1; \ fi diff --git a/generate-patch-release-notes.sh b/generate-patch-release-notes.sh index 2b9c13c38..536a14902 100755 --- a/generate-patch-release-notes.sh +++ b/generate-patch-release-notes.sh @@ -23,13 +23,17 @@ # CSI_RELEASE_TOKEN: Github token needed for generating release notes # GITHUB_USER: Github username to create PRs with # +# Required tools: +# - gh +# - release-notes (https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md) +# # Instructions: -# 1. Login with "gh auth login" -# 2. Copy this script to the kubernetes-csi directory (one directory above the -# repos) -# 3. Update the repos and versions in the $releases array -# 4. Set environment variables -# 5. Run script from the kubernetes-csi directory +# 1. Install the required tools +# 2. Login with "gh auth login" +# 3. Copy this script to the kubernetes-csi directory (one directory above the repos) +# 4. Update the repos and versions in the $releases array +# 5. Set environment variables +# 6. Run script from the kubernetes-csi directory # # Caveats: # - This script doesn't handle regenerating and updating existing PRs yet. diff --git a/prow.sh b/prow.sh index 049fb79af..67d8dfb2a 100755 --- a/prow.sh +++ b/prow.sh @@ -86,7 +86,7 @@ configvar CSI_PROW_BUILD_PLATFORMS "linux amd64 amd64; linux ppc64le ppc64le -pp # which is disabled with GOFLAGS=-mod=vendor). configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory" -configvar CSI_PROW_GO_VERSION_BUILD "1.22.3" "Go version for building the component" # depends on component's source code +configvar CSI_PROW_GO_VERSION_BUILD "1.23.1" "Go version for building the component" # depends on component's source code configvar CSI_PROW_GO_VERSION_E2E "" "override Go version for building the Kubernetes E2E test suite" # normally doesn't need to be set, see install_e2e configvar CSI_PROW_GO_VERSION_SANITY "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building the csi-sanity test suite" # depends on CSI_PROW_SANITY settings below configvar CSI_PROW_GO_VERSION_KIND "${CSI_PROW_GO_VERSION_BUILD}" "Go version for building 'kind'" # depends on CSI_PROW_KIND_VERSION below @@ -199,7 +199,7 @@ kindest/node:v1.18.20@sha256:738cdc23ed4be6cc0b7ea277a2ebcc454c8373d7d8fb991a7fc # If the deployment script is called with CSI_PROW_TEST_DRIVER= as # environment variable, then it must write a suitable test driver configuration # into that file in addition to installing the driver. -configvar CSI_PROW_DRIVER_VERSION "v1.12.0" "CSI driver version" +configvar CSI_PROW_DRIVER_VERSION "v1.15.0" "CSI driver version" configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_DEPLOYMENT_SUFFIX "" "additional suffix in kubernetes-x.yy[suffix].yaml files" @@ -231,8 +231,11 @@ configvar CSI_PROW_E2E_VERSION "$(version_to_git "${CSI_PROW_KUBERNETES_VERSION} configvar CSI_PROW_E2E_REPO "https://github.com/kubernetes/kubernetes" "E2E repo" configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" -# Local path for e2e tests. Set to "none" to disable. -configvar CSI_PROW_SIDECAR_E2E_IMPORT_PATH "none" "CSI Sidecar E2E package" +# Local path & package path for e2e tests. Set to "none" to disable. +# When using versioned go modules, the import path is the module path whereas the path +# should not contain the version and be the directory where the module is checked out. +configvar CSI_PROW_SIDECAR_E2E_IMPORT_PATH "none" "CSI Sidecar E2E package (go import path)" +configvar CSI_PROW_SIDECAR_E2E_PATH "${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}" "CSI Sidecar E2E path (directory)" # csi-sanity testing from the csi-test repo can be run against the installed # CSI driver. For this to work, deploying the driver must expose the Unix domain @@ -422,7 +425,7 @@ die () { exit 1 } -# Ensure that PATH has the desired version of the Go tools, then run command given as argument. +# Ensure we use the desired version of the Go tools, then run command given as argument. # Empty parameter uses the already installed Go. In Prow, that version is kept up-to-date by # bumping the container image regularly. run_with_go () { @@ -430,15 +433,16 @@ run_with_go () { version="$1" shift - if ! [ "$version" ] || go version 2>/dev/null | grep -q "go$version"; then - run "$@" - else - if ! [ -d "${CSI_PROW_WORK}/go-$version" ]; then - run curl --fail --location "https://dl.google.com/go/go$version.linux-amd64.tar.gz" | tar -C "${CSI_PROW_WORK}" -zxf - || die "installation of Go $version failed" - mv "${CSI_PROW_WORK}/go" "${CSI_PROW_WORK}/go-$version" + if [ "$version" ]; then + version=go$version + if [ "$(GOTOOLCHAIN=$version go version | cut -d' ' -f3)" != "$version" ]; then + die "Please install Go 1.21+" fi - PATH="${CSI_PROW_WORK}/go-$version/bin:$PATH" run "$@" + else + version=local fi + # Set GOMODCACHE to make sure Kubernetes does not need to download again. + GOTOOLCHAIN=$version GOMODCACHE="$(go env GOMODCACHE)" run "$@" } # Ensure that we have the desired version of kind. @@ -621,7 +625,7 @@ start_cluster () { go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes" # Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910 # shellcheck disable=SC2046 - (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image --image csiprow/node:latest --kube-root "${CSI_PROW_WORK}/src/kubernetes") || die "'kind build node-image' failed" + (cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image "${CSI_PROW_WORK}/src/kubernetes" --image csiprow/node:latest) || die "'kind build node-image' failed" csi_prow_kind_have_kubernetes=true fi image="csiprow/node:latest" @@ -1035,7 +1039,7 @@ run_e2e () ( trap move_junit EXIT if [ "${name}" == "local" ]; then - cd "${GOPATH}/src/${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}" && + cd "${GOPATH}/src/${CSI_PROW_SIDECAR_E2E_PATH}" && run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo --timeout="${CSI_PROW_GINKGO_TIMEOUT}" -v "$@" "${CSI_PROW_WORK}/e2e-local.test" -- -report-dir "${ARTIFACTS}" -report-prefix local else cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && diff --git a/pull-test.sh b/pull-test.sh index 5ea051eca..80317720d 100755 --- a/pull-test.sh +++ b/pull-test.sh @@ -20,6 +20,11 @@ set -ex +# Prow checks out repos with --filter=blob:none. This breaks +# "git subtree pull" unless we enable fetching missing file content. +GIT_NO_LAZY_FETCH=0 +export GIT_NO_LAZY_FETCH + # It must be called inside the updated csi-release-tools repo. CSI_RELEASE_TOOLS_DIR="$(pwd)"