diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..2f526620ac --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.github/ +.idea/ +bin/ +hack/ +charts/ +deploy/ +docs/ +examples/ diff --git a/.github/workflows/publish-ecr.yaml b/.github/workflows/publish-ecr.yaml index 1c32b44ee4..e6a3267431 100644 --- a/.github/workflows/publish-ecr.yaml +++ b/.github/workflows/publish-ecr.yaml @@ -39,7 +39,7 @@ jobs: echo "VERSION=${GITHUB_REF_NAME}" >> $GITHUB_ENV - name: Build, tag, and push manifest to Amazon ECR - run: make -j `nproc` all-push + run: make -j `nproc` all-push-with-a1compat ecr-public: name: Push to ECR Public diff --git a/.gitignore b/.gitignore index 000ff70b0d..4be608e597 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,7 @@ vendor/ # Files used by Makefile when upgrading sidecars hack/release-scripts/image-digests.yaml + +# E2E artifacts +_rundir/ +_artifacts/ diff --git a/Dockerfile b/Dockerfile index 5ea6b1b045..a60b4be999 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ COPY . . ARG TARGETOS ARG TARGETARCH ARG VERSION -RUN OS=$TARGETOS ARCH=$TARGETARCH make $TARGETOS/$TARGETARCH +RUN OS=$TARGETOS ARCH=$TARGETARCH make FROM public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-csi-ebs:latest-al23 AS linux-al2023 COPY --from=builder /go/src/github.com/kubernetes-sigs/aws-ebs-csi-driver/bin/aws-ebs-csi-driver /bin/aws-ebs-csi-driver diff --git a/Makefile b/Makefile index 32b52c84dd..f4a761c880 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Copyright 2019 The Kubernetes Authors. +# Copyright 2023 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,29 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -VERSION?=v1.25.0 +## Variables/Functions + +VERSION?=v1.19.0 PKG=github.com/kubernetes-sigs/aws-ebs-csi-driver GIT_COMMIT?=$(shell git rev-parse HEAD) BUILD_DATE?=$(shell date -u -Iseconds) - LDFLAGS?="-X ${PKG}/pkg/driver.driverVersion=${VERSION} -X ${PKG}/pkg/cloud.driverVersion=${VERSION} -X ${PKG}/pkg/driver.gitCommit=${GIT_COMMIT} -X ${PKG}/pkg/driver.buildDate=${BUILD_DATE} -s -w" -GO111MODULE=on -GOPATH=$(shell go env GOPATH) -GOOS=$(shell go env GOOS) -GOBIN=$(shell pwd)/bin +OS?=$(shell go env GOHOSTOS) +ARCH?=$(shell go env GOHOSTARCH) +ifeq ($(OS),windows) + BINARY=aws-ebs-csi-driver.exe + OSVERSION?=ltsc2022 +else + BINARY=aws-ebs-csi-driver + OSVERSION?=al2023 +endif + +GO_SOURCES=go.mod go.sum $(shell find pkg cmd -type f -name "*.go") REGISTRY?=gcr.io/k8s-staging-provider-aws IMAGE?=$(REGISTRY)/aws-ebs-csi-driver TAG?=$(GIT_COMMIT) -OUTPUT_TYPE?=docker - -OS?=linux -ARCH?=amd64 -OSVERSION?=al2023 - ALL_OS?=linux windows ALL_ARCH_linux?=amd64 arm64 ALL_OSVERSION_linux?=al2023 @@ -43,253 +45,193 @@ ALL_OS_ARCH_OSVERSION_linux=$(foreach arch, $(ALL_ARCH_linux), $(foreach osversi ALL_ARCH_windows?=amd64 ALL_OSVERSION_windows?=ltsc2019 ltsc2022 ALL_OS_ARCH_OSVERSION_windows=$(foreach arch, $(ALL_ARCH_windows), $(foreach osversion, ${ALL_OSVERSION_windows}, windows-$(arch)-${osversion})) - ALL_OS_ARCH_OSVERSION=$(foreach os, $(ALL_OS), ${ALL_OS_ARCH_OSVERSION_${os}}) +CLUSTER_NAME?=ebs-csi-e2e.k8s.local +CLUSTER_TYPE?=kops +WINDOWS?=false + # split words on hyphen, access by 1-index word-hyphen = $(word $2,$(subst -, ,$1)) .EXPORT_ALL_VARIABLES: -.PHONY: linux/$(ARCH) bin/aws-ebs-csi-driver -linux/$(ARCH): bin/aws-ebs-csi-driver -bin/aws-ebs-csi-driver: | bin - CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) go build -mod=mod -ldflags ${LDFLAGS} -o bin/aws-ebs-csi-driver ./cmd/ - -.PHONY: windows/$(ARCH) bin/aws-ebs-csi-driver.exe -windows/$(ARCH): bin/aws-ebs-csi-driver.exe -bin/aws-ebs-csi-driver.exe: | bin - CGO_ENABLED=0 GOOS=windows GOARCH=$(ARCH) go build -mod=mod -ldflags ${LDFLAGS} -o bin/aws-ebs-csi-driver.exe ./cmd/ - -# Builds all linux images (not windows because it can't be exported with OUTPUT_TYPE=docker) -.PHONY: all -all: all-image-docker - -# Builds all linux and windows images and pushes them -.PHONY: all-push -all-push: all-image-registry push-manifest - -.PHONY: push-manifest -push-manifest: create-manifest - docker manifest push --purge $(IMAGE):$(TAG) +## Default target +# When no target is supplied, make runs the first target that does not begin with a . +# Alias that to building the binary +.PHONY: default +default: bin/$(BINARY) -.PHONY: create-manifest -create-manifest: all-image-registry -# sed expression: -# LHS: match 0 or more not space characters -# RHS: replace with $(IMAGE):$(TAG)-& where & is what was matched on LHS - docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_OS_ARCH_OSVERSION) | sed -e "s~[^ ]*~$(IMAGE):$(TAG)\-&~g") - -# Only linux for OUTPUT_TYPE=docker because windows image cannot be exported -# "Currently, multi-platform images cannot be exported with the docker export type. The most common usecase for multi-platform images is to directly push to a registry (see registry)." -# https://docs.docker.com/engine/reference/commandline/buildx_build/#output -.PHONY: all-image-docker -all-image-docker: $(addprefix sub-image-docker-,$(ALL_OS_ARCH_OSVERSION_linux)) -.PHONY: all-image-registry -all-image-registry: sub-image-registry-linux-arm64-al2 $(addprefix sub-image-registry-,$(ALL_OS_ARCH_OSVERSION)) - -sub-image-%: - $(MAKE) OUTPUT_TYPE=$(call word-hyphen,$*,1) OS=$(call word-hyphen,$*,2) ARCH=$(call word-hyphen,$*,3) OSVERSION=$(call word-hyphen,$*,4) image - -.PHONY: image -image: .image-$(TAG)-$(OS)-$(ARCH)-$(OSVERSION) -.image-$(TAG)-$(OS)-$(ARCH)-$(OSVERSION): - docker buildx build \ - --platform=$(OS)/$(ARCH) \ - --progress=plain \ - --target=$(OS)-$(OSVERSION) \ - --output=type=$(OUTPUT_TYPE) \ - -t=$(IMAGE):$(TAG)-$(OS)-$(ARCH)-$(OSVERSION) \ - --build-arg=GOPROXY=$(GOPROXY) \ - --build-arg=VERSION=$(VERSION) \ - `./hack/provenance` \ - . - touch $@ +## Top level targets +# Targets intended to be executed by humans via `make target` +# See docs/makefile.md for more information .PHONY: clean clean: - rm -rf .*image-* bin/ + rm -rf bin/ -bin /tmp/helm /tmp/kubeval: - @mkdir -p $@ - -bin/helm: | /tmp/helm bin - @curl -o /tmp/helm/helm.tar.gz -sSL https://get.helm.sh/helm-v3.11.2-${GOOS}-amd64.tar.gz - @tar -zxf /tmp/helm/helm.tar.gz -C bin --strip-components=1 - @rm -rf /tmp/helm/* +.PHONY: test +test: + go test -v -race ./cmd/... ./pkg/... -bin/kubeval: | /tmp/kubeval bin - @curl -o /tmp/kubeval/kubeval.tar.gz -sSL https://github.com/instrumenta/kubeval/releases/download/0.16.1/kubeval-linux-amd64.tar.gz - @tar -zxf /tmp/kubeval/kubeval.tar.gz -C bin kubeval - @rm -rf /tmp/kubeval/* +.PHONY: test-sanity +test-sanity: + echo "succeed" -bin/mockgen: | bin - go install github.com/golang/mock/mockgen@v1.6.0 +.PHONY: tools +tools: bin/aws bin/ct bin/eksctl bin/ginkgo bin/golangci-lint bin/helm bin/kops bin/kubetest2 bin/mockgen -bin/golangci-lint: | bin - echo "Installing golangci-lint..." - curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.51.2 +.PHONY: update +update: update/gofmt update/kustomize update/mockgen update/gomod -.PHONY: kubeval -kubeval: bin/kubeval - bin/kubeval -d deploy/kubernetes/base,deploy/kubernetes/cluster,deploy/kubernetes/overlays -i kustomization.yaml,crd_.+\.yaml,controller_add +.PHONY: verify +verify: verify/golangci-lint verify/govet verify/update -.PHONY: mockgen -mockgen: bin/mockgen - ./hack/update-gomock +.PHONY: all-push +all-push: all-image-registry push-manifest -.PHONY: verify -verify: bin/golangci-lint - echo "verifying and linting files ..." - ./hack/verify-all - echo "Congratulations! All Go source files have been linted." +.PHONY: cluster/create +cluster/create: + ./hack/e2e/create-cluster.sh -.PHONY: test -test: - go test -v -race ./cmd/... ./pkg/... +.PHONY: cluster/delete +cluster/delete: + ./hack/e2e/delete-cluster.sh -.PHONY: test-sanity -test-sanity: - #go test -v ./tests/sanity/... - echo "succeed" +## E2E targets +# Targets to run e2e tests -.PHONY: test-e2e-single-az -test-e2e-single-az: - AWS_REGION=us-west-2 \ +.PHONY: e2e/single-az +e2e/single-az: AWS_AVAILABILITY_ZONES=us-west-2a \ - HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \ - EBS_INSTALL_SNAPSHOT="true" \ TEST_PATH=./tests/e2e/... \ GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" \ - GINKGO_SKIP="\"sc1\"|\"st1\"" \ ./hack/e2e/run.sh -.PHONY: test-e2e-multi-az -test-e2e-multi-az: - AWS_REGION=us-west-2 \ - AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \ - HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \ - EBS_INSTALL_SNAPSHOT="true" \ +.PHONY: e2e/multi-az +e2e/multi-az: TEST_PATH=./tests/e2e/... \ GINKGO_FOCUS="\[ebs-csi-e2e\] \[multi-az\]" \ ./hack/e2e/run.sh -.PHONY: test-e2e-external -test-e2e-external: - AWS_REGION=us-west-2 \ - AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \ - HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \ - EBS_INSTALL_SNAPSHOT="true" \ - TEST_PATH=./tests/e2e-kubernetes/... \ - GINKGO_FOCUS="External.Storage" \ - GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \ +.PHONY: e2e/external +e2e/external: COLLECT_METRICS="true" \ ./hack/e2e/run.sh -.PHONY: test-e2e-external-arm64 -test-e2e-external-arm64: - AWS_REGION=us-west-2 \ - AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \ - HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \ - EBS_INSTALL_SNAPSHOT="true" \ - TEST_PATH=./tests/e2e-kubernetes/... \ - GINKGO_FOCUS="External.Storage" \ - GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \ +.PHONY: e2e/external-arm64 +e2e/external-arm64: INSTANCE_TYPE="m7g.medium" \ AMI_PARAMETER="/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64" \ IMAGE_ARCH="arm64" \ ./hack/e2e/run.sh -.PHONY: test-e2e-external-eks -test-e2e-external-eks: - CLUSTER_TYPE=eksctl \ - HELM_VALUES_FILE="./hack/values_eksctl.yaml" \ - HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \ - EBS_INSTALL_SNAPSHOT="true" \ - EKSCTL_ADMIN_ROLE="Infra-prod-KopsDeleteAllLambdaServiceRoleF1578477-1ELDFIB4KCMXV" \ - AWS_REGION=us-west-2 \ - AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b \ - TEST_PATH=./tests/e2e-kubernetes/... \ - GINKGO_FOCUS="External.Storage" \ - GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \ - ./hack/e2e/run.sh - -.PHONY: test-e2e-external-eks-windows -test-e2e-external-eks-windows: - CLUSTER_TYPE=eksctl \ +.PHONY: e2e/external-windows +e2e/external-windows: WINDOWS=true \ - HELM_VALUES_FILE="./hack/values_eksctl.yaml" \ - HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \ - EKSCTL_ADMIN_ROLE="Infra-prod-KopsDeleteAllLambdaServiceRoleF1578477-1ELDFIB4KCMXV" \ - AWS_REGION=us-west-2 \ - AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b \ - TEST_PATH=./tests/e2e-kubernetes/... \ - GINKGO_FOCUS="External.Storage" \ GINKGO_SKIP="\[Disruptive\]|\[Serial\]|\[LinuxOnly\]|\[Feature:VolumeSnapshotDataSource\]|\(xfs\)|\(ext4\)|\(block volmode\)" \ GINKGO_PARALLEL=15 \ - NODE_OS_DISTRO="windows" \ + EBS_INSTALL_SNAPSHOT="false" \ ./hack/e2e/run.sh -.PHONY: test-e2e-external-kustomize -test-e2e-external-kustomize: - AWS_REGION=us-west-2 \ - AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \ - EBS_INSTALL_SNAPSHOT="true" \ - TEST_PATH=./tests/e2e-kubernetes/... \ - GINKGO_FOCUS="External.Storage" \ - GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \ +.PHONY: e2e/external-kustomize +e2e/external-kustomize: DEPLOY_METHOD="kustomize" \ ./hack/e2e/run.sh -.PHONY: test-helm-chart -test-helm-chart: - AWS_REGION=us-west-2 \ - AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \ - EBS_INSTALL_SNAPSHOT="true" \ +.PHONY: e2e/helm-ct +e2e/helm-chart: HELM_CT_TEST="true" \ ./hack/e2e/run.sh -.PHONY: verify-vendor -test: verify-vendor -verify: verify-vendor -verify-vendor: - @ echo; echo "### $@:" - @ ./hack/verify-vendor.sh - -.PHONY: verify-kustomize -verify: verify-kustomize -verify-kustomize: - @ echo; echo "### $@:" - @ ./hack/verify-kustomize - -.PHONY: generate-kustomize -generate-kustomize: bin/helm - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-attacher.yaml > ../../deploy/kubernetes/base/clusterrole-attacher.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-csi-node.yaml > ../../deploy/kubernetes/base/clusterrole-csi-node.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-provisioner.yaml > ../../deploy/kubernetes/base/clusterrole-provisioner.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-resizer.yaml > ../../deploy/kubernetes/base/clusterrole-resizer.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-snapshotter.yaml > ../../deploy/kubernetes/base/clusterrole-snapshotter.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-attacher.yaml > ../../deploy/kubernetes/base/clusterrolebinding-attacher.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-csi-node.yaml > ../../deploy/kubernetes/base/clusterrolebinding-csi-node.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-provisioner.yaml > ../../deploy/kubernetes/base/clusterrolebinding-provisioner.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-resizer.yaml > ../../deploy/kubernetes/base/clusterrolebinding-resizer.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-snapshotter.yaml > ../../deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/controller.yaml --api-versions 'snapshot.storage.k8s.io/v1' --set 'controller.userAgentExtra=kustomize' | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/controller.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/csidriver.yaml > ../../deploy/kubernetes/base/csidriver.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/node.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/node.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/poddisruptionbudget-controller.yaml --api-versions 'policy/v1/PodDisruptionBudget' | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/poddisruptionbudget-controller.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/serviceaccount-csi-controller.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/serviceaccount-csi-controller.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/serviceaccount-csi-node.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/serviceaccount-csi-node.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/role-leases.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/role-leases.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/rolebinding-leases.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/rolebinding-leases.yaml - -.PHONY: update-truth-sidecars -update-truth-sidecars: hack/release-scripts/get-latest-sidecar-images - ./hack/release-scripts/get-latest-sidecar-images - -.PHONY: generate-sidecar-tags -generate-sidecar-tags: update-truth-sidecars charts/aws-ebs-csi-driver/values.yaml deploy/kubernetes/overlays/stable/gcr/kustomization.yaml hack/release-scripts/generate-sidecar-tags - ./hack/release-scripts/generate-sidecar-tags - -.PHONY: update-sidecar-dependencies -update-sidecar-dependencies: update-truth-sidecars generate-sidecar-tags generate-kustomize +## CI aliases +# Targets intended to be executed mostly or only by CI jobs + +.PHONY: all-push-with-a1compat +all-push-with-a1compat: sub-image-linux-arm64-al2 all-image-registry push-manifest + +test-e2e-%: + ./hack/prow-e2e.sh $* + +test-helm-chart: + ./hack/prow-e2e.sh test-helm-chart + +## Builds + +bin/$(BINARY): $(GO_SOURCES) | bin + CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -mod=readonly -ldflags ${LDFLAGS} -o $@ ./cmd/ + +.PHONY: all-image-registry +all-image-registry: $(addprefix sub-image-,$(ALL_OS_ARCH_OSVERSION)) + +sub-image-%: + $(MAKE) OS=$(call word-hyphen,$*,1) ARCH=$(call word-hyphen,$*,2) OSVERSION=$(call word-hyphen,$*,3) image + +.PHONY: image +image: + docker buildx build \ + --platform=$(OS)/$(ARCH) \ + --progress=plain \ + --target=$(OS)-$(OSVERSION) \ + --output=type=registry \ + -t=$(IMAGE):$(TAG)-$(OS)-$(ARCH)-$(OSVERSION) \ + --build-arg=GOPROXY=$(GOPROXY) \ + --build-arg=VERSION=$(VERSION) \ + `./hack/provenance.sh` \ + . + +.PHONY: create-manifest +create-manifest: all-image-registry +# sed expression: +# LHS: match 0 or more not space characters +# RHS: replace with $(IMAGE):$(TAG)-& where & is what was matched on LHS + docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_OS_ARCH_OSVERSION) | sed -e "s~[^ ]*~$(IMAGE):$(TAG)\-&~g") + +.PHONY: push-manifest +push-manifest: create-manifest + docker manifest push --purge $(IMAGE):$(TAG) + +## Tools +# Tools necessary to perform other targets + +bin/%: hack/tools/install.sh hack/tools/python-runner.sh + @TOOLS_PATH="$(shell pwd)/bin" ./hack/tools/install.sh $* + +## Updaters +# Automatic generators/formatters for code + +.PHONY: update/gofmt +update/gofmt: + gofmt -s -w . + +.PHONY: update/kustomize +update/kustomize: bin/helm + ./hack/update-kustomize.sh + +.PHONY: update/mockgen +update/mockgen: bin/mockgen + ./hack/update-mockgen.sh + +.PHONY: update/gomod +update/gomod: + go mod tidy + +# Verifiers + +.PHONY: verify/golangci-lint +verify/golangci-lint: bin/golangci-lint + ./bin/golangci-lint run --deadline=10m + +.PHONY: verify/govet +verify/govet: + go vet $$(go list ./...) + +.PHONY: verify/update +verify/update: bin/helm bin/mockgen + ./hack/verify-update.sh + +## Internal + +# Directories +bin: + @mkdir -p $@ diff --git a/deploy/kubernetes/base/clusterrolebinding-attacher.yaml b/deploy/kubernetes/base/clusterrolebinding-attacher.yaml index 5715d2651b..12405a09b1 100644 --- a/deploy/kubernetes/base/clusterrolebinding-attacher.yaml +++ b/deploy/kubernetes/base/clusterrolebinding-attacher.yaml @@ -9,7 +9,6 @@ metadata: subjects: - kind: ServiceAccount name: ebs-csi-controller-sa - namespace: default roleRef: kind: ClusterRole name: ebs-external-attacher-role diff --git a/deploy/kubernetes/base/clusterrolebinding-csi-node.yaml b/deploy/kubernetes/base/clusterrolebinding-csi-node.yaml index 095db52510..4cc33b7e82 100644 --- a/deploy/kubernetes/base/clusterrolebinding-csi-node.yaml +++ b/deploy/kubernetes/base/clusterrolebinding-csi-node.yaml @@ -9,7 +9,6 @@ metadata: subjects: - kind: ServiceAccount name: ebs-csi-node-sa - namespace: default roleRef: kind: ClusterRole name: ebs-csi-node-role diff --git a/deploy/kubernetes/base/clusterrolebinding-provisioner.yaml b/deploy/kubernetes/base/clusterrolebinding-provisioner.yaml index 3544bc61e2..e95a94b962 100644 --- a/deploy/kubernetes/base/clusterrolebinding-provisioner.yaml +++ b/deploy/kubernetes/base/clusterrolebinding-provisioner.yaml @@ -9,7 +9,6 @@ metadata: subjects: - kind: ServiceAccount name: ebs-csi-controller-sa - namespace: default roleRef: kind: ClusterRole name: ebs-external-provisioner-role diff --git a/deploy/kubernetes/base/clusterrolebinding-resizer.yaml b/deploy/kubernetes/base/clusterrolebinding-resizer.yaml index c80a9a26bf..543086e74c 100644 --- a/deploy/kubernetes/base/clusterrolebinding-resizer.yaml +++ b/deploy/kubernetes/base/clusterrolebinding-resizer.yaml @@ -9,7 +9,6 @@ metadata: subjects: - kind: ServiceAccount name: ebs-csi-controller-sa - namespace: default roleRef: kind: ClusterRole name: ebs-external-resizer-role diff --git a/deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml b/deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml index 7946414d59..81ed7c2b80 100644 --- a/deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml +++ b/deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml @@ -9,7 +9,6 @@ metadata: subjects: - kind: ServiceAccount name: ebs-csi-controller-sa - namespace: default roleRef: kind: ClusterRole name: ebs-external-snapshotter-role diff --git a/hack/.gitignore b/hack/.gitignore deleted file mode 100644 index 3cc5e8f053..0000000000 --- a/hack/.gitignore +++ /dev/null @@ -1 +0,0 @@ -ebs-e2e-test/ diff --git a/hack/e2e/README.md b/hack/e2e/README.md deleted file mode 100644 index 914f671b68..0000000000 --- a/hack/e2e/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Usage - -run.sh will build and push a driver image, create a kops cluster, helm install the driver pointing to the built image, run ginkgo tests, then clean everything up. - -See below for an example. - -KOPS_STATE_FILE is an S3 bucket you have write access to. - -TEST_ID is a token used for idempotency. - -For more details, see the script itself. - -For more examples, see the top-level Makefile. - -``` -TEST_PATH=./tests/e2e-migration/... \ -EBS_CHECK_MIGRATION=true \ -TEST_ID=18512 \ -CLEAN=false \ -KOPS_STATE_FILE=s3://mattwon \ -AWS_REGION=us-west-2 \ -AWS_AVAILABILITY_ZONES=us-west-2a \ -GINKGO_FOCUS=Dynamic.\*xfs.\*should.store.data \ -GINKGO_NODES=1 \ -./hack/e2e/run.sh -``` - -# git read-tree - -Reference: https://stackoverflow.com/questions/23937436/add-subdirectory-of-remote-repo-with-git-subtree - -How to consume this directory by read-treeing the ebs repo: - -``` -git remote add ebs git@github.com:kubernetes-sigs/aws-ebs-csi-driver.git --no-tags -git fetch ebs -git read-tree --prefix=hack/e2e/ -u ebs/master:hack/e2e -``` - -To commit changes and submit them as a PR back to the ebs repo: - -``` -git diff ebs/master:hack/e2e HEAD:hack/e2e > /tmp/hack_e2e.diff -pushd $GOPATH/src/github.com/kubernetes-sigs/aws-ebs-csi-driver -git apply --reject --directory hack/e2e /tmp/hack_e2e.diff -git commit -``` - -To consume newer changes from the ebs repo: - -``` -git fetch ebs -git diff HEAD:hack/e2e ebs/master:hack/e2e > /tmp/hack_e2e.diff -git apply --reject --directory hack/e2e /tmp/hack_e2e.diff -git add hack/e2e -git commit -m "Update hack/e2e" -``` diff --git a/hack/e2e/chart-testing.sh b/hack/e2e/chart-testing.sh deleted file mode 100644 index 8e7011c2e6..0000000000 --- a/hack/e2e/chart-testing.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -uo pipefail - -function ct_install() { - INSTALL_PATH=${1} - CHART_TESTING_VERSION=${2} - if [[ ! -e ${INSTALL_PATH}/chart-testing ]]; then - CHART_TESTING_DOWNLOAD_URL="https://github.com/helm/chart-testing/releases/download/v${CHART_TESTING_VERSION}/chart-testing_${CHART_TESTING_VERSION}_linux_amd64.tar.gz" - curl --silent --location "${CHART_TESTING_DOWNLOAD_URL}" | tar xz -C "${INSTALL_PATH}" - chmod +x "${INSTALL_PATH}"/ct - fi - apt-get update && apt-get install -y yamllint -} diff --git a/hack/e2e/config.sh b/hack/e2e/config.sh new file mode 100644 index 0000000000..576a96527a --- /dev/null +++ b/hack/e2e/config.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" +TEST_DIR="${BASE_DIR}/csi-test-artifacts" +mkdir -p "${TEST_DIR}" +CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml +KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.kubeconfig"} + +AWS_REGION=${AWS_REGION:-us-west-2} +ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c} +FIRST_ZONE=$(echo "${ZONES}" | cut -d, -f1) +NODE_COUNT=${NODE_COUNT:-3} +INSTANCE_TYPE=${INSTANCE_TYPE:-c5.large} +WINDOWS=${WINDOWS:-"false"} + +# kops: must include patch version (e.g. 1.19.1) +# eksctl: mustn't include patch version (e.g. 1.19) +K8S_VERSION_KOPS=${K8S_VERSION_KOPS:-1.28.3} +K8S_VERSION_EKSCTL=${K8S_VERSION_EKSCTL:-1.28} + +EBS_INSTALL_SNAPSHOT=${EBS_INSTALL_SNAPSHOT:-"true"} +EBS_INSTALL_SNAPSHOT_VERSION=${EBS_INSTALL_SNAPSHOT_VERSION:-"v6.3.2"} + +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +KOPS_BUCKET=${KOPS_BUCKET:-${AWS_ACCOUNT_ID}-ebs-csi-e2e-kops} + +AMI_PARAMETER=${AMI_PARAMETER:-/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64} +AMI_ID=$(aws ssm get-parameters --names ${AMI_PARAMETER} --region ${AWS_REGION} --query 'Parameters[0].Value' --output text) + +CREATE_MISSING_ECR_REPO=${CREATE_MISSING_ECR_REPO:-"true"} +IMAGE_NAME=${IMAGE_NAME:-${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/aws-ebs-csi-driver} +IMAGE_TAG=${IMAGE_TAG:-$(md5sum <<< "${CLUSTER_NAME}.${CLUSTER_TYPE}" | awk '{ print $1 }')} +IMAGE_ARCH=${IMAGE_ARCH:-amd64} + +DEPLOY_METHOD=${DEPLOY_METHOD:-"helm"} +HELM_CT_TEST=${HELM_CT_TEST:-"false"} +HELM_EXTRA_FLAGS=${HELM_EXTRA_FLAGS:-} +COLLECT_METRICS=${COLLECT_METRICS:-"false"} + +TEST_PATH=${TEST_PATH:-"./tests/e2e-kubernetes/..."} +GINKGO_FOCUS=${GINKGO_FOCUS:-"External.Storage"} +GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]"} +GINKGO_PARALLEL=${GINKGO_PARALLEL:-25} + +# TODO: Left in for now, but look into if this is still necessary and remove if not +EKSCTL_ADMIN_ROLE=${EKSCTL_ADMIN_ROLE:-"Infra-prod-KopsDeleteAllLambdaServiceRoleF1578477-1ELDFIB4KCMXV"} + diff --git a/hack/e2e/create-cluster.sh b/hack/e2e/create-cluster.sh new file mode 100755 index 0000000000..b650c3d596 --- /dev/null +++ b/hack/e2e/create-cluster.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script creates a cluster for use of running the e2e tests +# CLUSTER_NAME and CLUSTER_TYPE are expected to be specified by the caller +# All other environment variables have default values (see config.sh) but +# many can be overridden on demand if needed + +set -euo pipefail + +BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" +BIN="${BASE_DIR}/../../bin" + +source "${BASE_DIR}/config.sh" +source "${BASE_DIR}/util.sh" +source "${BASE_DIR}/kops/kops.sh" +source "${BASE_DIR}/eksctl/eksctl.sh" + +if [[ "${CLUSTER_TYPE}" == "kops" ]]; then + BUCKET_CHECK=$("${BIN}/aws" s3api head-bucket --region us-east-1 --bucket "${KOPS_BUCKET}" 2>&1 || true) + if grep -q "Forbidden" <<< "${BUCKET_CHECK}"; then + echo "Kops requires a S3 bucket in order to store the state" >&2 + echo "This script is attempting to use a bucket called \`${KOPS_BUCKET}\`" >&2 + echo "That bucket already exists and you do not have access to it" >&2 + echo "You can change the bucket by setting the environment variable \$KOPS_BUCKET" >&2 + exit 1 + fi + if grep -q "Not Found" <<< "${BUCKET_CHECK}"; then + "${BIN}/aws" s3api create-bucket --region us-east-1 --bucket "${KOPS_BUCKET}" --acl private >/dev/null + fi + + kops_create_cluster \ + "$CLUSTER_NAME" \ + "${BIN}/kops" \ + "$ZONES" \ + "$NODE_COUNT" \ + "$INSTANCE_TYPE" \ + "$AMI_ID" \ + "$K8S_VERSION_KOPS" \ + "$CLUSTER_FILE" \ + "$KUBECONFIG" \ + "${BASE_DIR}/kops/patch-cluster.yaml" \ + "${BASE_DIR}/kops/patch-node.yaml" \ + "s3://${KOPS_BUCKET}" +elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then + eksctl_create_cluster \ + "$CLUSTER_NAME" \ + "${BIN}/eksctl" \ + "$ZONES" \ + "$INSTANCE_TYPE" \ + "$K8S_VERSION_EKSCTL" \ + "$CLUSTER_FILE" \ + "$KUBECONFIG" \ + "${BASE_DIR}/eksctl/patch.yaml" \ + "$EKSCTL_ADMIN_ROLE" \ + "$WINDOWS" \ + "${BASE_DIR}/eksctl/vpc-resource-controller-configmap.yaml" +else + echo "Cluster type ${CLUSTER_TYPE} is invalid, must be kops or eksctl" >&2 + exit 1 +fi + diff --git a/hack/e2e/delete-cluster.sh b/hack/e2e/delete-cluster.sh new file mode 100755 index 0000000000..dca150d0c5 --- /dev/null +++ b/hack/e2e/delete-cluster.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script deletes a cluster that was created by `create-cluster.sh` +# CLUSTER_NAME and CLUSTER_TYPE are expected to be specified by the caller +# All other environment variables have default values (see config.sh) but +# many can be overridden on demand if needed + +set -euo pipefail + +BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" +BIN="${BASE_DIR}/../../bin" + +source "${BASE_DIR}/config.sh" +source "${BASE_DIR}/util.sh" +source "${BASE_DIR}/kops/kops.sh" +source "${BASE_DIR}/eksctl/eksctl.sh" + +if [[ "${CLUSTER_TYPE}" == "kops" ]]; then + kops_delete_cluster \ + "${BIN}/kops" \ + "${CLUSTER_NAME}" \ + "s3://${KOPS_BUCKET}" +elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then + eksctl_delete_cluster \ + "${BIN}/eksctl" \ + "${CLUSTER_NAME}" +else + echo "Cluster type ${CLUSTER_TYPE} is invalid, must be kops or eksctl" >&2 + exit 1 +fi diff --git a/hack/e2e/ecr.sh b/hack/e2e/ecr.sh index dc82f1ae2c..5999417e40 100644 --- a/hack/e2e/ecr.sh +++ b/hack/e2e/ecr.sh @@ -1,9 +1,20 @@ #!/bin/bash -set -uo pipefail +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") -source "${BASE_DIR}"/util.sh +set -euo pipefail function ecr_build_and_push() { REGION=${1} @@ -11,26 +22,27 @@ function ecr_build_and_push() { IMAGE_NAME=${3} IMAGE_TAG=${4} IMAGE_ARCH=${5} - set +e - if docker images --format "{{.Repository}}:{{.Tag}}" | grep "${IMAGE_NAME}:${IMAGE_TAG}"; then - set -e - loudecho "Assuming ${IMAGE_NAME}:${IMAGE_TAG} has been built and pushed" + + loudecho "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}" + aws ecr get-login-password --region "${REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com" + + # Only setup buildx builder on Prow, allow local users to use docker cache + if [ -n "${PROW_JOB_ID:-}" ]; then + trap "docker buildx rm ebs-csi-multiarch-builder" EXIT + docker buildx create --use --name ebs-csi-multiarch-builder + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + fi + + export IMAGE="${IMAGE_NAME}" + export TAG="${IMAGE_TAG}" + if [[ "$WINDOWS" == true ]]; then + export ALL_OS="linux windows" + export ALL_OSVERSION_windows="ltsc2022" + export ALL_ARCH_linux="amd64" + export ALL_ARCH_windows="${IMAGE_ARCH}" else - set -e - loudecho "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}" - aws ecr get-login-password --region "${REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}".dkr.ecr."${REGION}".amazonaws.com - if [[ "$WINDOWS" == true ]]; then - export DOCKER_CLI_EXPERIMENTAL=enabled - export TAG=${IMAGE_TAG} - export IMAGE=${IMAGE_NAME} - trap "docker buildx rm multiarch-builder" EXIT - docker buildx create --use --name multiarch-builder - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - make all-push - else - IMAGE=${IMAGE_NAME} TAG=${IMAGE_TAG} OS=linux ARCH=${IMAGE_ARCH} OSVERSION=al2023 make image - docker tag "${IMAGE_NAME}":"${IMAGE_TAG}"-linux-${IMAGE_ARCH}-al2023 "${IMAGE_NAME}":"${IMAGE_TAG}" - docker push "${IMAGE_NAME}":"${IMAGE_TAG}" - fi + export ALL_OS="linux" + export ALL_ARCH_linux="${IMAGE_ARCH}" fi + make -j `nproc` all-push } diff --git a/hack/e2e/eksctl.sh b/hack/e2e/eksctl/eksctl.sh similarity index 60% rename from hack/e2e/eksctl.sh rename to hack/e2e/eksctl/eksctl.sh index 6d448dc84c..38a68bf387 100644 --- a/hack/e2e/eksctl.sh +++ b/hack/e2e/eksctl/eksctl.sh @@ -1,20 +1,27 @@ -#!/bin/bash +#!/EKSCTL_BIN/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script updates the kustomize templates in deploy/kubernetes/base/ by +# running `helm template` and stripping the namespace from the output set -euo pipefail -function eksctl_install() { - INSTALL_PATH=${1} - EKSCTL_VERSION=${2} - if [[ ! -e ${INSTALL_PATH}/eksctl ]]; then - EKSCTL_DOWNLOAD_URL="https://github.com/weaveworks/eksctl/releases/download/v${EKSCTL_VERSION}/eksctl_$(uname -s)_amd64.tar.gz" - curl --silent --location "${EKSCTL_DOWNLOAD_URL}" | tar xz -C "${INSTALL_PATH}" - chmod +x "${INSTALL_PATH}"/eksctl - fi -} - function eksctl_create_cluster() { CLUSTER_NAME=${1} - BIN=${2} + EKSCTL_BIN=${2} ZONES=${3} INSTANCE_TYPE=${4} K8S_VERSION=${5} @@ -27,12 +34,12 @@ function eksctl_create_cluster() { CLUSTER_NAME="${CLUSTER_NAME//./-}" - if eksctl_cluster_exists "${CLUSTER_NAME}" "${BIN}"; then + if eksctl_cluster_exists "${CLUSTER_NAME}" "${EKSCTL_BIN}"; then loudecho "Upgrading cluster $CLUSTER_NAME with $CLUSTER_FILE" - ${BIN} upgrade cluster -f "${CLUSTER_FILE}" + ${EKSCTL_BIN} upgrade cluster -f "${CLUSTER_FILE}" else loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)" - ${BIN} create cluster \ + ${EKSCTL_BIN} create cluster \ --managed \ --ssh-access=false \ --zones "${ZONES}" \ @@ -48,22 +55,22 @@ function eksctl_create_cluster() { fi loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE" - ${BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}" + ${EKSCTL_BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}" fi loudecho "Cluster ${CLUSTER_NAME} kubecfg written to ${KUBECONFIG}" loudecho "Getting cluster ${CLUSTER_NAME}" - ${BIN} get cluster "${CLUSTER_NAME}" + ${EKSCTL_BIN} get cluster "${CLUSTER_NAME}" if [[ -n "$EKSCTL_ADMIN_ROLE" ]]; then AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) ADMIN_ARN="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${EKSCTL_ADMIN_ROLE}" loudecho "Granting ${ADMIN_ARN} admin access to the cluster" - ${BIN} create iamidentitymapping --cluster "${CLUSTER_NAME}" --arn "${ADMIN_ARN}" --group system:masters --username admin + ${EKSCTL_BIN} create iamidentitymapping --cluster "${CLUSTER_NAME}" --arn "${ADMIN_ARN}" --group system:masters --username admin fi if [[ "$WINDOWS" == true ]]; then - ${BIN} create nodegroup \ + ${EKSCTL_BIN} create nodegroup \ --managed=true \ --ssh-access=false \ --cluster="${CLUSTER_NAME}" \ @@ -81,9 +88,9 @@ function eksctl_create_cluster() { function eksctl_cluster_exists() { CLUSTER_NAME=${1} - BIN=${2} + EKSCTL_BIN=${2} set +e - if ${BIN} get cluster "${CLUSTER_NAME}"; then + if ${EKSCTL_BIN} get cluster "${CLUSTER_NAME}"; then set -e return 0 else @@ -93,10 +100,13 @@ function eksctl_cluster_exists() { } function eksctl_delete_cluster() { - BIN=${1} + EKSCTL_BIN=${1} CLUSTER_NAME=${2} + + CLUSTER_NAME="${CLUSTER_NAME//./-}" + loudecho "Deleting cluster ${CLUSTER_NAME}" - ${BIN} delete cluster "${CLUSTER_NAME}" + ${EKSCTL_BIN} delete cluster "${CLUSTER_NAME}" } function eksctl_patch_cluster_file() { @@ -112,7 +122,7 @@ function eksctl_patch_cluster_file() { cp "$CLUSTER_FILE" "$CLUSTER_FILE_0" # Patch only the Cluster - kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$EKSCTL_PATCH_FILE")" -o yaml > "$CLUSTER_FILE_1" + kubectl patch --kubeconfig "/dev/null" -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$EKSCTL_PATCH_FILE")" -o yaml > "$CLUSTER_FILE_1" mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0" # Done patching, overwrite original CLUSTER_FILE diff --git a/hack/eksctl-patch.yaml b/hack/e2e/eksctl/patch.yaml similarity index 100% rename from hack/eksctl-patch.yaml rename to hack/e2e/eksctl/patch.yaml diff --git a/hack/values_eksctl.yaml b/hack/e2e/eksctl/values.yaml similarity index 100% rename from hack/values_eksctl.yaml rename to hack/e2e/eksctl/values.yaml diff --git a/hack/vpc-resource-controller-configmap.yaml b/hack/e2e/eksctl/vpc-resource-controller-configmap.yaml similarity index 100% rename from hack/vpc-resource-controller-configmap.yaml rename to hack/e2e/eksctl/vpc-resource-controller-configmap.yaml diff --git a/hack/e2e/helm.sh b/hack/e2e/helm.sh deleted file mode 100644 index 500441e1ae..0000000000 --- a/hack/e2e/helm.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -uo pipefail - -function helm_install() { - INSTALL_PATH=${1} - if [[ ! -e ${INSTALL_PATH}/helm ]]; then - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 get_helm.sh - export USE_SUDO=false - export HELM_INSTALL_DIR=${INSTALL_PATH} - ./get_helm.sh - rm get_helm.sh - fi -} diff --git a/hack/e2e/kops.sh b/hack/e2e/kops/kops.sh similarity index 69% rename from hack/e2e/kops.sh rename to hack/e2e/kops/kops.sh index b5e5dade43..419945d6ca 100644 --- a/hack/e2e/kops.sh +++ b/hack/e2e/kops/kops.sh @@ -1,30 +1,27 @@ -#!/bin/bash +#!/KOPS_BIN/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script updates the kustomize templates in deploy/kubernetes/base/ by +# running `helm template` and stripping the namespace from the output set -euo pipefail -OS_ARCH=$(go env GOOS)-amd64 - -BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") -source "${BASE_DIR}"/util.sh - -function kops_install() { - INSTALL_PATH=${1} - KOPS_VERSION=${2} - if [[ -e "${INSTALL_PATH}"/kops ]]; then - INSTALLED_KOPS_VERSION=$("${INSTALL_PATH}"/kops version) - if [[ "$INSTALLED_KOPS_VERSION" == *"$KOPS_VERSION"* ]]; then - echo "KOPS $INSTALLED_KOPS_VERSION already installed!" - return - fi - fi - KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/v${KOPS_VERSION}/kops-${OS_ARCH} - curl -L -X GET "${KOPS_DOWNLOAD_URL}" -o "${INSTALL_PATH}"/kops - chmod +x "${INSTALL_PATH}"/kops -} - function kops_create_cluster() { CLUSTER_NAME=${1} - BIN=${2} + KOPS_BIN=${2} ZONES=${3} NODE_COUNT=${4} INSTANCE_TYPE=${5} @@ -36,12 +33,12 @@ function kops_create_cluster() { KOPS_PATCH_NODE_FILE=${11} KOPS_STATE_FILE=${12} - if kops_cluster_exists "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}"; then + if kops_cluster_exists "${CLUSTER_NAME}" "${KOPS_BIN}" "${KOPS_STATE_FILE}"; then loudecho "Replacing cluster $CLUSTER_NAME with $CLUSTER_FILE" - ${BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}" + ${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}" else loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)" - ${BIN} create cluster --state "${KOPS_STATE_FILE}" \ + ${KOPS_BIN} create cluster --state "${KOPS_STATE_FILE}" \ --zones "${ZONES}" \ --node-count="${NODE_COUNT}" \ --node-size="${INSTANCE_TYPE}" \ @@ -59,26 +56,26 @@ function kops_create_cluster() { fi loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE" - ${BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}" + ${KOPS_BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}" fi loudecho "Updating cluster $CLUSTER_NAME with $CLUSTER_FILE" - ${BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes + ${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes loudecho "Exporting cluster ${CLUSTER_NAME} kubecfg to ${KUBECONFIG}" - ${BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}" + ${KOPS_BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}" loudecho "Validating cluster ${CLUSTER_NAME}" - ${BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m --kubeconfig "${KUBECONFIG}" + ${KOPS_BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m --kubeconfig "${KUBECONFIG}" return $? } function kops_cluster_exists() { CLUSTER_NAME=${1} - BIN=${2} + KOPS_BIN=${2} KOPS_STATE_FILE=${3} set +e - if ${BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then + if ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then set -e return 0 else @@ -88,11 +85,11 @@ function kops_cluster_exists() { } function kops_delete_cluster() { - BIN=${1} + KOPS_BIN=${1} CLUSTER_NAME=${2} KOPS_STATE_FILE=${3} loudecho "Deleting cluster ${CLUSTER_NAME}" - ${BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes + ${KOPS_BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes } # TODO switch this to python, work exclusively with yaml, use kops toolbox diff --git a/hack/kops-patch.yaml b/hack/e2e/kops/patch-cluster.yaml similarity index 100% rename from hack/kops-patch.yaml rename to hack/e2e/kops/patch-cluster.yaml diff --git a/hack/kops-patch-node.yaml b/hack/e2e/kops/patch-node.yaml similarity index 100% rename from hack/kops-patch-node.yaml rename to hack/e2e/kops/patch-node.yaml diff --git a/hack/values.yaml b/hack/e2e/kops/values.yaml similarity index 100% rename from hack/values.yaml rename to hack/e2e/kops/values.yaml diff --git a/hack/e2e/run.sh b/hack/e2e/run.sh index a083394d15..07ca41376d 100755 --- a/hack/e2e/run.sh +++ b/hack/e2e/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Kubernetes Authors. +# Copyright 2023 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,168 +14,55 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -euo pipefail - -BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") -source "${BASE_DIR}"/ecr.sh -source "${BASE_DIR}"/eksctl.sh -source "${BASE_DIR}"/helm.sh -source "${BASE_DIR}"/kops.sh -source "${BASE_DIR}"/util.sh -source "${BASE_DIR}"/chart-testing.sh -source "${BASE_DIR}"/metrics/metrics.sh - -DRIVER_NAME=${DRIVER_NAME:-aws-ebs-csi-driver} -CONTAINER_NAME=${CONTAINER_NAME:-ebs-plugin} - -TEST_ID=${TEST_ID:-$RANDOM} -CLUSTER_NAME=test-cluster-${TEST_ID}.k8s.local -CLUSTER_TYPE=${CLUSTER_TYPE:-kops} - -TEST_DIR=${BASE_DIR}/csi-test-artifacts -BIN_DIR=${TEST_DIR}/bin -CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml -KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.kubeconfig"} - -REGION=${AWS_REGION:-us-west-2} -ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c} -FIRST_ZONE=$(echo "${ZONES}" | cut -d, -f1) -NODE_COUNT=${NODE_COUNT:-3} -INSTANCE_TYPE=${INSTANCE_TYPE:-c5.large} - -AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) -IMAGE_NAME=${IMAGE_NAME:-${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${DRIVER_NAME}} -IMAGE_TAG=${IMAGE_TAG:-${TEST_ID}} -IMAGE_ARCH=${IMAGE_ARCH:-amd64} - -# kops: must include patch version (e.g. 1.19.1) -# eksctl: mustn't include patch version (e.g. 1.19) -K8S_VERSION_KOPS=${K8S_VERSION_KOPS:-${K8S_VERSION:-1.28.3}} -K8S_VERSION_EKSCTL=${K8S_VERSION_EKSCTL:-${K8S_VERSION:-1.28}} - -KOPS_VERSION=${KOPS_VERSION:-1.28.0} -KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-shared-e2e} -KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-./hack/kops-patch.yaml} -KOPS_PATCH_NODE_FILE=${KOPS_PATCH_NODE_FILE:-./hack/kops-patch-node.yaml} -AMI_PARAMETER=${AMI_PARAMETER:-/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64} -AMI_ID=$(aws ssm get-parameters --names ${AMI_PARAMETER} --region ${REGION} --query 'Parameters[0].Value' --output text) - -EKSCTL_VERSION=${EKSCTL_VERSION:-0.164.0} -EKSCTL_PATCH_FILE=${EKSCTL_PATCH_FILE:-./hack/eksctl-patch.yaml} -VPC_CONFIGMAP_FILE=${VPC_CONFIGMAP_FILE:-./hack/vpc-resource-controller-configmap.yaml} -EKSCTL_ADMIN_ROLE=${EKSCTL_ADMIN_ROLE:-} -# Creates a windows node group. -WINDOWS=${WINDOWS:-"false"} +# This script builds and deploys the EBS CSI Driver and runs e2e tests +# CLUSTER_NAME and CLUSTER_TYPE are expected to be specified by the caller +# All other environment variables have default values (see config.sh) but +# many can be overridden on demand if needed -# Valid deploy methods: "helm" (default), "kustomize" -DEPLOY_METHOD=${DEPLOY_METHOD:-"helm"} - -HELM_VALUES_FILE=${HELM_VALUES_FILE:-./hack/values.yaml} -HELM_EXTRA_FLAGS=${HELM_EXTRA_FLAGS:-} - -TEST_PATH=${TEST_PATH:-"./tests/e2e/..."} -ARTIFACTS=${ARTIFACTS:-"${TEST_DIR}/artifacts"} -GINKGO_FOCUS=${GINKGO_FOCUS:-"\[ebs-csi-e2e\]"} -GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]"} -GINKGO_NODES=${GINKGO_NODES:-4} -GINKGO_PARALLEL=${GINKGO_PARALLEL:-25} -NODE_OS_DISTRO=${NODE_OS_DISTRO:-"linux"} -TEST_EXTRA_FLAGS=${TEST_EXTRA_FLAGS:-} - -EBS_INSTALL_SNAPSHOT=${EBS_INSTALL_SNAPSHOT:-"false"} -# https://github.com/kubernetes-csi/external-snapshotter -EBS_INSTALL_SNAPSHOT_VERSION=${EBS_INSTALL_SNAPSHOT_VERSION:-"v6.3.2"} +set -euo pipefail -HELM_CT_TEST=${HELM_CT_TEST:-"false"} -# https://github.com/helm/chart-testing -CHART_TESTING_VERSION=${CHART_TESTING_VERSION:-3.8.0} -CLEAN=${CLEAN:-"true"} +BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" +BIN="${BASE_DIR}/../../bin" -COLLECT_METRICS=${COLLECT_METRICS:-"false"} +source "${BASE_DIR}/config.sh" +source "${BASE_DIR}/util.sh" +source "${BASE_DIR}/ecr.sh" -loudecho "Testing in region ${REGION} and zones ${ZONES}" -mkdir -p "${BIN_DIR}" -export PATH=${PATH}:${BIN_DIR} +## Setup if [[ "${CLUSTER_TYPE}" == "kops" ]]; then - loudecho "Installing kops ${KOPS_VERSION} to ${BIN_DIR}" - kops_install "${BIN_DIR}" "${KOPS_VERSION}" - KOPS_BIN=${BIN_DIR}/kops + HELM_VALUES_FILE="${BASE_DIR}/kops/values.yaml" + K8S_VERSION="${K8S_VERSION_KOPS}" elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then - loudecho "Installing eksctl ${EKSCTL_VERSION} to ${BIN_DIR}" - eksctl_install "${BIN_DIR}" "${EKSCTL_VERSION}" - EKSCTL_BIN=${BIN_DIR}/eksctl + HELM_VALUES_FILE="${BASE_DIR}/eksctl/values.yaml" + K8S_VERSION="${K8S_VERSION_EKSCTL}" else - loudecho "${CLUSTER_TYPE} must be kops or eksctl!" + echo "Cluster type ${CLUSTER_TYPE} is invalid, must be kops or eksctl" >&2 exit 1 fi -loudecho "Installing helm to ${BIN_DIR}" -helm_install "${BIN_DIR}" -HELM_BIN=${BIN_DIR}/helm - -if [[ "${HELM_CT_TEST}" == true ]]; then - loudecho "Installing chart-testing ${CHART_TESTING_VERSION} to ${BIN_DIR}" - ct_install "${BIN_DIR}" "${CHART_TESTING_VERSION}" - CHART_TESTING_BIN=${BIN_DIR}/ct +if [[ "$WINDOWS" == true ]]; then + NODE_OS_DISTRO="windows" else - loudecho "Installing ginkgo to ${BIN_DIR}" - GINKGO_BIN=${BIN_DIR}/ginkgo - if [[ ! -e ${GINKGO_BIN} ]]; then - pushd /tmp - GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} go install github.com/onsi/ginkgo/v2/ginkgo@v2.11.0 - popd - ginkgo version - fi - loudecho "Installing kubetest2 to ${BIN_DIR}" - KUBETEST2_BIN=${BIN_DIR}/kubetest2 - if [[ ! -e ${KUBETEST2_BIN} ]]; then - pushd /tmp - GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} go install sigs.k8s.io/kubetest2/...@latest - popd + NODE_OS_DISTRO="linux" +fi + +## Build image + +if [[ "${CREATE_MISSING_ECR_REPO}" == true ]]; then + REPO_CHECK=$(aws ecr describe-repositories --region "${AWS_REGION}") + if [ $(jq '.repositories | map(.repositoryName) | index("aws-ebs-csi-driver")' <<< "${REPO_CHECK}") == "null" ]; then + aws ecr create-repository --region "${AWS_REGION}" --repository-name aws-ebs-csi-driver > /dev/null fi fi -ecr_build_and_push "${REGION}" \ +ecr_build_and_push "${AWS_REGION}" \ "${AWS_ACCOUNT_ID}" \ "${IMAGE_NAME}" \ "${IMAGE_TAG}" \ "${IMAGE_ARCH}" -if [[ "${CLUSTER_TYPE}" == "kops" ]]; then - kops_create_cluster \ - "$CLUSTER_NAME" \ - "$KOPS_BIN" \ - "$ZONES" \ - "$NODE_COUNT" \ - "$INSTANCE_TYPE" \ - "$AMI_ID" \ - "$K8S_VERSION_KOPS" \ - "$CLUSTER_FILE" \ - "$KUBECONFIG" \ - "$KOPS_PATCH_FILE" \ - "$KOPS_PATCH_NODE_FILE" \ - "$KOPS_STATE_FILE" - if [[ $? -ne 0 ]]; then - exit 1 - fi -elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then - eksctl_create_cluster \ - "$CLUSTER_NAME" \ - "$EKSCTL_BIN" \ - "$ZONES" \ - "$INSTANCE_TYPE" \ - "$K8S_VERSION_EKSCTL" \ - "$CLUSTER_FILE" \ - "$KUBECONFIG" \ - "$EKSCTL_PATCH_FILE" \ - "$EKSCTL_ADMIN_ROLE" \ - "$WINDOWS" \ - "$VPC_CONFIGMAP_FILE" - if [[ $? -ne 0 ]]; then - exit 1 - fi -fi +## Deploy if [[ "${EBS_INSTALL_SNAPSHOT}" == true ]]; then loudecho "Installing snapshot controller and CRDs" @@ -186,37 +73,20 @@ if [[ "${EBS_INSTALL_SNAPSHOT}" == true ]]; then kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml fi -if [[ "${HELM_CT_TEST}" == true ]]; then - loudecho "Test and lint Helm chart with chart-testing" - if [ -n "${PROW_JOB_ID:-}" ]; then - # Prow-specific setup - # Required becuase chart_testing ALWAYS needs a remote - git remote add ct https://github.com/kubernetes-sigs/aws-ebs-csi-driver.git - git fetch ct "${PULL_BASE_REF}" - export CT_REMOTE="ct" - export CT_TARGET_BRANCH="${PULL_BASE_REF}" - fi - set -x - set +e - export KUBECONFIG="${KUBECONFIG}" - ${CHART_TESTING_BIN} lint-and-install --config ${PWD}/tests/ct-config.yaml --helm-extra-set-args="--set=image.repository=${IMAGE_NAME},image.tag=${IMAGE_TAG},node.tolerateAllTaints=false" - TEST_PASSED=$? - set -e - set +x -else - loudecho "Deploying driver via ${DEPLOY_METHOD}" +if [[ "${HELM_CT_TEST}" != true ]]; then startSec=$(date +'%s') if [[ ${DEPLOY_METHOD} == "helm" ]]; then - HELM_ARGS=(upgrade --install "${DRIVER_NAME}" + HELM_ARGS=(upgrade --install "aws-ebs-csi-driver" --namespace kube-system --set image.repository="${IMAGE_NAME}" --set image.tag="${IMAGE_TAG}" --set node.enableWindows="${WINDOWS}" + --set=controller.k8sTagClusterId="${CLUSTER_NAME}" --timeout 10m0s --wait --kubeconfig "${KUBECONFIG}" - ./charts/"${DRIVER_NAME}") + ./charts/aws-ebs-csi-driver) if [[ -f "$HELM_VALUES_FILE" ]]; then HELM_ARGS+=(-f "${HELM_VALUES_FILE}") fi @@ -225,34 +95,57 @@ else HELM_ARGS+=("${EXPANDED_HELM_EXTRA_FLAGS}") fi set -x - "${HELM_BIN}" "${HELM_ARGS[@]}" + "${BIN}/helm" "${HELM_ARGS[@]}" set +x elif [[ ${DEPLOY_METHOD} == "kustomize" ]]; then + set -x kubectl --kubeconfig "${KUBECONFIG}" apply -k "./deploy/kubernetes/overlays/stable" kubectl --kubeconfig "${KUBECONFIG}" --namespace kube-system wait --timeout 10m0s --for "condition=ready" pod -l "app.kubernetes.io/name=aws-ebs-csi-driver" + set +x fi endSec=$(date +'%s') deployTimeSeconds=$(((endSec - startSec) / 1)) loudecho "Driver deployment complete, time used: $deployTimeSeconds seconds" +fi + +## Run tests + +if [[ "${HELM_CT_TEST}" == true ]]; then + loudecho "Test and lint Helm chart with chart-testing" + if [ -n "${PROW_JOB_ID:-}" ]; then + # Prow-specific setup + # Required becuase chart_testing ALWAYS needs a remote + git remote add ct https://github.com/kubernetes-sigs/aws-ebs-csi-driver.git + git fetch ct "${PULL_BASE_REF}" + export CT_REMOTE="ct" + export CT_TARGET_BRANCH="${PULL_BASE_REF}" + fi + set -x + set +e + export KUBECONFIG="${KUBECONFIG}" + ${CHART_TESTING_BIN} lint-and-install --config="${BASE_DIR}/../../tests/ct-config.yaml" --helm-extra-set-args="--set=image.repository=${IMAGE_NAME},image.tag=${IMAGE_TAG},node.tolerateAllTaints=false" + TEST_PASSED=$? + set -e + set +x +else loudecho "Testing focus ${GINKGO_FOCUS}" if [[ $TEST_PATH == "./tests/e2e-kubernetes/..." ]]; then - pushd ${PWD}/tests/e2e-kubernetes + pushd "${BASE_DIR}/../../tests/e2e-kubernetes" packageVersion=$(echo $(cut -d '.' -f 1,2 <<< $K8S_VERSION)) set -x set +e - kubetest2 noop \ + "${BIN}/kubetest2" noop \ --run-id="e2e-kubernetes" \ --test=ginkgo \ -- \ --skip-regex="${GINKGO_SKIP}" \ --focus-regex="${GINKGO_FOCUS}" \ - --test-package-version=$(curl -L https://dl.k8s.io/release/stable-$packageVersion.txt) \ + --test-package-version=$(curl -L https://dl.k8s.io/release/stable-${packageVersion}.txt) \ --parallel=${GINKGO_PARALLEL} \ - --test-args="-storage.testdriver=${PWD}/manifests.yaml -kubeconfig=$KUBECONFIG -node-os-distro=${NODE_OS_DISTRO}" - + --test-args="-storage.testdriver=${PWD}/manifests.yaml -kubeconfig=${KUBECONFIG} -node-os-distro=${NODE_OS_DISTRO}" TEST_PASSED=$? set -e set +x @@ -260,68 +153,67 @@ else fi if [[ $TEST_PATH == "./tests/e2e/..." ]]; then - eval "EXPANDED_TEST_EXTRA_FLAGS=$TEST_EXTRA_FLAGS" set -x set +e - ${GINKGO_BIN} -p -nodes="${GINKGO_NODES}" -v --focus="${GINKGO_FOCUS}" --skip="${GINKGO_SKIP}" "${TEST_PATH}" -- -kubeconfig="${KUBECONFIG}" -report-dir="${ARTIFACTS}" -gce-zone="${FIRST_ZONE}" "${EXPANDED_TEST_EXTRA_FLAGS}" + "${BIN}/ginkgo" -p -nodes="${GINKGO_PARALLEL}" -v \ + --focus="${GINKGO_FOCUS}" \ + --skip="${GINKGO_SKIP}" \ + "${BASE_DIR}/../../tests/e2e/..." \ + -- \ + -kubeconfig="${KUBECONFIG}" \ + -report-dir="${TEST_DIR}/artifacts" \ + -gce-zone="${FIRST_ZONE}" TEST_PASSED=$? set -e set +x fi - PODS=$(kubectl get pod -n kube-system -l "app.kubernetes.io/name=${DRIVER_NAME},app.kubernetes.io/instance=${DRIVER_NAME}" -o json --kubeconfig "${KUBECONFIG}" | jq -r .items[].metadata.name) + PODS=$(kubectl get pod -n kube-system -l "app.kubernetes.io/name=aws-ebs-csi-driver,app.kubernetes.io/instance=aws-ebs-csi-driver" -o json --kubeconfig "${KUBECONFIG}" | jq -r .items[].metadata.name) while IFS= read -r POD; do - loudecho "Printing pod ${POD} ${CONTAINER_NAME} container logs" + loudecho "Printing pod ${POD} container logs" set +e - kubectl logs "${POD}" -n kube-system "${CONTAINER_NAME}" \ - --kubeconfig "${KUBECONFIG}" + kubectl logs "${POD}" -n kube-system --all-containers --ignore-errors --kubeconfig "${KUBECONFIG}" set -e done <<< "${PODS}" fi -OVERALL_TEST_PASSED="${TEST_PASSED}" - if [[ "${COLLECT_METRICS}" == true ]]; then metrics_collector "$KUBECONFIG" \ "$AWS_ACCOUNT_ID" \ "$AWS_REGION" \ "$NODE_OS_DISTRO" \ "$deployTimeSeconds" \ - "$DRIVER_NAME" \ + "aws-ebs-csi-driver" \ "$VERSION" fi -if [[ "${CLEAN}" == true ]]; then - loudecho "Cleaning" +## Cleanup - if [[ "${HELM_CT_TEST}" != true ]]; then +if [[ "${HELM_CT_TEST}" != true ]]; then loudecho "Removing driver via ${DEPLOY_METHOD}" if [[ ${DEPLOY_METHOD} == "helm" ]]; then - ${HELM_BIN} del "${DRIVER_NAME}" \ + ${BIN}/helm del "aws-ebs-csi-driver" \ --namespace kube-system \ --kubeconfig "${KUBECONFIG}" elif [[ ${DEPLOY_METHOD} == "kustomize" ]]; then - kubectl --kubeconfig "${KUBECONFIG}" delete -k "./deploy/kubernetes/overlays/stable" + kubectl --kubeconfig "${KUBECONFIG}" delete -k "${BASE_DIR}/../../deploy/kubernetes/overlays/stable" fi - fi +fi - if [[ "${CLUSTER_TYPE}" == "kops" ]]; then - kops_delete_cluster \ - "${KOPS_BIN}" \ - "${CLUSTER_NAME}" \ - "${KOPS_STATE_FILE}" - elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then - eksctl_delete_cluster \ - "${EKSCTL_BIN}" \ - "${CLUSTER_NAME}" - fi -else - loudecho "Not cleaning" +if [[ "${EBS_INSTALL_SNAPSHOT}" == true ]]; then + loudecho "Installing snapshot controller and CRDs" + kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml + kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml + kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml + kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml + kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml fi -loudecho "OVERALL_TEST_PASSED: ${OVERALL_TEST_PASSED}" -if [[ $OVERALL_TEST_PASSED -ne 0 ]]; then +## Output result + +loudecho "TEST_PASSED: ${TEST_PASSED}" +if [[ $TEST_PASSED -ne 0 ]]; then loudecho "FAIL!" exit 1 else diff --git a/hack/e2e/run2.sh b/hack/e2e/run2.sh new file mode 100755 index 0000000000..17765c5520 --- /dev/null +++ b/hack/e2e/run2.sh @@ -0,0 +1,327 @@ +#!/bin/bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") +source "${BASE_DIR}"/ecr.sh +source "${BASE_DIR}"/eksctl.sh +source "${BASE_DIR}"/kops.sh +source "${BASE_DIR}"/util.sh +source "${BASE_DIR}"/metrics/metrics.sh + +DRIVER_NAME=${DRIVER_NAME:-aws-ebs-csi-driver} +CONTAINER_NAME=${CONTAINER_NAME:-ebs-plugin} + +TEST_ID=${TEST_ID:-$RANDOM} +CLUSTER_NAME=test-cluster-${TEST_ID}.k8s.local +CLUSTER_TYPE=${CLUSTER_TYPE:-kops} + +TEST_DIR=${BASE_DIR}/csi-test-artifacts +BIN_DIR=${TEST_DIR}/bin +CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml +KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.kubeconfig"} + +REGION=${AWS_REGION:-us-west-2} +ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c} +FIRST_ZONE=$(echo "${ZONES}" | cut -d, -f1) +NODE_COUNT=${NODE_COUNT:-3} +INSTANCE_TYPE=${INSTANCE_TYPE:-c5.large} + +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +IMAGE_NAME=${IMAGE_NAME:-${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${DRIVER_NAME}} +IMAGE_TAG=${IMAGE_TAG:-${TEST_ID}} +IMAGE_ARCH=${IMAGE_ARCH:-amd64} + +# kops: must include patch version (e.g. 1.19.1) +# eksctl: mustn't include patch version (e.g. 1.19) +K8S_VERSION_KOPS=${K8S_VERSION_KOPS:-${K8S_VERSION:-1.28.3}} +K8S_VERSION_EKSCTL=${K8S_VERSION_EKSCTL:-${K8S_VERSION:-1.28}} + +KOPS_VERSION=${KOPS_VERSION:-1.28.0} +KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-shared-e2e} +KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-./hack/kops-patch.yaml} +KOPS_PATCH_NODE_FILE=${KOPS_PATCH_NODE_FILE:-./hack/kops-patch-node.yaml} +AMI_PARAMETER=${AMI_PARAMETER:-/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64} +AMI_ID=$(aws ssm get-parameters --names ${AMI_PARAMETER} --region ${REGION} --query 'Parameters[0].Value' --output text) + +EKSCTL_VERSION=${EKSCTL_VERSION:-0.164.0} +EKSCTL_PATCH_FILE=${EKSCTL_PATCH_FILE:-./hack/eksctl-patch.yaml} +VPC_CONFIGMAP_FILE=${VPC_CONFIGMAP_FILE:-./hack/vpc-resource-controller-configmap.yaml} +EKSCTL_ADMIN_ROLE=${EKSCTL_ADMIN_ROLE:-} +# Creates a windows node group. +WINDOWS=${WINDOWS:-"false"} + +# Valid deploy methods: "helm" (default), "kustomize" +DEPLOY_METHOD=${DEPLOY_METHOD:-"helm"} + +HELM_VALUES_FILE=${HELM_VALUES_FILE:-./hack/values.yaml} +HELM_EXTRA_FLAGS=${HELM_EXTRA_FLAGS:-} + +TEST_PATH=${TEST_PATH:-"./tests/e2e/..."} +ARTIFACTS=${ARTIFACTS:-"${TEST_DIR}/artifacts"} +GINKGO_FOCUS=${GINKGO_FOCUS:-"\[ebs-csi-e2e\]"} +GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]"} +GINKGO_NODES=${GINKGO_NODES:-4} +GINKGO_PARALLEL=${GINKGO_PARALLEL:-25} +NODE_OS_DISTRO=${NODE_OS_DISTRO:-"linux"} +TEST_EXTRA_FLAGS=${TEST_EXTRA_FLAGS:-} + +EBS_INSTALL_SNAPSHOT=${EBS_INSTALL_SNAPSHOT:-"false"} +# https://github.com/kubernetes-csi/external-snapshotter +EBS_INSTALL_SNAPSHOT_VERSION=${EBS_INSTALL_SNAPSHOT_VERSION:-"v6.3.2"} + +HELM_CT_TEST=${HELM_CT_TEST:-"false"} +# https://github.com/helm/chart-testing +CHART_TESTING_VERSION=${CHART_TESTING_VERSION:-3.8.0} +CLEAN=${CLEAN:-"true"} + +COLLECT_METRICS=${COLLECT_METRICS:-"false"} + +loudecho "Testing in region ${REGION} and zones ${ZONES}" +mkdir -p "${BIN_DIR}" +export PATH=${PATH}:${BIN_DIR} + +if [[ "${CLUSTER_TYPE}" == "kops" ]]; then + loudecho "Installing kops ${KOPS_VERSION} to ${BIN_DIR}" + kops_install "${BIN_DIR}" "${KOPS_VERSION}" + KOPS_BIN=${BIN_DIR}/kops +elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then + loudecho "Installing eksctl ${EKSCTL_VERSION} to ${BIN_DIR}" + eksctl_install "${BIN_DIR}" "${EKSCTL_VERSION}" + EKSCTL_BIN=${BIN_DIR}/eksctl +else + loudecho "${CLUSTER_TYPE} must be kops or eksctl!" + exit 1 +fi + +loudecho "Installing helm to ${BIN_DIR}" +helm_install "${BIN_DIR}" +HELM_BIN=${BIN_DIR}/helm + +if [[ "${HELM_CT_TEST}" == true ]]; then + loudecho "Installing chart-testing ${CHART_TESTING_VERSION} to ${BIN_DIR}" + ct_install "${BIN_DIR}" "${CHART_TESTING_VERSION}" + CHART_TESTING_BIN=${BIN_DIR}/ct +else + loudecho "Installing ginkgo to ${BIN_DIR}" + GINKGO_BIN=${BIN_DIR}/ginkgo + if [[ ! -e ${GINKGO_BIN} ]]; then + pushd /tmp + GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} go install github.com/onsi/ginkgo/v2/ginkgo@v2.11.0 + popd + ginkgo version + fi + loudecho "Installing kubetest2 to ${BIN_DIR}" + KUBETEST2_BIN=${BIN_DIR}/kubetest2 + if [[ ! -e ${KUBETEST2_BIN} ]]; then + pushd /tmp + GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} go install sigs.k8s.io/kubetest2/...@latest + popd + fi +fi + +ecr_build_and_push "${REGION}" \ + "${AWS_ACCOUNT_ID}" \ + "${IMAGE_NAME}" \ + "${IMAGE_TAG}" \ + "${IMAGE_ARCH}" + +if [[ "${CLUSTER_TYPE}" == "kops" ]]; then + kops_create_cluster \ + "$CLUSTER_NAME" \ + "$KOPS_BIN" \ + "$ZONES" \ + "$NODE_COUNT" \ + "$INSTANCE_TYPE" \ + "$AMI_ID" \ + "$K8S_VERSION_KOPS" \ + "$CLUSTER_FILE" \ + "$KUBECONFIG" \ + "$KOPS_PATCH_FILE" \ + "$KOPS_PATCH_NODE_FILE" \ + "$KOPS_STATE_FILE" + if [[ $? -ne 0 ]]; then + exit 1 + fi +elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then + eksctl_create_cluster \ + "$CLUSTER_NAME" \ + "$EKSCTL_BIN" \ + "$ZONES" \ + "$INSTANCE_TYPE" \ + "$K8S_VERSION_EKSCTL" \ + "$CLUSTER_FILE" \ + "$KUBECONFIG" \ + "$EKSCTL_PATCH_FILE" \ + "$EKSCTL_ADMIN_ROLE" \ + "$WINDOWS" \ + "$VPC_CONFIGMAP_FILE" + if [[ $? -ne 0 ]]; then + exit 1 + fi +fi + +if [[ "${EBS_INSTALL_SNAPSHOT}" == true ]]; then + loudecho "Installing snapshot controller and CRDs" + kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml + kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml + kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml + kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml + kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml +fi + +if [[ "${HELM_CT_TEST}" == true ]]; then + loudecho "Test and lint Helm chart with chart-testing" + if [ -n "${PROW_JOB_ID:-}" ]; then + # Prow-specific setup + # Required becuase chart_testing ALWAYS needs a remote + git remote add ct https://github.com/kubernetes-sigs/aws-ebs-csi-driver.git + git fetch ct "${PULL_BASE_REF}" + export CT_REMOTE="ct" + export CT_TARGET_BRANCH="${PULL_BASE_REF}" + fi + set -x + set +e + export KUBECONFIG="${KUBECONFIG}" + ${CHART_TESTING_BIN} lint-and-install --config ${PWD}/tests/ct-config.yaml --helm-extra-set-args="--set=image.repository=${IMAGE_NAME},image.tag=${IMAGE_TAG},node.tolerateAllTaints=false" + TEST_PASSED=$? + set -e + set +x +else + loudecho "Deploying driver via ${DEPLOY_METHOD}" + startSec=$(date +'%s') + + if [[ ${DEPLOY_METHOD} == "helm" ]]; then + HELM_ARGS=(upgrade --install "${DRIVER_NAME}" + --namespace kube-system + --set image.repository="${IMAGE_NAME}" + --set image.tag="${IMAGE_TAG}" + --set node.enableWindows="${WINDOWS}" + --timeout 10m0s + --wait + --kubeconfig "${KUBECONFIG}" + ./charts/"${DRIVER_NAME}") + if [[ -f "$HELM_VALUES_FILE" ]]; then + HELM_ARGS+=(-f "${HELM_VALUES_FILE}") + fi + eval "EXPANDED_HELM_EXTRA_FLAGS=$HELM_EXTRA_FLAGS" + if [[ -n "$EXPANDED_HELM_EXTRA_FLAGS" ]]; then + HELM_ARGS+=("${EXPANDED_HELM_EXTRA_FLAGS}") + fi + set -x + "${HELM_BIN}" "${HELM_ARGS[@]}" + set +x + elif [[ ${DEPLOY_METHOD} == "kustomize" ]]; then + kubectl --kubeconfig "${KUBECONFIG}" apply -k "./deploy/kubernetes/overlays/stable" + kubectl --kubeconfig "${KUBECONFIG}" --namespace kube-system wait --timeout 10m0s --for "condition=ready" pod -l "app.kubernetes.io/name=aws-ebs-csi-driver" + fi + + endSec=$(date +'%s') + deployTimeSeconds=$(((endSec - startSec) / 1)) + loudecho "Driver deployment complete, time used: $deployTimeSeconds seconds" + loudecho "Testing focus ${GINKGO_FOCUS}" + + if [[ $TEST_PATH == "./tests/e2e-kubernetes/..." ]]; then + pushd ${PWD}/tests/e2e-kubernetes + packageVersion=$(echo $(cut -d '.' -f 1,2 <<< $K8S_VERSION)) + + set -x + set +e + kubetest2 noop \ + --run-id="e2e-kubernetes" \ + --test=ginkgo \ + -- \ + --skip-regex="${GINKGO_SKIP}" \ + --focus-regex="${GINKGO_FOCUS}" \ + --test-package-version=$(curl -L https://dl.k8s.io/release/stable-$packageVersion.txt) \ + --parallel=${GINKGO_PARALLEL} \ + --test-args="-storage.testdriver=${PWD}/manifests.yaml -kubeconfig=$KUBECONFIG -node-os-distro=${NODE_OS_DISTRO}" + + TEST_PASSED=$? + set -e + set +x + popd + fi + + if [[ $TEST_PATH == "./tests/e2e/..." ]]; then + eval "EXPANDED_TEST_EXTRA_FLAGS=$TEST_EXTRA_FLAGS" + set -x + set +e + ${GINKGO_BIN} -p -nodes="${GINKGO_NODES}" -v --focus="${GINKGO_FOCUS}" --skip="${GINKGO_SKIP}" "${TEST_PATH}" -- -kubeconfig="${KUBECONFIG}" -report-dir="${ARTIFACTS}" -gce-zone="${FIRST_ZONE}" "${EXPANDED_TEST_EXTRA_FLAGS}" + TEST_PASSED=$? + set -e + set +x + fi + + PODS=$(kubectl get pod -n kube-system -l "app.kubernetes.io/name=${DRIVER_NAME},app.kubernetes.io/instance=${DRIVER_NAME}" -o json --kubeconfig "${KUBECONFIG}" | jq -r .items[].metadata.name) + + while IFS= read -r POD; do + loudecho "Printing pod ${POD} ${CONTAINER_NAME} container logs" + set +e + kubectl logs "${POD}" -n kube-system "${CONTAINER_NAME}" \ + --kubeconfig "${KUBECONFIG}" + set -e + done <<< "${PODS}" +fi + +OVERALL_TEST_PASSED="${TEST_PASSED}" + +if [[ "${COLLECT_METRICS}" == true ]]; then + metrics_collector "$KUBECONFIG" \ + "$AWS_ACCOUNT_ID" \ + "$AWS_REGION" \ + "$NODE_OS_DISTRO" \ + "$deployTimeSeconds" \ + "$DRIVER_NAME" \ + "$VERSION" +fi + +if [[ "${CLEAN}" == true ]]; then + loudecho "Cleaning" + + if [[ "${HELM_CT_TEST}" != true ]]; then + loudecho "Removing driver via ${DEPLOY_METHOD}" + if [[ ${DEPLOY_METHOD} == "helm" ]]; then + ${HELM_BIN} del "${DRIVER_NAME}" \ + --namespace kube-system \ + --kubeconfig "${KUBECONFIG}" + elif [[ ${DEPLOY_METHOD} == "kustomize" ]]; then + kubectl --kubeconfig "${KUBECONFIG}" delete -k "./deploy/kubernetes/overlays/stable" + fi + fi + + if [[ "${CLUSTER_TYPE}" == "kops" ]]; then + kops_delete_cluster \ + "${KOPS_BIN}" \ + "${CLUSTER_NAME}" \ + "${KOPS_STATE_FILE}" + elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then + eksctl_delete_cluster \ + "${EKSCTL_BIN}" \ + "${CLUSTER_NAME}" + fi +else + loudecho "Not cleaning" +fi + +loudecho "OVERALL_TEST_PASSED: ${OVERALL_TEST_PASSED}" +if [[ $OVERALL_TEST_PASSED -ne 0 ]]; then + loudecho "FAIL!" + exit 1 +else + loudecho "SUCCESS!" +fi diff --git a/hack/provenance b/hack/provenance.sh similarity index 100% rename from hack/provenance rename to hack/provenance.sh diff --git a/hack/prow-e2e.sh b/hack/prow-e2e.sh new file mode 100644 index 0000000000..f382a5d4e1 --- /dev/null +++ b/hack/prow-e2e.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +case ${1} in + test-e2e-single-az) + TEST="single-az" + ;; + test-e2e-multi-az) + TEST="multi-az" + ;; + test-e2e-external) + TEST="external" + ;; + test-e2e-external-arm64) + TEST="external" + export INSTANCE_TYPE="m7g.medium" + ;; + test-e2e-external-eks) + TEST="external" + export CLUSTER_TYPE="eksctl" + ;; + test-e2e-external-eks-windows) + TEST="external" + export CLUSTER_TYPE="eksctl" + export WINDOWS="true" + ;; + test-e2e-external-kustomize) + TEST="external-kustomize" + ;; + test-helm-chart) + TEST="helm-ct" + ;; + *) + echo "Unknown e2e test ${1}" >&2 + exit 1 + ;; +esac + +export CLUSTER_NAME="ebs-csi-e2e-${RANDOM}.k8s.local" +export KOPS_BUCKET="k8s-kops-csi-shared-e2e" + +make cluster/create || exit 1 +make e2e/${TEST} +E2E_PASSED=$? +make cluster/delete + +loudecho "E2E_PASSED: ${E2E_PASSED}" +if [[ $E2E_PASSED -ne 0 ]]; then + echo "FAIL!" + exit 1 +else + echo "SUCCESS!" +fi diff --git a/hack/prow.sh b/hack/prow.sh index a5fd118efc..afcfc856ec 100755 --- a/hack/prow.sh +++ b/hack/prow.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Kubernetes Authors. +# Copyright 2023 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -42,4 +42,4 @@ loudecho "Push manifest list containing amazon linux and windows based images to export REGISTRY=$REGISTRY_NAME export TAG=$GIT_TAG export VERSION=$PULL_BASE_REF -IMAGE=gcr.io/k8s-staging-provider-aws/aws-ebs-csi-driver make all-push +IMAGE=gcr.io/k8s-staging-provider-aws/aws-ebs-csi-driver make -j `nproc` all-push-with-a1compat diff --git a/hack/release b/hack/release deleted file mode 100755 index 26ffa1d0f6..0000000000 --- a/hack/release +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/local/bin/python3 - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import hashlib -import json -import os -import requests - -def file_sha512(fileName, repoName): - download(fileName, repoName) - with open(fileName, 'rb') as file: - m = hashlib.sha512() - blob = file.read() - m.update(blob) - print("[{}](https://github.com/{}/archive/{}) | `{}`" - .format(fileName,repoName, fileName,m.hexdigest())) - os.remove(fileName) - -def download(fileName, repoName): - url = 'https://github.com/{}/archive/{}'.format(repoName, fileName) - r = requests.get(url, allow_redirects=True) - open(fileName, 'wb').write(r.content) - -def print_header(repo, version): - # Title - print('# {}'.format(version)) - - # documentation section - print('[Documentation](https://github.com/{}/blob/{}/docs/README.md)\n' - .format(repo,version)) - - # sha512 - print('filename | sha512 hash') - print('--------- | ------------') - file_sha512(version+".zip", repo) - file_sha512(version+".tar.gz", repo) - -class Github: - def __init__(self, user, token): - self._url = 'https://api.github.com' - self._user = user - self._token = token - - def get_commits(self, repo, since, branch): - resp = requests.get('{}/repos/{}/compare/{}...{}'.format(self._url, repo, since, branch), - auth=(self._user, self._token)) - jsonResp = json.loads(resp.content) - return jsonResp['commits'] - - def to_pr_numbers(self, repo, commit): - sha = commit['sha'] - resp = requests.get('{}/repos/{}/commits/{}/pulls'.format(self._url, repo, sha), - headers={'Accept': 'application/vnd.github.groot-preview+json'}, - auth=(self._user, self._token)) - jsonResp = json.loads(resp.content) - ret = [] - for pr in jsonResp: - ret.append(pr['number']) - - return ret - - def get_pr(self, repo, pr_number): - resp = requests.get('{}/repos/{}/pulls/{}'.format(self._url, repo, pr_number), - auth=(self._user, self._token)) - jsonResp = json.loads(resp.content) - return jsonResp - - def print_release_note(self, repo, since, branch): - # remove merge commits - commits = self.get_commits(repo, since, branch) - commits = filter(lambda c: not c['commit']['message'].startswith('Merge pull request'), commits) - pr_numbers = set() - for commit in commits: - numbers = self.to_pr_numbers(repo, commit) - for pr in numbers: - pr_numbers.add(pr) - - # dedupe pr numbers - pr_numbers = sorted(list(pr_numbers)) - - for number in pr_numbers: - pr = self.get_pr(repo, number) - if 'user' in pr: - user = pr['user']['login'] - print('* {} ([#{}]({}), [@{}](https://github.com/{}))'.format(pr['title'], pr['number'], pr['html_url'], user, user)) - -def print_sha(args): - version = args.version - repo = args.repo - print_header(repo, version) - -def print_notes(args): - repo = args.repo - since = args.since - branch = args.branch - user = args.github_user - token = args.github_token - - g = Github(user, token) - g.print_release_note(repo, since, branch) - -if __name__=="__main__": - parser = argparse.ArgumentParser(description='Generate release CHANGELOG') - parser.add_argument('--repo', metavar='repo', type=str, default='kubernetes-sigs/aws-ebs-csi-driver', help='the full github repository name') - parser.add_argument('--github-user', metavar='user', type=str, help='the github user for github api') - parser.add_argument('--github-token', metavar='token', type=str, help='the github token for github api') - - subParsers = parser.add_subparsers(title='subcommands', description='[note|sha]') - - noteParser = subParsers.add_parser('note', help='generate release notes') - noteParser.add_argument('--since', metavar='since', type=str, required=True, help='since version tag, e.g. if releasing v1.3.5 then set this to v1.3.4') - noteParser.add_argument('--branch', metavar='branch', type=str, required=True, help='release branch, e.g. if releasing v1.3.5 then set this to release-1.3') - noteParser.set_defaults(func=print_notes) - - shaParser = subParsers.add_parser('sha', help='generate SHA for released version tag') - shaParser.add_argument('--version', metavar='version', type=str, required=True, help='the version to release') - shaParser.set_defaults(func=print_sha) - - args = parser.parse_args() - args.func(args) diff --git a/hack/test-integration.sh b/hack/test-integration.sh deleted file mode 100755 index 906e551dff..0000000000 --- a/hack/test-integration.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -if ! [[ "$0" =~ hack/test-integration.sh ]]; then - echo "must be run from repository root" - exit 127 -fi - -export GO111MODULE=on -go test -c ./tests/integration/... -o bin/integration.test && \ - sudo -E bin/integration.test -test.v -ginkgo.v diff --git a/hack/tools/install.sh b/hack/tools/install.sh new file mode 100755 index 0000000000..fdf2031f37 --- /dev/null +++ b/hack/tools/install.sh @@ -0,0 +1,175 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +# https://pypi.org/project/awscli/ +AWSCLI_VERSION="1.31.7" +# https://github.com/helm/chart-testing +CT_VERSION="v3.10.1" +# https://github.com/eksctl-io/eksctl +EKSCTL_VERSION="v0.165.0" +# https://github.com/onsi/ginkgo +GINKGO_VERSION="v2.13.2" +# https://github.com/golangci/golangci-lint +GOLANGCI_LINT_VERSION="v1.55.2" +# https://github.com/helm/helm +HELM_VERSION="v3.13.2" +# https://github.com/kubernetes/kops +KOPS_VERSION="v1.28.1" +# https://github.com/golang/mock +MOCKGEN_VERSION="v1.6.0" +# https://pypi.org/project/yamale/ +YAMALE_VERSION="4.0.4" +# https://pypi.org/project/yamllint/ +YAMLLINT_VERSION="1.32.0" + +OS="$(go env GOHOSTOS)" +ARCH="$(go env GOHOSTARCH)" + +# Installation helpers + +function install_binary() { + INSTALL_PATH="${1}" + DOWNLOAD_URL="${2}" + BINARY_NAME="${3}" + + curl --location "${DOWNLOAD_URL}" --output "${INSTALL_PATH}/${BINARY_NAME}" + chmod +x "${INSTALL_PATH}/${BINARY_NAME}" +} + +function install_go() { + INSTALL_PATH="${1}" + PACKAGE="${2}" + + export GOBIN="${INSTALL_PATH}" + go install "${PACKAGE}" +} + +function install_pip() { + INSTALL_PATH="${1}" + PACKAGE="${2}" + COMMAND="${3}" + + source "${INSTALL_PATH}/venv/bin/activate" + python3 -m pip install "${PACKAGE}" + cp "$(dirname "${0}")/python-runner.sh" "${INSTALL_PATH}/${COMMAND}" +} + +function install_tar_binary() { + INSTALL_PATH="${1}" + DOWNLOAD_URL="${2}" + BINARY_PATH="${3}" + + BINARY_NAME="$(basename "${BINARY_PATH}")" + + if [ "${DOWNLOAD_URL##*.}" = "gz" ]; then + TAR_EXTRA_FLAGS="-z" + elif [ "${DOWNLOAD_URL##*.}" = "xz" ]; then + TAR_EXTRA_FLAGS="-J" + else + TAR_EXTRA_FLAGS="" + fi + + curl --location "${DOWNLOAD_URL}" | tar "$TAR_EXTRA_FLAGS" --extract --touch --transform "s/.*/${BINARY_NAME}/" -C "${INSTALL_PATH}" "${BINARY_PATH}" + chmod +x "${INSTALL_PATH}/${BINARY_NAME}" +} + +# Tool-specific installers + +function install_aws() { + INSTALL_PATH="${1}" + + install_pip "${INSTALL_PATH}" "awscli==${AWSCLI_VERSION}" "aws" +} + +function install_ct() { + INSTALL_PATH="${1}" + + install_tar_binary "${INSTALL_PATH}" "https://github.com/helm/chart-testing/releases/download/${CT_VERSION}/chart-testing_${CT_VERSION:1}_${OS}_${ARCH}.tar.gz" "ct" + install_pip "${INSTALL_PATH}" "yamale==${YAMALE_VERSION}" "yamale" + install_pip "${INSTALL_PATH}" "yamllint==${YAMLLINT_VERSION}" "yamllint" +} + +function install_eksctl() { + INSTALL_PATH="${1}" + + install_tar_binary "${INSTALL_PATH}" "https://github.com/weaveworks/eksctl/releases/download/${EKSCTL_VERSION}/eksctl_${OS^}_${ARCH}.tar.gz" "eksctl" +} + +function install_ginkgo() { + INSTALL_PATH="${1}" + + install_go "${INSTALL_PATH}" "github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}" +} + +function install_golangci-lint() { + INSTALL_PATH="${1}" + + # golangci-lint recommends against installing with `go install`: https://golangci-lint.run/usage/install/#install-from-source + install_tar_binary "${INSTALL_PATH}" "https://github.com/golangci/golangci-lint/releases/download/${GOLANGCI_LINT_VERSION}/golangci-lint-${GOLANGCI_LINT_VERSION:1}-${OS}-${ARCH}.tar.gz" "golangci-lint-${GOLANGCI_LINT_VERSION:1}-${OS}-${ARCH}/golangci-lint" +} + +function install_helm() { + INSTALL_PATH="${1}" + + install_tar_binary "${INSTALL_PATH}" "https://get.helm.sh/helm-${HELM_VERSION}-${OS}-${ARCH}.tar.gz" "${OS}-${ARCH}/helm" +} + +function install_kops() { + INSTALL_PATH="${1}" + + install_binary "${INSTALL_PATH}" "https://github.com/kubernetes/kops/releases/download/${KOPS_VERSION}/kops-${OS}-${ARCH}" "kops" +} + +function install_kubetest2 { + INSTALL_PATH="${1}" + + install_go "${INSTALL_PATH}" "sigs.k8s.io/kubetest2/...@latest" +} + +function install_mockgen() { + INSTALL_PATH="${1}" + + install_go "${INSTALL_PATH}" "github.com/golang/mock/mockgen@${MOCKGEN_VERSION}" +} + +# Utility functions + +function create_environment() { + INSTALL_PATH="${1}" + + if command -v "python3"; then + PYTHON_CMD="python3" + else + PYTHON_CMD="python" + fi + VIRTUAL_ENV_DISABLE_PROMPT=1 "${PYTHON_CMD}" -m venv "${INSTALL_PATH}/venv" +} + +function install_tool() { + INSTALL_PATH="${1}" + TOOL="${2}" + + "install_${TOOL}" "${INSTALL_PATH}" +} + +# Script dispatcher + +if [ ! -d "${TOOLS_PATH}/venv" ]; then + create_environment "${TOOLS_PATH}" +fi +install_tool "${TOOLS_PATH}" "${1}" diff --git a/hack/verify-govet b/hack/tools/python-runner.sh similarity index 69% rename from hack/verify-govet rename to hack/tools/python-runner.sh index f33dade338..67c8434eab 100755 --- a/hack/verify-govet +++ b/hack/tools/python-runner.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Kubernetes Authors. +# Copyright 2023 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,10 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -euo pipefail +# This script is used as a stub for python commands installed to bin/ +# It activates the venv inside bin/venv/ and then passes through -echo "Verifying govet" - -go vet $(go list ./... | grep -v vendor) - -echo "Done" +source "$(dirname "${0}")/venv/bin/activate" +exec "$(basename "${0}")" "$@" diff --git a/hack/update-gofmt b/hack/update-gofmt deleted file mode 100755 index dd016e1762..0000000000 --- a/hack/update-gofmt +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -find . -name "*.go" | grep -v "\/vendor\/" | xargs gofmt -s -w diff --git a/hack/update-gomod b/hack/update-gomod deleted file mode 100755 index d89c2a7edd..0000000000 --- a/hack/update-gomod +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -euo pipefail -set -x - -VERSION=${1#"v"} -if [ -z "$VERSION" ]; then - echo "Must specify version!" - exit 1 -fi -MODS=($( - curl -sS https://raw.githubusercontent.com/kubernetes/kubernetes/v${VERSION}/go.mod | - sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p' -)) -echo $MODS -for MOD in "${MODS[@]}"; do - echo $MOD - V=$( - go mod download -json "${MOD}@kubernetes-${VERSION}" | - sed -n 's|.*"Version": "\(.*\)".*|\1|p' - ) - go mod edit "-replace=${MOD}=${MOD}@${V}" -done diff --git a/hack/update-kustomize.sh b/hack/update-kustomize.sh new file mode 100755 index 0000000000..480ea3e0a4 --- /dev/null +++ b/hack/update-kustomize.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script updates the kustomize templates in deploy/kubernetes/base/ by +# running `helm template` and stripping the namespace from the output + +set -euo pipefail + +BIN="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/../bin" +TEMP_DIR=$(mktemp -d) +trap "rm -rf \"${TEMP_DIR}\"" EXIT +cp "deploy/kubernetes/base/kustomization.yaml" "${TEMP_DIR}/kustomization.yaml" + +"${BIN}/helm" template --output-dir "${TEMP_DIR}" --skip-tests --api-versions 'snapshot.storage.k8s.io/v1' --api-versions 'policy/v1/PodDisruptionBudget' --set 'controller.userAgentExtra=kustomize' kustomize charts/aws-ebs-csi-driver > /dev/null +rm -rf "deploy/kubernetes/base" +mv "${TEMP_DIR}/aws-ebs-csi-driver/templates" "deploy/kubernetes/base" + +sed -i '/namespace:/d' deploy/kubernetes/base/* +cp "${TEMP_DIR}/kustomization.yaml" "deploy/kubernetes/base/kustomization.yaml" diff --git a/hack/update-gomock b/hack/update-mockgen.sh similarity index 50% rename from hack/update-gomock rename to hack/update-mockgen.sh index ee6600dda4..a6a2a475a7 100755 --- a/hack/update-gomock +++ b/hack/update-mockgen.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Kubernetes Authors. +# Copyright 2023 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,16 +16,18 @@ set -euo pipefail -mockgen -package cloud -destination=./pkg/cloud/mock_cloud.go -source pkg/cloud/cloud_interface.go -mockgen -package cloud -destination=./pkg/cloud/mock_metadata.go -source pkg/cloud/metadata_interface.go -mockgen -package driver -destination=./pkg/driver/mock_mount.go -source pkg/driver/mount.go -mockgen -package mounter -destination=./pkg/mounter/mock_mount_windows.go -source pkg/mounter/safe_mounter_windows.go +BIN="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/../bin" + +# Source-based mocking for internal interfaces +"${BIN}/mockgen" -package cloud -destination=./pkg/cloud/mock_cloud.go -source pkg/cloud/cloud_interface.go +"${BIN}/mockgen" -package cloud -destination=./pkg/cloud/mock_metadata.go -source pkg/cloud/metadata_interface.go +"${BIN}/mockgen" -package driver -destination=./pkg/driver/mock_mount.go -source pkg/driver/mount.go +"${BIN}/mockgen" -package mounter -destination=./pkg/mounter/mock_mount_windows.go -source pkg/mounter/safe_mounter_windows.go # Reflection-based mocking for external dependencies -mockgen -package cloud -destination=./pkg/cloud/mock_ec2.go github.com/aws/aws-sdk-go/service/ec2/ec2iface EC2API -mockgen -package driver -destination=./pkg/driver/mock_k8s_client.go -mock_names='Interface=MockKubernetesClient' k8s.io/client-go/kubernetes Interface -mockgen -package driver -destination=./pkg/driver/mock_k8s_corev1.go k8s.io/client-go/kubernetes/typed/core/v1 CoreV1Interface,NodeInterface -mockgen -package driver -destination=./pkg/driver/mock_k8s_storagev1.go k8s.io/client-go/kubernetes/typed/storage/v1 VolumeAttachmentInterface,StorageV1Interface +"${BIN}/mockgen" -package cloud -destination=./pkg/cloud/mock_ec2.go github.com/aws/aws-sdk-go/service/ec2/ec2iface EC2API +"${BIN}/mockgen" -package driver -destination=./pkg/driver/mock_k8s_client.go -mock_names='Interface=MockKubernetesClient' k8s.io/client-go/kubernetes Interface +"${BIN}/mockgen" -package driver -destination=./pkg/driver/mock_k8s_corev1.go k8s.io/client-go/kubernetes/typed/core/v1 CoreV1Interface,NodeInterface # Fixes "Mounter Type cannot implement 'Mounter' as it has a non-exported method and is defined in a different package" # See https://github.com/kubernetes/mount-utils/commit/a20fcfb15a701977d086330b47b7efad51eb608e for context. diff --git a/hack/verify-all b/hack/verify-all deleted file mode 100755 index e6cabf8811..0000000000 --- a/hack/verify-all +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -PKG_ROOT=$(git rev-parse --show-toplevel) - -${PKG_ROOT}/hack/verify-gofmt -${PKG_ROOT}/hack/verify-govet -${PKG_ROOT}/bin/golangci-lint run --deadline=10m -${PKG_ROOT}/hack/verify-vendor.sh diff --git a/hack/verify-gofmt b/hack/verify-gofmt deleted file mode 100755 index ae0106d65f..0000000000 --- a/hack/verify-gofmt +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -echo "Verifying gofmt" - -diff=$(find . -name "*.go" | grep -v "\/vendor\/" | xargs gofmt -s -d 2>&1) -if [[ -n "${diff}" ]]; then - echo "${diff}" - echo - echo "Please run hack/update-gofmt to fix the issue(s)" - exit 1 -fi -echo "No issue found" diff --git a/hack/verify-kustomize b/hack/verify-kustomize deleted file mode 100755 index ad4fb3cda9..0000000000 --- a/hack/verify-kustomize +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Copyright 2022 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -echo "Verifying kustomize" -diff=$(git status --porcelain=v1) -if [[ -n "${diff}" ]]; then - echo "${diff}" - echo - echo "Please commit all changes before verifying" - exit 1 -fi -make generate-kustomize -echo -diff=$(git status --porcelain=v1) -if [[ -n "${diff}" ]]; then - echo "${diff}" - echo - echo "Please run make generate-kustomize to fix the issue(s)" - exit 1 -fi -echo "No issue found" diff --git a/hack/verify-update.sh b/hack/verify-update.sh new file mode 100755 index 0000000000..c2c818c926 --- /dev/null +++ b/hack/verify-update.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script verifies that `make update` does not need to run +# It does so by creating a temporary copy of the repo and running `make update` +# in the copy, and then checking if it produces a diff to the local copy + +set -euo pipefail + +ROOT="$(dirname "${0}")/../" +TEST_DIR=$(mktemp -d) +trap "rm -rf \"${TEST_DIR}\"" EXIT +cp -rf "${ROOT}/." "${TEST_DIR}" + +if ! make -C "${TEST_DIR}" update > /dev/null; then + echo "\`make update\` failed!" + exit 1 +fi + +if ! diff -r "${TEST_DIR}" "${ROOT}"; then + echo "Auto-generation/formatting needs to run!" + echo "Run \`make update\` to fix!" + exit 1 +fi diff --git a/hack/verify-vendor.sh b/hack/verify-vendor.sh deleted file mode 100755 index e849dd46d0..0000000000 --- a/hack/verify-vendor.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -echo "Repo uses 'go mod'." -if ! (set -x; env GO111MODULE=on go mod tidy); then - echo "ERROR: vendor check failed." - exit 1 -elif [ "$(git status --porcelain -- go.mod go.sum | wc -l)" -gt 0 ]; then - echo "ERROR: go module files *not* up-to-date, they did get modified by 'GO111MODULE=on go mod tidy':"; - git diff -- go.mod go.sum - exit 1 -elif [ -d vendor ]; then - if ! (set -x; env GO111MODULE=on go mod vendor); then - echo "ERROR: vendor check failed." - exit 1 - elif [ "$(git status --porcelain -- vendor | wc -l)" -gt 0 ]; then - echo "ERROR: vendor directory *not* up-to-date, it did get modified by 'GO111MODULE=on go mod vendor':" - git status -- vendor - git diff -- vendor - exit 1 - else - echo "Go dependencies and vendor directory up-to-date." - fi -else - echo "Go dependencies up-to-date." -fi