From d8de59998b9e5481fc5d46ec0cfe11964f1f9b93 Mon Sep 17 00:00:00 2001 From: Naadir Jeewa Date: Wed, 16 Sep 2020 13:19:32 +0100 Subject: [PATCH] WIP: kubetest Signed-off-by: Naadir Jeewa --- .gitignore | 2 + Makefile | 236 ++++++++------- common.mk | 65 ++++ hack/tools/Makefile | 115 +++++++ hack/tools/tools.mk | 12 + scripts/ci-conformance.sh | 64 ++++ scripts/ci-e2e.sh | 2 +- test/e2e/Makefile | 65 ---- test/e2e/common.go | 58 +--- test/e2e/config/docker-ci.yaml | 8 + test/e2e/config/docker-dev.yaml | 8 + test/e2e/conformance/ci_artifacts.go | 137 +++++++++ test/e2e/conformance/ci_artifacts_test.go | 39 +++ test/e2e/conformance/conformance.go | 139 +++++++++ .../e2e/conformance/conformance_suite_test.go | 179 +++++++++++ test/e2e/conformance/conformance_test.go | 39 +++ .../cluster-template-ci.yaml | 2 +- .../cluster-template-kcp-adoption.yaml | 2 +- .../cluster-template.yaml | 2 +- .../platform-kustomization.yaml | 19 ++ test/e2e/data/kubetest/conformance-fast.yaml | 8 + test/e2e/data/kubetest/conformance.yaml | 7 + test/e2e/e2e_suite_test.go | 162 ++++------ test/e2e/internal/setup/setup.go | 230 ++++++++++++++ test/e2e/kcp_adoption.go | 24 +- test/e2e/kcp_upgrade.go | 24 +- test/e2e/kcp_upgrade_ci_artifacts.go | 216 +++++++++++++ test/e2e/kcp_upgrade_ci_artifacts_test.go | 39 +++ test/e2e/md_upgrades.go | 24 +- test/e2e/mhc_remediations.go | 24 +- test/e2e/quick_start.go | 24 +- test/e2e/self_hosted.go | 29 +- test/framework/alltypes_helpers.go | 16 + test/framework/bootstrap/kind_provider.go | 2 +- test/framework/bootstrap/kind_util.go | 2 +- test/framework/cluster_helpers.go | 10 +- test/framework/cluster_proxy.go | 2 +- test/framework/clusterctl/client.go | 2 +- .../clusterctl/clusterctl_helpers.go | 2 +- test/framework/clusterctl/e2e_config.go | 13 +- test/framework/config.go | 14 + test/framework/control_plane.go | 2 +- test/framework/controlplane_helpers.go | 10 +- test/framework/convenience.go | 30 ++ test/framework/daemonset_helpers.go | 2 +- test/framework/deployment_helpers.go | 6 +- test/framework/deprecated.go | 2 +- .../log/log.go => ginkgoextensions/output.go} | 18 +- test/framework/kubernetesversions/versions.go | 78 +++++ .../log.go => framework/kubetest/bindata.go} | 12 +- .../data/debian_injection_script.envsubst.sh | 106 +++++++ .../kubetest/data/kustomization.yaml | 8 + test/framework/kubetest/run.go | 211 +++++++++++++ test/framework/kubetest/setup.go | 63 ++++ test/framework/kubetest/template.go | 177 +++++++++++ .../kubetest/zz_generated.bindata.go | 285 ++++++++++++++++++ test/framework/log/log.go | 167 ++++++++++ test/framework/machine_helpers.go | 4 +- test/framework/machinedeployment_helpers.go | 2 +- test/framework/machines.go | 4 +- test/framework/namespace_helpers.go | 2 +- test/framework/pod_helpers.go | 2 +- test/framework/suite_helpers.go | 46 +++ test/infrastructure/docker/.gitignore | 2 + test/infrastructure/docker/Makefile | 64 ++-- .../api/v1alpha3/dockermachine_types.go | 5 + ...cture.cluster.x-k8s.io_dockermachines.yaml | 5 + ...uster.x-k8s.io_dockermachinetemplates.yaml | 5 + .../controllers/dockermachine_controller.go | 25 +- versions.mk | 16 + 70 files changed, 2984 insertions(+), 442 deletions(-) create mode 100644 common.mk create mode 100644 hack/tools/Makefile create mode 100644 hack/tools/tools.mk create mode 100644 scripts/ci-conformance.sh delete mode 100644 test/e2e/Makefile create mode 100644 test/e2e/conformance/ci_artifacts.go create mode 100644 test/e2e/conformance/ci_artifacts_test.go create mode 100644 test/e2e/conformance/conformance.go create mode 100644 test/e2e/conformance/conformance_suite_test.go create mode 100644 test/e2e/conformance/conformance_test.go create mode 100644 test/e2e/data/infrastructure-docker/platform-kustomization.yaml create mode 100644 test/e2e/data/kubetest/conformance-fast.yaml create mode 100644 test/e2e/data/kubetest/conformance.yaml create mode 100644 test/e2e/internal/setup/setup.go create mode 100644 test/e2e/kcp_upgrade_ci_artifacts.go create mode 100644 test/e2e/kcp_upgrade_ci_artifacts_test.go rename test/framework/{internal/log/log.go => ginkgoextensions/output.go} (65%) create mode 100644 test/framework/kubernetesversions/versions.go rename test/{e2e/internal/log/log.go => framework/kubetest/bindata.go} (68%) create mode 100644 test/framework/kubetest/data/debian_injection_script.envsubst.sh create mode 100644 test/framework/kubetest/data/kustomization.yaml create mode 100644 test/framework/kubetest/run.go create mode 100644 test/framework/kubetest/setup.go create mode 100644 test/framework/kubetest/template.go create mode 100644 test/framework/kubetest/zz_generated.bindata.go create mode 100644 test/framework/log/log.go create mode 100644 test/framework/suite_helpers.go create mode 100644 versions.mk diff --git a/.gitignore b/.gitignore index f86f8b5418d2..50698281f676 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,5 @@ clusterctl-settings.json # test results _artifacts + +*.sentinel diff --git a/Makefile b/Makefile index d573a9523d97..16a3b21ec256 100644 --- a/Makefile +++ b/Makefile @@ -16,53 +16,22 @@ # https://suva.sh/posts/well-documented-makefiles # Ensure Make is run with bash shell as some syntax below is bash-specific -SHELL:=/usr/bin/env bash -.DEFAULT_GOAL:=help - -# Use GOPROXY environment variable if set -GOPROXY := $(shell go env GOPROXY) -ifeq ($(GOPROXY),) -GOPROXY := https://proxy.golang.org -endif -export GOPROXY - -# Active module mode, as we use go modules to manage dependencies -export GO111MODULE=on +ROOT_DIR_RELATIVE := . +include $(ROOT_DIR_RELATIVE)/common.mk +include $(ROOT_DIR_RELATIVE)/hack/tools/tools.mk # Default timeout for starting/stopping the Kubebuilder test control plane export KUBEBUILDER_CONTROLPLANE_START_TIMEOUT ?=60s export KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT ?=60s -# This option is for running docker manifest command -export DOCKER_CLI_EXPERIMENTAL := enabled - # Directories. EXP_DIR := exp -TOOLS_DIR := hack/tools -TOOLS_BIN_DIR := $(TOOLS_DIR)/bin BIN_DIR := bin E2E_FRAMEWORK_DIR := test/framework CAPD_DIR := test/infrastructure/docker -RELEASE_NOTES_BIN := bin/release-notes -RELEASE_NOTES := $(TOOLS_DIR)/$(RELEASE_NOTES_BIN) -LINK_CHECKER_BIN := bin/liche -LINK_CHECKER := $(TOOLS_DIR)/$(LINK_CHECKER_BIN) -GO_APIDIFF_BIN := bin/go-apidiff -GO_APIDIFF := $(TOOLS_DIR)/$(GO_APIDIFF_BIN) -ENVSUBST_BIN := bin/envsubst -ENVSUBST := $(TOOLS_DIR)/$(ENVSUBST_BIN) - -# Binaries. -# Need to use abspath so we can invoke these from subdirectories -KUSTOMIZE := $(abspath $(TOOLS_BIN_DIR)/kustomize) -CONTROLLER_GEN := $(abspath $(TOOLS_BIN_DIR)/controller-gen) -GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/golangci-lint) -CONVERSION_GEN := $(abspath $(TOOLS_BIN_DIR)/conversion-gen) -ENVSUBST := $(abspath $(TOOLS_BIN_DIR)/envsubst) # Bindata. -GOBINDATA := $(abspath $(TOOLS_BIN_DIR)/go-bindata) GOBINDATA_CLUSTERCTL_DIR := cmd/clusterctl/config CLOUDINIT_PKG_DIR := bootstrap/kubeadm/internal/cloudinit CLOUDINIT_GENERATED := $(CLOUDINIT_PKG_DIR)/zz_generated.bindata.go @@ -92,43 +61,68 @@ ALL_ARCH = amd64 arm arm64 ppc64le s390x # Allow overriding the imagePullPolicy PULL_POLICY ?= Always -# Hosts running SELinux need :z added to volume mounts -SELINUX_ENABLED := $(shell cat /sys/fs/selinux/enforce 2> /dev/null || echo 0) - -ifeq ($(SELINUX_ENABLED),1) - DOCKER_VOL_OPTS?=:z -endif - # Set build time variables including version details LDFLAGS := $(shell hack/version.sh) all: test managers clusterctl -help: ## Display this help - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[0-9A-Za-z_-]+:.*?##/ { printf " \033[36m%-45s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - ## -------------------------------------- ## Testing ## -------------------------------------- +TEST_ARGS ?= +GOTEST_OPTS ?= +GOTEST_CMD ?= go + .PHONY: test -test: ## Run tests. - source ./scripts/fetch_ext_bins.sh; fetch_tools; setup_envs; go test -v ./... $(TEST_ARGS) +test: ## Run tests. Parameters: GOTEST_CMD (command to use for testing. default="go"), TEST_ARGS (args to pass to the test package. default="") + source ./scripts/fetch_ext_bins.sh + fetch_tools + setup_envs + $(GOTEST_CMD) test -v $(GOTEST_OPTS) ./... $(TEST_ARGS) .PHONY: test-cover -test-cover: ## Run tests with code coverage and code generate reports - source ./scripts/fetch_ext_bins.sh; fetch_tools; setup_envs; go test -v -coverprofile=out/coverage.out ./... $(TEST_ARGS) +test-cover: ## Run tests with code coverage and code generate reports. See test for accepted parameters. + $(MAKE) test GOTEST_OPTS="-coverprofile=out/coverage.out" TEST_ARGS=$(TEST_ARGS) GOTEST_CMD=$(GOTEST_CMD) go tool cover -func=out/coverage.out -o out/coverage.txt go tool cover -html=out/coverage.out -o out/coverage.html -.PHONY: docker-build-e2e -docker-build-e2e: ## Rebuild all Cluster API provider images to be used in the e2e tests - make docker-build REGISTRY=gcr.io/k8s-staging-cluster-api PULL_POLICY=IfNotPresent - $(MAKE) -C test/infrastructure/docker docker-build REGISTRY=gcr.io/k8s-staging-cluster-api +GINKGO_NODES ?= 1 +GINKGO_NOCOLOR ?= false +GINKGO_ARGS ?= +ARTIFACTS ?= $(abspath _artifacts) +E2E_CONF_FILE ?= config/docker-dev.yaml +SKIP_RESOURCE_CLEANUP ?= false +USE_EXISTING_CLUSTER ?= false +GINKGO_FOCUS ?= +export GINKGO_NODES GINKGO_NOCOLOR GINKGO_ARGS ARTIFACTS E2E_CONF_FILE SKIP_RESOURCE_CLEANUP USE_EXISTING_CLUSTER GINKGO_FOCUS .PHONY: test-e2e -test-e2e: ## Run the e2e tests - $(MAKE) -C test/e2e run +test-e2e: $(GINKGO) ## Run the e2e tests. Parameters: GINKGO_NODES (number of parallel Ginkgo runners. default=1), GINKGO_NOCOLOR (whether or not to use color output for Ginkgo. default=false), ARTIFACTS (where to save output. default=_artifacts), E2E_CONF_FILE (clusterctl framework e2e conf file location. default=config/docker-dev.yaml), SKIP_RESOURCE_CLEANUP (default=false), USE_EXISTING_CLUSTER (use current kubectl context, default=false) + $(MAKE) test-e2e-run TEST_DIR=test/e2e + +.PHONY: test-conformance +test-conformance: $(GINKGO) $(KUSTOMIZE) ## Run e2e conformance tests. See test-e2e for accepted parameters. + $(MAKE) test-e2e-run TEST_DIR=test/e2e/conformance + +.PHONY: test-e2e-run +test-e2e-run: docker-build-e2e + $(GINKGO) -v -trace \ + -tags=e2e -focus="$(GINKGO_FOCUS)" \ + -nodes=$(GINKGO_NODES) \ + --noColor=$(GINKGO_NOCOLOR) \ + $(GINKGO_ARGS) $(TEST_DIR) \ + -- \ + -e2e.artifacts-folder="$(ARTIFACTS)" \ + -e2e.config="$(E2E_CONF_FILE)" \ + -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \ + -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) + +.PHONY: pull-cert-manager +pull-cert-manager: ## Use to cache cert manager images on the top-level Docker + docker pull quay.io/jetstack/cert-manager-cainjector:$(CERT_MANAGER_VERSION) + docker pull quay.io/jetstack/cert-manager-webhook:$(CERT_MANAGER_VERSION) + docker pull quay.io/jetstack/cert-manager-controller:$(CERT_MANAGER_VERSION) ## -------------------------------------- ## Binaries @@ -156,57 +150,38 @@ managers: ## Build all managers clusterctl: ## Build clusterctl binary go build -ldflags "$(LDFLAGS)" -o bin/clusterctl sigs.k8s.io/cluster-api/cmd/clusterctl -$(KUSTOMIZE): $(TOOLS_DIR)/go.mod # Build kustomize from tools folder. - cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/kustomize sigs.k8s.io/kustomize/kustomize/v3 - -$(CONTROLLER_GEN): $(TOOLS_DIR)/go.mod # Build controller-gen from tools folder. - cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/controller-gen sigs.k8s.io/controller-tools/cmd/controller-gen - -$(GOLANGCI_LINT): $(TOOLS_DIR)/go.mod # Build golangci-lint from tools folder. - cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/golangci-lint github.com/golangci/golangci-lint/cmd/golangci-lint - -$(CONVERSION_GEN): $(TOOLS_DIR)/go.mod - cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/conversion-gen k8s.io/code-generator/cmd/conversion-gen - -$(GOBINDATA): $(TOOLS_DIR)/go.mod # Build go-bindata from tools folder. - cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/go-bindata github.com/go-bindata/go-bindata/go-bindata - -$(RELEASE_NOTES): $(TOOLS_DIR)/go.mod - cd $(TOOLS_DIR) && go build -tags=tools -o $(RELEASE_NOTES_BIN) ./release - -$(LINK_CHECKER): $(TOOLS_DIR)/go.mod - cd $(TOOLS_DIR) && go build -tags=tools -o $(LINK_CHECKER_BIN) github.com/raviqqe/liche - -$(GO_APIDIFF): $(TOOLS_DIR)/go.mod - cd $(TOOLS_DIR) && go build -tags=tools -o $(GO_APIDIFF_BIN) github.com/joelanford/go-apidiff - -$(ENVSUBST): $(TOOLS_DIR)/go.mod - cd $(TOOLS_DIR) && go build -tags=tools -o $(ENVSUBST_BIN) github.com/drone/envsubst/cmd/envsubst - -envsubst: $(ENVSUBST) ## Build a local copy of envsubst. -kustomize: $(KUSTOMIZE) ## Build a local copy of kustomize. - .PHONY: e2e-framework e2e-framework: ## Builds the CAPI e2e framework - cd $(E2E_FRAMEWORK_DIR); go build ./... + go build ./test/framework/... ## -------------------------------------- ## Linting ## -------------------------------------- +LINT_ARGS ?= + .PHONY: lint lint-full lint: $(GOLANGCI_LINT) ## Lint codebase - $(GOLANGCI_LINT) run -v - cd $(E2E_FRAMEWORK_DIR); $(GOLANGCI_LINT) run -v - cd $(CAPD_DIR); $(GOLANGCI_LINT) run -v + $(GOLANGCI_LINT) run -v $(LINT_ARGS) + $(MAKE) -C $(CAPD_DIR) lint LINT_ARGS=$(LINT_ARGS) -lint-full: $(GOLANGCI_LINT) ## Run slower linters to detect possible issues - $(GOLANGCI_LINT) run -v --fast=false - cd $(E2E_FRAMEWORK_DIR); $(GOLANGCI_LINT) run -v --fast=false - cd $(CAPD_DIR); $(GOLANGCI_LINT) run -v --fast=false +lint-full: ## Run slower linters to detect possible issues + $(MAKE) lint LINT_ARGS=--fast=false -apidiff: $(GO_APIDIFF) ## Check for API differences - $(GO_APIDIFF) $(shell git rev-parse origin/master) --print-compatible +REMOTE ?=upstream + +apidiff: $(GO_APIDIFF) ## Check for API differences during development. Parameters: REMOTE (which git remote to use. Default: upstream) + @if !(git diff --quiet HEAD); then \ + git diff; \ + echo "commit changes first"; exit 1; \ + fi + + HEAD_COMMIT=$$(git rev-parse $(REMOTE)/master); \ + TMP_DIR=$$(mktemp -d -t apidiff-XXXXXXXXXX); \ + git clone . $${TMP_DIR}; \ + cd $${TMP_DIR}; \ + $(abspath $(GO_APIDIFF)) $${HEAD_COMMIT} --print-compatible; \ + rm -rf $${TMP_DIR} ## -------------------------------------- ## Generate / Manifests @@ -217,10 +192,11 @@ generate: ## Generate code $(MAKE) generate-manifests $(MAKE) generate-go $(MAKE) generate-bindata - $(MAKE) -C test/infrastructure/docker generate + $(MAKE) -C $(CAPD_DIR) generate .PHONY: generate-go -generate-go: ## Runs Go related generate targets +generate-go: $(GOBINDATA) ## Runs Go related generate targets + go generate ./... $(MAKE) generate-go-core $(MAKE) generate-go-kubeadm-bootstrap $(MAKE) generate-go-kubeadm-control-plane @@ -327,7 +303,7 @@ generate-kubeadm-control-plane-manifests: $(CONTROLLER_GEN) ## Generate manifest .PHONY: modules modules: ## Runs go mod to ensure modules are up to date. go mod tidy - cd $(TOOLS_DIR); go mod tidy + $(MAKE) -C $(TOOLS_DIR) modules $(MAKE) -C $(CAPD_DIR) modules ## -------------------------------------- @@ -335,28 +311,37 @@ modules: ## Runs go mod to ensure modules are up to date. ## -------------------------------------- .PHONY: docker-build -docker-build: ## Build the docker images for controller managers - $(MAKE) ARCH=$(ARCH) docker-build-core - $(MAKE) ARCH=$(ARCH) docker-build-kubeadm-bootstrap - $(MAKE) ARCH=$(ARCH) docker-build-kubeadm-control-plane +docker-build: docker-build-core docker-build-kubeadm-bootstrap docker-build-kubeadm-control-plane ## Build the docker images for controller managers .PHONY: docker-build-core -docker-build-core: ## Build the docker image for core controller manager +docker-build-core: .docker-build-core.sentinel ## Build the docker image for core controller manager + +CORE_SRCS := $(call rwildcard,controllers,*.*) $(call rwildcard,cmd,*.*) $(call rwildcard,feature,*.*) $(call rwildcard,exp,*.*) $(call rwildcard,api,*.*) $(call rwildcard,third_party,*.*) go.mod go.sum + +.docker-build-core.sentinel: $(CORE_SRCS) DOCKER_BUILDKIT=1 docker build --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) - $(MAKE) set-manifest-image MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./config/manager/manager_image_patch.yaml" - $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./config/manager/manager_pull_policy.yaml" + touch $@ .PHONY: docker-build-kubeadm-bootstrap -docker-build-kubeadm-bootstrap: ## Build the docker image for kubeadm bootstrap controller manager +docker-build-kubeadm-bootstrap: bootstrap/kubeadm/.docker-build-kubeadm-bootstrap.sentinel ## Build the docker image for kubeadm bootstrap controller manager + +KUBEADM_BOOTSTRAP_SRCS := $(CORE_SRCS) $(call rwildcard,bootstrap,*.*) +bootstrap/kubeadm/.docker-build-kubeadm-bootstrap.sentinel: $(KUBEADM_BOOTSTRAP_SRCS) DOCKER_BUILDKIT=1 docker build --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg package=./bootstrap/kubeadm --build-arg ldflags="$(LDFLAGS)" . -t $(KUBEADM_BOOTSTRAP_CONTROLLER_IMG)-$(ARCH):$(TAG) - $(MAKE) set-manifest-image MANIFEST_IMG=$(KUBEADM_BOOTSTRAP_CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./bootstrap/kubeadm/config/manager/manager_image_patch.yaml" - $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./bootstrap/kubeadm/config/manager/manager_pull_policy.yaml" + touch $@ .PHONY: docker-build-kubeadm-control-plane -docker-build-kubeadm-control-plane: ## Build the docker image for kubeadm control plane controller manager +docker-build-kubeadm-control-plane: controlplane/kubeadm/.docker-build-kubeadm-control-plane.sentinel ## Build the docker image for kubeadm control plane controller manager + +KUBEADM_CONTROLPLANE_SRCS := $(KUBEADM_BOOTSTRAP_SRCS) $(call rwildcard,controlplane,*.*) +controlplane/kubeadm/.docker-build-kubeadm-control-plane.sentinel: $(KUBEADM_CONTROLPLANE_SRCS) DOCKER_BUILDKIT=1 docker build --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg package=./controlplane/kubeadm --build-arg ldflags="$(LDFLAGS)" . -t $(KUBEADM_CONTROL_PLANE_CONTROLLER_IMG)-$(ARCH):$(TAG) - $(MAKE) set-manifest-image MANIFEST_IMG=$(KUBEADM_CONTROL_PLANE_CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./controlplane/kubeadm/config/manager/manager_image_patch.yaml" - $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./controlplane/kubeadm/config/manager/manager_pull_policy.yaml" + touch $@ + +.PHONY: docker-build-e2e +docker-build-e2e: ## Rebuild all Cluster API provider images to be used for e2e testing + $(MAKE) docker-build REGISTRY=gcr.io/k8s-staging-cluster-api PULL_POLICY=IfNotPresent + $(MAKE) -C $(CAPD_DIR) docker-build REGISTRY=gcr.io/k8s-staging-cluster-api .PHONY: docker-push docker-push: ## Push the docker images @@ -454,10 +439,10 @@ release: clean-release ## Builds and push container images using the latest git $(MAKE) set-manifest-pull-policy PULL_POLICY=IfNotPresent TARGET_RESOURCE="./controlplane/kubeadm/config/manager/manager_pull_policy.yaml" $(MAKE) release-manifests # Release CAPD components and add them to the release dir - $(MAKE) -C test/infrastructure/docker/ release - cp test/infrastructure/docker/out/infrastructure-components.yaml $(RELEASE_DIR)/infrastructure-components-development.yaml + $(MAKE) -C $(CAPD_DIR) release + cp $(CAPD_DIR)/out/infrastructure-components.yaml $(RELEASE_DIR)/infrastructure-components-development.yaml # Adds CAPD templates - cp test/infrastructure/docker/templates/* $(RELEASE_DIR)/ + cp $(CAPD_DIR)/templates/* $(RELEASE_DIR)/ .PHONY: release-manifests release-manifests: $(RELEASE_DIR) $(KUSTOMIZE) ## Builds the manifests to publish with a release @@ -529,6 +514,20 @@ docker-build-example-provider: ## Build the docker image for example provider clean: ## Remove all generated files $(MAKE) clean-bin $(MAKE) clean-book + $(MAKE) clean-artifacts + $(MAKE) clean-envtest + $(MAKE) clean-sentinels + +.PHONY: clean-sentinels +clean-sentinels: ## Delete docker build sentinels + rm -f .docker-build-core.sentinel + rm -f bootstrap/kubeadm/.docker-build-kubeadm-bootstrap.sentinel + rm -f controlplane/kubeadm/.docker-build-kubeadm-control-plane.sentinel + $(MAKE) -C $(CAPD_DIR) clean-sentinels + +.PHONY: clean-artifacts +clean-artifacts: ## Remove all test artifacts + rm -rf _artifacts .PHONY: clean-bin clean-bin: ## Remove all generated binaries @@ -547,11 +546,15 @@ clean-book: ## Remove all generated GitBook files clean-bindata: ## Remove bindata generated folder rm -rf $(GOBINDATA_CLUSTERCTL_DIR)/manifest -.PHONY: clean-manifests ## Reset manifests in config directories back to master -clean-manifests: +.PHONY: clean-manifests +clean-manifests: ## Reset manifests in config directories back to master @read -p "WARNING: This will reset all config directories to local master. Press [ENTER] to continue." git checkout master config bootstrap/kubeadm/config controlplane/kubeadm/config test/infrastructure/docker/config +.PHONY: clean-envtest +clean-envtest: ## Remove kubebuilder envtest binaries and tars + -rm -rf /tmp/kubebuilder* + .PHONY: format-tiltfile format-tiltfile: ## Format Tiltfile ./hack/verify-starlark.sh fix @@ -588,12 +591,13 @@ verify-gen: generate .PHONY: verify-docker-provider verify-docker-provider: @echo "Verifying CAPD" - cd $(CAPD_DIR); $(MAKE) verify + $(MAKE) -C $(CAPD_DIR) verify .PHONY: verify-book-links verify-book-links: $(LINK_CHECKER) # Ignore localhost links and set concurrency to a reasonable number $(LINK_CHECKER) -r docs/book -x "^https?://" -c 10 + ## -------------------------------------- ## Others / Utilities ## -------------------------------------- diff --git a/common.mk b/common.mk new file mode 100644 index 000000000000..59e63bbdcd49 --- /dev/null +++ b/common.mk @@ -0,0 +1,65 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include $(ROOT_DIR_RELATIVE)/versions.mk + +# Ensure Make is run with bash shell as some syntax below is bash-specific +SHELL:=bash +.ONESHELL: +.SHELLFLAGS := -eu -o pipefail -c +.DELETE_ON_ERROR: +MAKEFLAGS += --warn-undefined-variables +MAKEFLAGS += --no-builtin-rules + +TOOLS_DIR := $(ROOT_DIR_RELATIVE)/hack/tools +TOOLS_DIR_DEPS := $(TOOLS_DIR)/go.sum $(TOOLS_DIR)/go.mod $(TOOLS_DIR)/Makefile +TOOLS_BIN_DIR := $(TOOLS_DIR)/bin + +PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH) +export PATH + +UID := $(shell id -u) +GID := $(shell id -g) + +rwildcard=$(foreach d,$(wildcard $(1:=/*)),$(call rwildcard,$d,$2) $(filter $(subst *,%,$2),$d)) + +# Hosts running SELinux need :z added to volume mounts +SELINUX_ENABLED := $(shell cat /sys/fs/selinux/enforce 2> /dev/null || echo 0) + +ifeq ($(SELINUX_ENABLED),1) + DOCKER_VOL_OPTS?=:z +endif + +# This option is for running docker manifest command +export DOCKER_CLI_EXPERIMENTAL := enabled + +# Use GOPROXY environment variable if set +GOPROXY := $(shell go env GOPROXY) +ifeq ($(GOPROXY),) +GOPROXY := https://proxy.golang.org +endif +export GOPROXY + + +$(TOOLS_BIN_DIR)/%: $(TOOLS_DIR_DEPS) + make -C $(TOOLS_DIR) $(@:hack/tools/%=%) + +## -------------------------------------- +## Help +## -------------------------------------- + +.DEFAULT_GOAL:=help + +help: ## Display this help + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/hack/tools/Makefile b/hack/tools/Makefile new file mode 100644 index 000000000000..1ff0f8cdf08c --- /dev/null +++ b/hack/tools/Makefile @@ -0,0 +1,115 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ROOT_DIR_RELATIVE := ../.. +include $(ROOT_DIR_RELATIVE)/common.mk + +UNAME := $(shell uname -s) + +# Directories. +BIN_DIR := bin +SHARE_DIR := share + +OS := $(shell go env GOOS) +RUST_TARGET := unknown-$(OS)-gnu +MDBOOK_EXTRACT_COMMAND := tar xfvz $(SHARE_DIR)/mdbook.tar.gz -C bin +MDBOOK_ARCHIVE_EXT := .tar.gz + +ifeq ($(OS), windows) + RUST_TARGET := pc-windows-msvc + MDBOOK_ARCHIVE_EXT := .zip + MDBOOK_EXTRACT_COMMAND := unzip -d /tmp +endif + +ifeq ($(OS), darwin) + RUST_TARGET := apple-darwin +endif + +## -------------------------------------- +## Tooling Binaries +## -------------------------------------- + +.PHONY: modules +modules: ## Runs go mod to ensure modules are up to date. + go mod tidy + +$(BIN_DIR): + mkdir -p $@ + +$(SHARE_DIR): + mkdir -p $@ + +CONTROLLER_GEN := $(BIN_DIR)/controller-gen +$(CONTROLLER_GEN): $(BIN_DIR) go.mod go.sum # Build controller-gen from tools folder. + go build -tags=tools -o $@ sigs.k8s.io/controller-tools/cmd/controller-gen + +CONVERSION_GEN := $(BIN_DIR)/conversion-gen +$(CONVERSION_GEN): $(BIN_DIR) go.mod go.sum + go build -tags=tools -o $@ k8s.io/code-generator/cmd/conversion-gen + +DEFAULTER_GEN := $(BIN_DIR)/defaulter-gen +$(DEFAULTER_GEN): $(BIN_DIR) go.mod go.sum + go build -tags=tools -o $@ k8s.io/code-generator/cmd/defaulter-gen + +ENVSUBST := $(BIN_DIR)/envsubst +$(ENVSUBST): $(BIN_DIR) go.mod go.sum # Build envsubst from tools folder. + go build -tags=tools -o $@ github.com/a8m/envsubst/cmd/envsubst + +GINKGO := $(BIN_DIR)/ginkgo +$(GINKGO): $(BIN_DIR) go.mod go.sum + go build -tags=tools -o $@ github.com/onsi/ginkgo/ginkgo + +GOLANGCI_LINT := $(BIN_DIR)/golangci-lint +$(GOLANGCI_LINT): $(BIN_DIR) go.mod go.sum # Build golangci-lint from tools folder. + go build -tags=tools -o $@ github.com/golangci/golangci-lint/cmd/golangci-lint + +KIND := $(BIN_DIR)/kind +$(KIND): $(BIN_DIR) go.mod go.sum + go build -tags tools -o $@ sigs.k8s.io/kind + +KUSTOMIZE := $(BIN_DIR)/kustomize +$(KUSTOMIZE): $(BIN_DIR) go.mod go.sum # Build kustomize from tools folder. + go build -tags=tools -o $@ sigs.k8s.io/kustomize/kustomize/v3 + +MDBOOK_EMBED := $(BIN_DIR)/mdbook-embed +$(MDBOOK_EMBED): $(BIN_DIR) go.mod go.sum + go build -tags=tools -o $(BIN_DIR)/mdbook-embed ./mdbook/embed + +MDBOOK_RELEASELINK := $(BIN_DIR)/mdbook-releaselink +$(MDBOOK_RELEASELINK): $(BIN_DIR) go.mod go.sum + go build -tags=tools -o $(BIN_DIR)/mdbook-releaselink ./mdbook/releaselink + +MDBOOK_TABULATE := $(BIN_DIR)/mdbook-tabulate +$(MDBOOK_TABULATE): $(BIN_DIR) go.mod go.sum + go build -tags=tools -o $(BIN_DIR)/mdbook-tabulate ./mdbook/tabulate + +MOCKGEN := $(BIN_DIR)/mockgen +$(MOCKGEN): $(BIN_DIR) go.mod go.sum # Build mockgen from tools folder. + go build -tags=tools -o $@ github.com/golang/mock/mockgen + +RELEASE_NOTES := $(BIN_DIR)/release-notes +$(RELEASE_NOTES): $(BIN_DIR) go.mod go.sum + go build -tags tools -o $@ ./release + +LINK_CHECKER := $(BIN_DIR)/link-checker +$(LINK_CHECKER): $(BIN_DIR) go.mod go.sum + go build -tags tools -o $@ github.com/raviqqe/liche + +GOBINDATA := $(BIN_DIR)/go-bindata +$(GOBINDATA): $(BIN_DIR) go.mod go.sum + go build -tags tools -o $@ github.com/go-bindata/go-bindata/go-bindata + +GOAPIDIFF := $(BIN_DIR)/go-apidiff +$(GOAPIDIFF): $(BIN_DIR) go.mod go.sum + go build -tags tools -o $@ github.com/joelanford/go-apidiff diff --git a/hack/tools/tools.mk b/hack/tools/tools.mk new file mode 100644 index 000000000000..a280d05b4ffc --- /dev/null +++ b/hack/tools/tools.mk @@ -0,0 +1,12 @@ +# Binaries. +KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize +CONTROLLER_GEN:= $(TOOLS_BIN_DIR)/controller-gen +GOLANGCI_LINT := $(TOOLS_BIN_DIR)/golangci-lint +CONVERSION_GEN := $(TOOLS_BIN_DIR)/conversion-gen +DEFAULTER_GEN := $(TOOLS_BIN_DIR)/defaulter-gen +GINKGO := $(TOOLS_BIN_DIR)/ginkgo +ENVSUBST := $(TOOLS_BIN_DIR)/envsubst +GOBINDATA := $(TOOLS_BIN_DIR)/go-bindata +RELEASE_NOTES := $(TOOLS_BIN_DIR)/release-notes +GO_APIDIFF := $(TOOLS_BIN_DIR)/go-apidiff +LINK_CHECKER := $(TOOLS_BIN_DIR)/link-checker diff --git a/scripts/ci-conformance.sh b/scripts/ci-conformance.sh new file mode 100644 index 000000000000..570e74518238 --- /dev/null +++ b/scripts/ci-conformance.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +REPO_ROOT=$(git rev-parse --show-toplevel) +cd "${REPO_ROOT}" || exit 1 + +# shellcheck source=./hack/ensure-go.sh +source "${REPO_ROOT}/hack/ensure-go.sh" +# shellcheck source=./hack/ensure-kubectl.sh +source "${REPO_ROOT}/hack/ensure-kubectl.sh" +# shellcheck source=./hack/ensure-kustomize.sh +source "${REPO_ROOT}/hack/ensure-kustomize.sh" + +# Configure provider images generation; +# please ensure the generated image name matches image names used in the E2E_CONF_FILE +export REGISTRY=gcr.io/k8s-staging-cluster-api +export TAG=ci +export ARCH=amd64 +export PULL_POLICY=IfNotPresent + +## Rebuild all Cluster API provider images +make docker-build + +## Rebuild CAPD provider images +make -C test/infrastructure/docker docker-build + +## Pulling cert manager images so we can pre-load in kind nodes +docker pull quay.io/jetstack/cert-manager-cainjector:v0.16.1 +docker pull quay.io/jetstack/cert-manager-webhook:v0.16.1 +docker pull quay.io/jetstack/cert-manager-controller:v0.16.1 + +## Pulling kind images used by tests +docker pull kindest/node:v1.18.2 +docker pull kindest/node:v1.17.2 + +# Configure e2e tests +export GINKGO_NODES=3 +export GINKGO_NOCOLOR=true +export GINKGO_ARGS="--failFast" # Other ginkgo args that need to be appended to the command. +export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/docker-ci.yaml" +export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}" +export SKIP_RESOURCE_CLEANUP=false +export USE_EXISTING_CLUSTER=false + +# Run conformance tests +mkdir -p "$ARTIFACTS" +make test-conformance diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index ad3ef8d5243b..eb151d571dc5 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -61,4 +61,4 @@ export USE_EXISTING_CLUSTER=false # Run e2e tests mkdir -p "$ARTIFACTS" -make -C test/e2e/ run +make test-e2e diff --git a/test/e2e/Makefile b/test/e2e/Makefile deleted file mode 100644 index b14832073601..000000000000 --- a/test/e2e/Makefile +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2020 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# If you update this file, please follow: -# https://suva.sh/posts/well-documented-makefiles/ - -# Use GOPROXY environment variable if set - -.DEFAULT_GOAL:=help - -GOPROXY := $(shell go env GOPROXY) -ifeq ($(GOPROXY),) -GOPROXY := https://proxy.golang.org -endif -export GOPROXY - -REPO_ROOT := $(shell git rev-parse --show-toplevel) - -help: ## Display this help - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -## -------------------------------------- -## Binaries -## -------------------------------------- - -TOOLS_DIR := $(REPO_ROOT)/hack/tools -BIN_DIR := bin -TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR) -GINKGO := $(TOOLS_BIN_DIR)/ginkgo - -.PHONY: ginkgo -ginkgo: - cd $(TOOLS_DIR) && go build -tags=tools -o $(BIN_DIR)/ginkgo github.com/onsi/ginkgo/ginkgo - -## -------------------------------------- -## Testing -## -------------------------------------- - -TEST_E2E_DIR := $(REPO_ROOT)/test/e2e - -GINKGO_FOCUS ?= -GINKGO_NODES ?= 1 -E2E_CONF_FILE ?= ${REPO_ROOT}/test/e2e/config/docker-dev.yaml -ARTIFACTS ?= ${REPO_ROOT}/_artifacts -SKIP_RESOURCE_CLEANUP ?= false -USE_EXISTING_CLUSTER ?= false -GINKGO_NOCOLOR ?= false - -.PHONY: run -run: ginkgo ## Run the end-to-end tests - cd $(TEST_E2E_DIR); $(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) $(GINKGO_ARGS) . -- \ - -e2e.artifacts-folder="$(ARTIFACTS)" \ - -e2e.config="$(E2E_CONF_FILE)" \ - -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) diff --git a/test/e2e/common.go b/test/e2e/common.go index 896a1d9f6d92..dbd70b77ccce 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -17,18 +17,11 @@ limitations under the License. package e2e import ( - "context" "fmt" - "path/filepath" - - . "github.com/onsi/ginkgo" "github.com/blang/semver" "github.com/onsi/gomega/types" - corev1 "k8s.io/api/core/v1" - clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" - "sigs.k8s.io/cluster-api/test/framework" - "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" ) // Test suite constants for e2e config variables @@ -42,54 +35,9 @@ const ( CoreDNSVersionUpgradeTo = "COREDNS_VERSION_UPGRADE_TO" ) +// Byf is deprecated. Use "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" as dot import instead. func Byf(format string, a ...interface{}) { - By(fmt.Sprintf(format, a...)) -} - -func setupSpecNamespace(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string) (*corev1.Namespace, context.CancelFunc) { - Byf("Creating a namespace for hosting the %q test spec", specName) - namespace, cancelWatches := framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ - Creator: clusterProxy.GetClient(), - ClientSet: clusterProxy.GetClientSet(), - Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), - LogFolder: filepath.Join(artifactFolder, "clusters", clusterProxy.GetName()), - }) - - return namespace, cancelWatches -} - -func dumpSpecResourcesAndCleanup(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string, namespace *corev1.Namespace, cancelWatches context.CancelFunc, cluster *clusterv1.Cluster, intervalsGetter func(spec, key string) []interface{}, skipCleanup bool) { - Byf("Dumping logs from the %q workload cluster", cluster.Name) - - // Dump all the logs from the workload cluster before deleting them. - clusterProxy.CollectWorkloadClusterLogs(ctx, cluster.Namespace, cluster.Name, filepath.Join(artifactFolder, "clusters", cluster.Name, "machines")) - - Byf("Dumping all the Cluster API resources in the %q namespace", namespace.Name) - - // Dump all Cluster API related resources to artifacts before deleting them. - framework.DumpAllResources(ctx, framework.DumpAllResourcesInput{ - Lister: clusterProxy.GetClient(), - Namespace: namespace.Name, - LogPath: filepath.Join(artifactFolder, "clusters", clusterProxy.GetName(), "resources"), - }) - - if !skipCleanup { - Byf("Deleting cluster %s/%s", cluster.Namespace, cluster.Name) - // While https://github.com/kubernetes-sigs/cluster-api/issues/2955 is addressed in future iterations, there is a chance - // that cluster variable is not set even if the cluster exists, so we are calling DeleteAllClustersAndWait - // instead of DeleteClusterAndWait - framework.DeleteAllClustersAndWait(ctx, framework.DeleteAllClustersAndWaitInput{ - Client: clusterProxy.GetClient(), - Namespace: namespace.Name, - }, intervalsGetter(specName, "wait-delete-cluster")...) - - Byf("Deleting namespace used for hosting the %q test spec", specName) - framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{ - Deleter: clusterProxy.GetClient(), - Name: namespace.Name, - }) - } - cancelWatches() + ginkgoextensions.Byf(format, a...) } // HaveValidVersion succeeds if version is a valid semver version diff --git a/test/e2e/config/docker-ci.yaml b/test/e2e/config/docker-ci.yaml index 54f469ef52ff..0c1d7e2f8af1 100644 --- a/test/e2e/config/docker-ci.yaml +++ b/test/e2e/config/docker-ci.yaml @@ -35,6 +35,8 @@ providers: replacements: - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 + - old: "gcr.io/k8s-staging-cluster-api/cluster-api-controller:master" + new: "gcr.io/k8s-staging-cluster-api/cluster-api-controller-amd64:ci" - name: kubeadm type: BootstrapProvider @@ -45,6 +47,8 @@ providers: replacements: - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 + - old: "gcr.io/k8s-staging-cluster-api/cluster-api-controller:master" + new: "gcr.io/k8s-staging-cluster-api/cluster-api-controller-amd64:ci" - name: kubeadm type: ControlPlaneProvider @@ -55,6 +59,8 @@ providers: replacements: - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 + - old: "gcr.io/k8s-staging-cluster-api/cluster-api-controller:master" + new: "gcr.io/k8s-staging-cluster-api/cluster-api-controller-amd64:ci" - name: docker type: InfrastructureProvider @@ -65,6 +71,8 @@ providers: replacements: - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 + - old: "gcr.io/k8s-staging-cluster-api/cluster-api-controller:master" + new: "gcr.io/k8s-staging-cluster-api/cluster-api-controller-amd64:ci" files: # Add a metadata for docker provider - sourcePath: "../data/infrastructure-docker/metadata.yaml" diff --git a/test/e2e/config/docker-dev.yaml b/test/e2e/config/docker-dev.yaml index bd9775a2c9d7..6ba27b91d99f 100644 --- a/test/e2e/config/docker-dev.yaml +++ b/test/e2e/config/docker-dev.yaml @@ -50,6 +50,8 @@ providers: new: "--enable-leader-election=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 + - old: "gcr.io/k8s-staging-cluster-api/cluster-api-controller:master" + new: "gcr.io/k8s-staging-cluster-api/cluster-api-controller-amd64:dev" - name: kubeadm type: BootstrapProvider @@ -64,6 +66,8 @@ providers: new: "--enable-leader-election=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 + - old: "gcr.io/k8s-staging-cluster-api/kubeadm-bootstrap-controller:master" + new: "gcr.io/k8s-staging-cluster-api/kubeadm-bootstrap-controller-amd64:dev" - name: kubeadm type: ControlPlaneProvider @@ -78,6 +82,8 @@ providers: new: "--enable-leader-election=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 + - old: "gcr.io/k8s-staging-cluster-api/kubeadm-control-plane-controller:master" + new: "gcr.io/k8s-staging-cluster-api/kubeadm-control-plane-controller-amd64:dev" - name: docker type: InfrastructureProvider @@ -92,6 +98,8 @@ providers: new: "--enable-leader-election=false" - old: --metrics-addr=127.0.0.1:8080 new: --metrics-addr=:8080 + - old: "gcr.io/k8s-staging-cluster-api/capd-manager:dev" + new: "gcr.io/k8s-staging-cluster-api/capd-manager-amd64:dev" files: # Add a metadata for docker provider - sourcePath: "../data/infrastructure-docker/metadata.yaml" diff --git a/test/e2e/conformance/ci_artifacts.go b/test/e2e/conformance/ci_artifacts.go new file mode 100644 index 000000000000..4cdba47343e1 --- /dev/null +++ b/test/e2e/conformance/ci_artifacts.go @@ -0,0 +1,137 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conformance + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/test/framework/kubetest" + "sigs.k8s.io/cluster-api/util" +) + +// ConformanceCIArtifactsSpecInput is the input for ConformanceSpec. +type ConformanceCIArtifactsSpecInput struct { + E2EConfig *clusterctl.E2EConfig + ClusterctlConfigPath string + BootstrapClusterProxy framework.ClusterProxy + ArtifactFolder string + SkipCleanup bool +} + +// ConformanceSpec implements a spec that mimics the operation described in the Cluster API quick start, that is +// creating a workload cluster. +// This test is meant to provide a first, fast signal to detect regression; it is recommended to use it as a PR blocker test. +func ConformanceCIArtifactsSpec(ctx context.Context, inputGetter func() ConformanceCIArtifactsSpecInput) { + var ( + specName = "conformance-with-ci-artifacts" + input ConformanceCIArtifactsSpecInput + namespace *corev1.Namespace + cancelWatches context.CancelFunc + cluster *clusterv1.Cluster + ) + + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) + + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) + }) + + It("Should create a workload cluster", func() { + + By("Creating a workload cluster") + name := fmt.Sprintf("cluster-%s", util.RandomString(6)) + kubernetesVersion, err := kubetest.FetchKubernetesCIVersion() + Expect(err).NotTo(HaveOccurred()) + cluster, _, _ = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "with-ci-artifacts", + Namespace: namespace.Name, + ClusterName: name, + KubernetesVersion: kubernetesVersion, + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + + workloadProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, name) + By("Running conformance suite") + + kubetestPath, err := filepath.Abs(path.Join("..", "data", "kubetest", "conformance.yaml")) + Expect(err).ToNot(HaveOccurred()) + + Expect(kubetest.Run(kubetest.RunInput{ + ClusterProxy: workloadProxy, + KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), + ConfigFilePath: kubetestPath, + })).To(Succeed()) + + By("PASSED!") + }) + + AfterEach(func() { + // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) + }) +} diff --git a/test/e2e/conformance/ci_artifacts_test.go b/test/e2e/conformance/ci_artifacts_test.go new file mode 100644 index 000000000000..409ef0532f17 --- /dev/null +++ b/test/e2e/conformance/ci_artifacts_test.go @@ -0,0 +1,39 @@ +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conformance + +import ( + "context" + + . "github.com/onsi/ginkgo" +) + +var _ = Describe("[Conformance] When running conformance with CI Artifacts", func() { + + ConformanceCIArtifactsSpec(context.TODO(), func() ConformanceCIArtifactsSpecInput { + return ConformanceCIArtifactsSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + } + }) + +}) diff --git a/test/e2e/conformance/conformance.go b/test/e2e/conformance/conformance.go new file mode 100644 index 000000000000..b50bb3b18198 --- /dev/null +++ b/test/e2e/conformance/conformance.go @@ -0,0 +1,139 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conformance + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/test/framework/kubetest" + "sigs.k8s.io/cluster-api/util" +) + +const ( + KubernetesVersion = "KUBERNETES_VERSION" +) + +// ConformanceSpecInput is the input for ConformanceSpec. +type ConformanceSpecInput struct { + E2EConfig *clusterctl.E2EConfig + ClusterctlConfigPath string + BootstrapClusterProxy framework.ClusterProxy + ArtifactFolder string + SkipCleanup bool +} + +// ConformanceSpec implements a spec that mimics the operation described in the Cluster API quick start, that is +// creating a workload cluster. +// This test is meant to provide a first, fast signal to detect regression; it is recommended to use it as a PR blocker test. +func ConformanceSpec(ctx context.Context, inputGetter func() ConformanceSpecInput) { + var ( + specName = "conformance" + input ConformanceSpecInput + namespace *corev1.Namespace + cancelWatches context.CancelFunc + cluster *clusterv1.Cluster + ) + + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) + + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) + }) + + It("Should create a workload cluster", func() { + + By("Creating a workload cluster") + name := fmt.Sprintf("cluster-%s", util.RandomString(6)) + cluster, _, _ = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: clusterctl.DefaultFlavor, + Namespace: namespace.Name, + ClusterName: name, + KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + + workloadProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, name) + By("Running conformance suite") + + kubetestPath, err := filepath.Abs(path.Join("..", "data", "kubetest", "conformance.yaml")) + Expect(err).ToNot(HaveOccurred()) + + Expect(kubetest.Run(kubetest.RunInput{ + ClusterProxy: workloadProxy, + KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), + ConfigFilePath: kubetestPath, + })).To(Succeed()) + + By("PASSED!") + }) + + AfterEach(func() { + // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) + }) +} diff --git a/test/e2e/conformance/conformance_suite_test.go b/test/e2e/conformance/conformance_suite_test.go new file mode 100644 index 000000000000..0d72d3c2bf6d --- /dev/null +++ b/test/e2e/conformance/conformance_suite_test.go @@ -0,0 +1,179 @@ +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conformance + +import ( + "flag" + "os" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/bootstrap" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" + "sigs.k8s.io/yaml" +) + +// Test suite flags +var ( + // configPath is the path to the e2e config file. + configPath string + + // useExistingCluster instructs the test to use the current cluster instead of creating a new one (default discovery rules apply). + useExistingCluster bool + + // artifactFolder is the folder to store e2e test artifacts. + artifactFolder string + + // skipCleanup prevents cleanup of test resources e.g. for debug purposes. + skipCleanup bool +) + +// Test suite global vars +var ( + // e2eConfig to be used for this test, read from configPath. + e2eConfig *clusterctl.E2EConfig + + // clusterctlConfigPath to be used for this test, created by generating a clusterctl local repository + // with the providers specified in the configPath. + clusterctlConfigPath string + + // bootstrapClusterProvider manages provisioning of the the bootstrap cluster to be used for the e2e tests. + // Please note that provisioning will be skipped if e2e.use-existing-cluster is provided. + bootstrapClusterProvider bootstrap.ClusterProvider + + // bootstrapClusterProxy allows to interact with the bootstrap cluster to be used for the e2e tests. + bootstrapClusterProxy framework.ClusterProxy +) + +func init() { + flag.StringVar(&configPath, "e2e.config", "", "path to the e2e config file") + flag.StringVar(&artifactFolder, "e2e.artifacts-folder", "", "folder where e2e test artifact should be stored") + flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped") + flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)") +} + +func TestE2E(t *testing.T) { + artifactFolder := framework.ResolveArtifactsDirectory(artifactFolder) + RegisterFailHandler(Fail) + RunSpecsWithDefaultAndCustomReporters(t, "capi-conformance", []Reporter{framework.CreateJUnitReporterForProw(artifactFolder)}) +} + +type suiteConfig struct { + ArtifactFolder string `json:"artifactFolder,omitempty"` + ConfigPath string `json:"configPath,omitempty"` + ClusterctlConfigPath string `json:"clusterctlConfigPath,omitempty"` + KubeconfigPath string `json:"kubeconfigPath,omitempty"` +} + +// Using a SynchronizedBeforeSuite for controlling how to create resources shared across ParallelNodes (~ginkgo threads). +// The local clusterctl repository & the bootstrap cluster are created once and shared across all the tests. +var _ = SynchronizedBeforeSuite(func() []byte { + // Before all ParallelNodes. + + Expect(configPath).To(BeAnExistingFile(), "Invalid test suite argument. e2e.config should be an existing file.") + artifactFolder := framework.ResolveArtifactsDirectory(artifactFolder) + log := ginkgoextensions.Log.WithValues("config-path", configPath, "artifacts-directory", artifactFolder) + Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid test suite argument. Can't create e2e.artifacts-folder %q", artifactFolder) + + By("Initializing a runtime.Scheme with all the GVK relevant for this test") + scheme := setup.InitScheme() + + log.Info("Loading the e2e test configuration") + e2eConfig = setup.LoadE2EConfig(configPath) + + log.Info("Creating a clusterctl local repository in artifacts directory") + + setup.CreateCIArtifactsTemplate(artifactFolder, "..", e2eConfig) + + clusterctlConfigPath = setup.CreateClusterctlLocalRepository(e2eConfig, filepath.Join(artifactFolder, "repository")) + + log.Info("Setting up the bootstrap cluster") + // e2eConfig, scheme, useExistingCluster + bootstrapClusterProvider, bootstrapClusterProxy = setup.CreateBootstrapCluster( + setup.CreateBootstrapClusterInput{ + E2EConfig: e2eConfig, + Scheme: scheme, + UseExistingCluster: useExistingCluster, + }, + ) + + log.Info("Initializing the bootstrap cluster") + // setup.InitBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder) + setup.InitBootstrapCluster( + setup.InitBootstrapClusterInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + E2EConfig: e2eConfig, + ClusterctlConfig: clusterctlConfigPath, + ArtifactsDirectory: artifactFolder, + }, + ) + + conf := suiteConfig{ + ArtifactFolder: artifactFolder, + ConfigPath: configPath, + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + } + + data, err := yaml.Marshal(conf) + Expect(err).NotTo(HaveOccurred()) + return data + +}, func(data []byte) { + // Before each ParallelNode. + + conf := &suiteConfig{} + err := yaml.UnmarshalStrict(data, conf) + Expect(err).NotTo(HaveOccurred()) + + artifactFolder = conf.ArtifactFolder + configPath = conf.ConfigPath + clusterctlConfigPath = conf.ClusterctlConfigPath + kubeconfigPath := conf.KubeconfigPath + + e2eConfig = setup.LoadE2EConfig(configPath) + bootstrapClusterProxy = framework.NewClusterProxy("bootstrap", kubeconfigPath, initScheme()) +}) + +// Using a SynchronizedAfterSuite for controlling how to delete resources shared across ParallelNodes (~ginkgo threads). +// The bootstrap cluster is shared across all the tests, so it should be deleted only after all ParallelNodes completes. +// The local clusterctl repository is preserved like everything else created into the artifact folder. +var _ = SynchronizedAfterSuite(func() { + // After each ParallelNode. +}, func() { + // After all ParallelNodes. + log := ginkgoextensions.Log.WithValues("config-path", configPath, "artifacts-directory", artifactFolder) + log.Info("Tearing down the management cluster") + if !skipCleanup { + setup.TearDown(bootstrapClusterProvider, bootstrapClusterProxy) + } +}) + +func initScheme() *runtime.Scheme { + sc := runtime.NewScheme() + framework.TryAddDefaultSchemes(sc) + return sc +} diff --git a/test/e2e/conformance/conformance_test.go b/test/e2e/conformance/conformance_test.go new file mode 100644 index 000000000000..4de215981062 --- /dev/null +++ b/test/e2e/conformance/conformance_test.go @@ -0,0 +1,39 @@ +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conformance + +import ( + "context" + + . "github.com/onsi/ginkgo" +) + +var _ = Describe("[Conformance] When running conformance", func() { + + ConformanceSpec(context.TODO(), func() ConformanceSpecInput { + return ConformanceSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + } + }) + +}) diff --git a/test/e2e/data/infrastructure-docker/cluster-template-ci.yaml b/test/e2e/data/infrastructure-docker/cluster-template-ci.yaml index ce66062a76e1..5cc5964cf1e5 100644 --- a/test/e2e/data/infrastructure-docker/cluster-template-ci.yaml +++ b/test/e2e/data/infrastructure-docker/cluster-template-ci.yaml @@ -51,7 +51,7 @@ spec: controllerManager: extraArgs: {enable-hostpath-provisioner: 'true'} apiServer: - certSANs: [localhost, 127.0.0.1] + certSANs: [localhost, "host.docker.internal", 127.0.0.1] initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock diff --git a/test/e2e/data/infrastructure-docker/cluster-template-kcp-adoption.yaml b/test/e2e/data/infrastructure-docker/cluster-template-kcp-adoption.yaml index 0ed2d0f33c4d..e6b44fd41b52 100644 --- a/test/e2e/data/infrastructure-docker/cluster-template-kcp-adoption.yaml +++ b/test/e2e/data/infrastructure-docker/cluster-template-kcp-adoption.yaml @@ -54,7 +54,7 @@ spec: controllerManager: extraArgs: {enable-hostpath-provisioner: 'true'} apiServer: - certSANs: [localhost, 127.0.0.1] + certSANs: [localhost, "host.docker.internal", 127.0.0.1] initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock diff --git a/test/e2e/data/infrastructure-docker/cluster-template.yaml b/test/e2e/data/infrastructure-docker/cluster-template.yaml index ce66062a76e1..5cc5964cf1e5 100644 --- a/test/e2e/data/infrastructure-docker/cluster-template.yaml +++ b/test/e2e/data/infrastructure-docker/cluster-template.yaml @@ -51,7 +51,7 @@ spec: controllerManager: extraArgs: {enable-hostpath-provisioner: 'true'} apiServer: - certSANs: [localhost, 127.0.0.1] + certSANs: [localhost, "host.docker.internal", 127.0.0.1] initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock diff --git a/test/e2e/data/infrastructure-docker/platform-kustomization.yaml b/test/e2e/data/infrastructure-docker/platform-kustomization.yaml new file mode 100644 index 000000000000..f07050ebbf2b --- /dev/null +++ b/test/e2e/data/infrastructure-docker/platform-kustomization.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: DockerMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + customImage: kindest/node:${DOCKER_NODE_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3 +kind: DockerMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + customImage: kindest/node:${DOCKER_NODE_VERSION} + bootstrapTimeout: 10m diff --git a/test/e2e/data/kubetest/conformance-fast.yaml b/test/e2e/data/kubetest/conformance-fast.yaml new file mode 100644 index 000000000000..6f7936378b14 --- /dev/null +++ b/test/e2e/data/kubetest/conformance-fast.yaml @@ -0,0 +1,8 @@ +ginkgo.focus: \[Conformance\] +ginkgo.skip: \[sig-scheduling\].*\[Serial\] +disable-log-dump: true +ginkgo.progress: true +ginkgo.slowSpecThreshold: 120.0 +ginkgo.flakeAttempts: 3 +ginkgo.trace: true +ginkgo.v: true diff --git a/test/e2e/data/kubetest/conformance.yaml b/test/e2e/data/kubetest/conformance.yaml new file mode 100644 index 000000000000..d748f432888b --- /dev/null +++ b/test/e2e/data/kubetest/conformance.yaml @@ -0,0 +1,7 @@ +ginkgo.focus: \[Conformance\] +disable-log-dump: true +ginkgo.progress: true +ginkgo.slowSpecThreshold: 120.0 +ginkgo.flakeAttempts: 3 +ginkgo.trace: true +ginkgo.v: true diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 0c77876e4223..bae2e66084f6 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -19,23 +19,21 @@ limitations under the License. package e2e import ( - "context" "flag" - "fmt" "os" "path/filepath" - "strings" "testing" . "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/bootstrap" "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" + "sigs.k8s.io/yaml" ) // Test suite flags @@ -78,15 +76,16 @@ func init() { } func TestE2E(t *testing.T) { - // If running in prow, make sure to use the artifacts folder that will be reported in test grid (ignoring the value provided by flag). - if prowArtifactFolder, exists := os.LookupEnv("ARTIFACTS"); exists { - artifactFolder = prowArtifactFolder - } - + artifactFolder := framework.ResolveArtifactsDirectory(artifactFolder) RegisterFailHandler(Fail) - junitPath := filepath.Join(artifactFolder, fmt.Sprintf("junit.e2e_suite.%d.xml", config.GinkgoConfig.ParallelNode)) - junitReporter := reporters.NewJUnitReporter(junitPath) - RunSpecsWithDefaultAndCustomReporters(t, "capi-e2e", []Reporter{junitReporter}) + RunSpecsWithDefaultAndCustomReporters(t, "capi-e2e", []Reporter{framework.CreateJUnitReporterForProw(artifactFolder)}) +} + +type suiteConfig struct { + ArtifactFolder string `json:"artifactFolder,omitempty"` + ConfigPath string `json:"configPath,omitempty"` + ClusterctlConfigPath string `json:"clusterctlConfigPath,omitempty"` + KubeconfigPath string `json:"kubeconfigPath,omitempty"` } // Using a SynchronizedBeforeSuite for controlling how to create resources shared across ParallelNodes (~ginkgo threads). @@ -95,43 +94,65 @@ var _ = SynchronizedBeforeSuite(func() []byte { // Before all ParallelNodes. Expect(configPath).To(BeAnExistingFile(), "Invalid test suite argument. e2e.config should be an existing file.") + artifactFolder := framework.ResolveArtifactsDirectory(artifactFolder) + log := ginkgoextensions.Log.WithValues("config-path", configPath, "artifacts-directory", artifactFolder) Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid test suite argument. Can't create e2e.artifacts-folder %q", artifactFolder) By("Initializing a runtime.Scheme with all the GVK relevant for this test") - scheme := initScheme() - - Byf("Loading the e2e test configuration from %q", configPath) - e2eConfig = loadE2EConfig(configPath) + scheme := setup.InitScheme() + + log.Info("Loading the e2e test configuration") + e2eConfig = setup.LoadE2EConfig(configPath) + + log.Info("Creating a clusterctl local repository in artifacts directory") + setup.CreateCIArtifactsTemplate(artifactFolder, ".", e2eConfig) + clusterctlConfigPath = setup.CreateClusterctlLocalRepository(e2eConfig, filepath.Join(artifactFolder, "repository")) + + log.Info("Setting up the bootstrap cluster") + // e2eConfig, scheme, useExistingCluster + bootstrapClusterProvider, bootstrapClusterProxy = setup.CreateBootstrapCluster( + setup.CreateBootstrapClusterInput{ + E2EConfig: e2eConfig, + Scheme: scheme, + UseExistingCluster: useExistingCluster, + }, + ) - Byf("Creating a clusterctl local repository into %q", artifactFolder) - clusterctlConfigPath = createClusterctlLocalRepository(e2eConfig, filepath.Join(artifactFolder, "repository")) + log.Info("Initializing the bootstrap cluster") + // setup.InitBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder) + setup.InitBootstrapCluster( + setup.InitBootstrapClusterInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + E2EConfig: e2eConfig, + ClusterctlConfig: clusterctlConfigPath, + ArtifactsDirectory: artifactFolder, + }, + ) - By("Setting up the bootstrap cluster") - bootstrapClusterProvider, bootstrapClusterProxy = setupBootstrapCluster(e2eConfig, scheme, useExistingCluster) + conf := suiteConfig{ + ArtifactFolder: artifactFolder, + ConfigPath: configPath, + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + } - By("Initializing the bootstrap cluster") - initBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder) + data, err := yaml.Marshal(conf) + Expect(err).NotTo(HaveOccurred()) + return data - return []byte( - strings.Join([]string{ - artifactFolder, - configPath, - clusterctlConfigPath, - bootstrapClusterProxy.GetKubeconfigPath(), - }, ","), - ) }, func(data []byte) { // Before each ParallelNode. - parts := strings.Split(string(data), ",") - Expect(parts).To(HaveLen(4)) + conf := &suiteConfig{} + err := yaml.UnmarshalStrict(data, conf) + Expect(err).NotTo(HaveOccurred()) - artifactFolder = parts[0] - configPath = parts[1] - clusterctlConfigPath = parts[2] - kubeconfigPath := parts[3] + artifactFolder = conf.ArtifactFolder + configPath = conf.ConfigPath + clusterctlConfigPath = conf.ClusterctlConfigPath + kubeconfigPath := conf.KubeconfigPath - e2eConfig = loadE2EConfig(configPath) + e2eConfig = setup.LoadE2EConfig(configPath) bootstrapClusterProxy = framework.NewClusterProxy("bootstrap", kubeconfigPath, initScheme(), framework.WithMachineLogCollector(framework.DockerLogCollector{})) }) @@ -142,10 +163,10 @@ var _ = SynchronizedAfterSuite(func() { // After each ParallelNode. }, func() { // After all ParallelNodes. - - By("Tearing down the management cluster") + log := ginkgoextensions.Log.WithValues("config-path", configPath, "artifacts-directory", artifactFolder) + log.Info("Tearing down the management cluster") if !skipCleanup { - tearDown(bootstrapClusterProvider, bootstrapClusterProxy) + setup.TearDown(bootstrapClusterProvider, bootstrapClusterProxy) } }) @@ -154,62 +175,3 @@ func initScheme() *runtime.Scheme { framework.TryAddDefaultSchemes(sc) return sc } - -func loadE2EConfig(configPath string) *clusterctl.E2EConfig { - config := clusterctl.LoadE2EConfig(context.TODO(), clusterctl.LoadE2EConfigInput{ConfigPath: configPath}) - Expect(config).ToNot(BeNil(), "Failed to load E2E config from %s", configPath) - - // Read CNI file and set CNI_RESOURCES environmental variable - Expect(config.Variables).To(HaveKey(CNIPath), "Missing %s variable in the config", CNIPath) - clusterctl.SetCNIEnvVar(config.GetVariable(CNIPath), CNIResources) - - return config -} - -func createClusterctlLocalRepository(config *clusterctl.E2EConfig, repositoryFolder string) string { - clusterctlConfig := clusterctl.CreateRepository(context.TODO(), clusterctl.CreateRepositoryInput{ - E2EConfig: config, - RepositoryFolder: repositoryFolder, - }) - Expect(clusterctlConfig).To(BeAnExistingFile(), "The clusterctl config file does not exists in the local repository %s", repositoryFolder) - return clusterctlConfig -} - -func setupBootstrapCluster(config *clusterctl.E2EConfig, scheme *runtime.Scheme, useExistingCluster bool) (bootstrap.ClusterProvider, framework.ClusterProxy) { - var clusterProvider bootstrap.ClusterProvider - kubeconfigPath := "" - if !useExistingCluster { - clusterProvider = bootstrap.CreateKindBootstrapClusterAndLoadImages(context.TODO(), bootstrap.CreateKindBootstrapClusterAndLoadImagesInput{ - Name: config.ManagementClusterName, - RequiresDockerSock: config.HasDockerProvider(), - Images: config.Images, - }) - Expect(clusterProvider).ToNot(BeNil(), "Failed to create a bootstrap cluster") - - kubeconfigPath = clusterProvider.GetKubeconfigPath() - Expect(kubeconfigPath).To(BeAnExistingFile(), "Failed to get the kubeconfig file for the bootstrap cluster") - } - - clusterProxy := framework.NewClusterProxy("bootstrap", kubeconfigPath, scheme) - Expect(clusterProxy).ToNot(BeNil(), "Failed to get a bootstrap cluster proxy") - - return clusterProvider, clusterProxy -} - -func initBootstrapCluster(bootstrapClusterProxy framework.ClusterProxy, config *clusterctl.E2EConfig, clusterctlConfig, artifactFolder string) { - clusterctl.InitManagementClusterAndWatchControllerLogs(context.TODO(), clusterctl.InitManagementClusterAndWatchControllerLogsInput{ - ClusterProxy: bootstrapClusterProxy, - ClusterctlConfigPath: clusterctlConfig, - InfrastructureProviders: config.InfrastructureProviders(), - LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), - }, config.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) -} - -func tearDown(bootstrapClusterProvider bootstrap.ClusterProvider, bootstrapClusterProxy framework.ClusterProxy) { - if bootstrapClusterProxy != nil { - bootstrapClusterProxy.Dispose(context.TODO()) - } - if bootstrapClusterProvider != nil { - bootstrapClusterProvider.Dispose(context.TODO()) - } -} diff --git a/test/e2e/internal/setup/setup.go b/test/e2e/internal/setup/setup.go new file mode 100644 index 000000000000..3b84ee2bdda7 --- /dev/null +++ b/test/e2e/internal/setup/setup.go @@ -0,0 +1,230 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package setup + +import ( + "context" + "fmt" + "io/ioutil" + "path/filepath" + + . "github.com/onsi/gomega" + . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" + "sigs.k8s.io/cluster-api/test/framework/kubetest" + + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/bootstrap" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/util" + + corev1 "k8s.io/api/core/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" +) + +// Test suite constants for e2e config variables +const ( + CNIPath = "CNI" + CNIResources = "CNI_RESOURCES" +) + +func InitScheme() *runtime.Scheme { + sc := runtime.NewScheme() + framework.TryAddDefaultSchemes(sc) + return sc +} + +func LoadE2EConfig(configPath string) *clusterctl.E2EConfig { + config := clusterctl.LoadE2EConfig(context.TODO(), clusterctl.LoadE2EConfigInput{ConfigPath: configPath}) + Expect(config).ToNot(BeNil(), "Failed to load E2E config from %s", configPath) + + // Read CNI file and set CNI_RESOURCES environmental variable + Expect(config.Variables).To(HaveKey(CNIPath), "Missing %s variable in the config", CNIPath) + clusterctl.SetCNIEnvVar(config.GetVariable(CNIPath), CNIResources) + + return config +} + +func CreateClusterctlLocalRepository(config *clusterctl.E2EConfig, repositoryFolder string) string { + clusterctlConfig := clusterctl.CreateRepository(context.TODO(), clusterctl.CreateRepositoryInput{ + E2EConfig: config, + RepositoryFolder: repositoryFolder, + }) + Expect(clusterctlConfig).To(BeAnExistingFile(), "The clusterctl config file does not exists in the local repository %s", repositoryFolder) + return clusterctlConfig +} + +type CreateBootstrapClusterInput struct { + E2EConfig *clusterctl.E2EConfig + Scheme *runtime.Scheme + UseExistingCluster bool +} + +func CreateCIArtifactsTemplate(artifactFolder string, srcFolder string, e2eConfig *clusterctl.E2EConfig) string { + template, err := ioutil.ReadFile(srcFolder + "/data/infrastructure-docker/cluster-template.yaml") + Expect(err).NotTo(HaveOccurred()) + + platformKustomization, err := ioutil.ReadFile(srcFolder + "/data/infrastructure-docker/platform-kustomization.yaml") + Expect(err).NotTo(HaveOccurred()) + + ciTemplate, err := kubetest.GenerateCIArtifactsInjectedTemplateForDebian(kubetest.GenerateCIArtifactsInjectedTemplateForDebianInput{ + ArtifactsDirectory: artifactFolder, + SourceTemplate: template, + PlatformKustomization: platformKustomization, + }) + + Expect(err).NotTo(HaveOccurred()) + for i, p := range e2eConfig.Providers { + if p.Name != "docker" { + continue + } + e2eConfig.Providers[i].Files = append(e2eConfig.Providers[i].Files, clusterctl.Files{ + SourcePath: ciTemplate, + TargetName: "cluster-template-with-ci-artifacts.yaml", + }) + } + return ciTemplate +} + +func CreateBootstrapCluster(input CreateBootstrapClusterInput) (bootstrap.ClusterProvider, framework.ClusterProxy) { + var clusterProvider bootstrap.ClusterProvider + kubeconfigPath := "" + Expect(input.E2EConfig).ToNot(BeNil(), "E2EConfig must be provided") + Expect(input.Scheme).ToNot(BeNil(), "Scheme must be provided") + if !input.UseExistingCluster { + clusterProvider = bootstrap.CreateKindBootstrapClusterAndLoadImages(context.TODO(), bootstrap.CreateKindBootstrapClusterAndLoadImagesInput{ + Name: input.E2EConfig.ManagementClusterName, + RequiresDockerSock: input.E2EConfig.HasDockerProvider(), + Images: input.E2EConfig.Images, + }) + Expect(clusterProvider).ToNot(BeNil(), "Failed to create a bootstrap cluster") + + kubeconfigPath = clusterProvider.GetKubeconfigPath() + Expect(kubeconfigPath).To(BeAnExistingFile(), "Failed to get the kubeconfig file for the bootstrap cluster") + } + + clusterProxy := framework.NewClusterProxy("bootstrap", kubeconfigPath, input.Scheme) + Expect(clusterProxy).ToNot(BeNil(), "Failed to get a bootstrap cluster proxy") + + return clusterProvider, clusterProxy +} + +type InitBootstrapClusterInput struct { + BootstrapClusterProxy framework.ClusterProxy + E2EConfig *clusterctl.E2EConfig + ClusterctlConfig string + ArtifactsDirectory string +} + +func InitBootstrapCluster(input InitBootstrapClusterInput) { + input.ArtifactsDirectory = framework.ResolveArtifactsDirectory(input.ArtifactsDirectory) + Expect(input.E2EConfig).ToNot(BeNil(), "E2EConfig must be provided") + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "BootstrapClusterProxy must be provided") + + clusterctl.InitManagementClusterAndWatchControllerLogs(context.TODO(), clusterctl.InitManagementClusterAndWatchControllerLogsInput{ + ClusterProxy: input.BootstrapClusterProxy, + ClusterctlConfigPath: input.ClusterctlConfig, + InfrastructureProviders: input.E2EConfig.InfrastructureProviders(), + LogFolder: filepath.Join(input.ArtifactsDirectory, "clusters", input.BootstrapClusterProxy.GetName()), + }, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-controllers")...) +} + +func TearDown(bootstrapClusterProvider bootstrap.ClusterProvider, bootstrapClusterProxy framework.ClusterProxy) { + if bootstrapClusterProxy != nil { + bootstrapClusterProxy.Dispose(context.TODO()) + } + if bootstrapClusterProvider != nil { + bootstrapClusterProvider.Dispose(context.TODO()) + } +} + +type CreateSpecNamespaceInput struct { + SpecName string + ClusterProxy framework.ClusterProxy + ArtifactsDirectory string +} + +func CreateSpecNamespace(ctx context.Context, input CreateSpecNamespaceInput) (*corev1.Namespace, context.CancelFunc) { + name := fmt.Sprintf("%s-%s", input.SpecName, util.RandomString(6)) + log := Log.WithValues("spec-name", input.SpecName, "namespace", name) + log.Info("Creating a namespace for hosting the %q test spec", input.SpecName) + namespace, cancelWatches := framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ + Creator: input.ClusterProxy.GetClient(), + ClientSet: input.ClusterProxy.GetClientSet(), + Name: name, + LogFolder: filepath.Join(framework.ResolveArtifactsDirectory(input.ArtifactsDirectory), "clusters", input.ClusterProxy.GetName()), + }) + + return namespace, cancelWatches +} + +type DumpSpecResourcesAndCleanupInput struct { + SpecName string + ClusterProxy framework.ClusterProxy + ArtifactsDirectory string + Namespace *corev1.Namespace + CancelWatches context.CancelFunc + Cluster *clusterv1.Cluster + IntervalsGetter func(spec, key string) []interface{} + SkipCleanup bool +} + +func DumpSpecResourcesAndCleanup(ctx context.Context, input DumpSpecResourcesAndCleanupInput) { + Expect(input.ClusterProxy).ToNot(BeNil(), "ClusterProxy must be provided") + Expect(input.Namespace).ToNot(BeNil(), "Namespace must be provided") + Expect(input.IntervalsGetter).ToNot(BeNil(), "IntervalsGetter must be provided") + logPath := filepath.Join(framework.ResolveArtifactsDirectory(input.ArtifactsDirectory), "clusters", input.ClusterProxy.GetName(), "resources") + log := Log.WithValues( + "namespace", input.Namespace.Name, + "cluster-namespace", input.Cluster.Namespace, + "cluster-name", input.Cluster.Name, + "spec-name", input.SpecName, + "log-path", logPath, + ) + + // Dump all the logs from the workload cluster before deleting them. + input.ClusterProxy.CollectWorkloadClusterLogs(ctx, + input.Cluster.Namespace, + input.Cluster.Name, + filepath.Join(input.ArtifactsDirectory, "clusters", input.Cluster.Name, "machines")) + + log.Info("Dumping all the Cluster API resources in namespace") + // Dump all Cluster API related resources to artifacts before deleting them. + framework.DumpAllResources(ctx, framework.DumpAllResourcesInput{ + Lister: input.ClusterProxy.GetClient(), + Namespace: input.Namespace.Name, + LogPath: logPath, + }) + + if !input.SkipCleanup { + log.Info("Deleting cluster") + // While https://github.com/kubernetes-sigs/cluster-api/issues/2955 is addressed in future iterations, there is a chance + // that cluster variable is not set even if the cluster exists, so we are calling DeleteAllClustersAndWait + // instead of DeleteClusterAndWait + framework.DeleteAllClustersAndWait(ctx, framework.DeleteAllClustersAndWaitInput{ + Client: input.ClusterProxy.GetClient(), + Namespace: input.Namespace.Name, + }, input.IntervalsGetter(input.SpecName, "wait-delete-cluster")...) + + log.Info("Deleting namespace used for hosting the test spec") + framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{ + Deleter: input.ClusterProxy.GetClient(), + Name: input.Namespace.Name, + }) + } + input.CancelWatches() +} diff --git a/test/e2e/kcp_adoption.go b/test/e2e/kcp_adoption.go index 628446efafe6..fbb97450a9a8 100644 --- a/test/e2e/kcp_adoption.go +++ b/test/e2e/kcp_adoption.go @@ -33,6 +33,7 @@ import ( "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" @@ -78,7 +79,14 @@ func KCPAdoptionSpec(ctx context.Context, inputGetter func() KCPAdoptionSpecInpu Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. - namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) }) It("Should adopt up-to-date control plane Machines without modification", func() { @@ -222,6 +230,18 @@ func KCPAdoptionSpec(ctx context.Context, inputGetter func() KCPAdoptionSpecInpu AfterEach(func() { // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. - dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) }) } diff --git a/test/e2e/kcp_upgrade.go b/test/e2e/kcp_upgrade.go index ab8a69bff8a4..1f3a6aebc7a8 100644 --- a/test/e2e/kcp_upgrade.go +++ b/test/e2e/kcp_upgrade.go @@ -29,6 +29,7 @@ import ( "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" @@ -67,7 +68,14 @@ func KCPUpgradeSpec(ctx context.Context, inputGetter func() KCPUpgradeSpecInput) Expect(input.E2EConfig.Variables).To(HaveKey(CoreDNSVersionUpgradeTo)) // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. - namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) }) It("Should successfully upgrade Kubernetes, DNS, kube-proxy, and etcd in a single control plane cluster", func() { @@ -149,6 +157,18 @@ func KCPUpgradeSpec(ctx context.Context, inputGetter func() KCPUpgradeSpecInput) AfterEach(func() { // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. - dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) }) } diff --git a/test/e2e/kcp_upgrade_ci_artifacts.go b/test/e2e/kcp_upgrade_ci_artifacts.go new file mode 100644 index 000000000000..e1e7a278dd32 --- /dev/null +++ b/test/e2e/kcp_upgrade_ci_artifacts.go @@ -0,0 +1,216 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/test/framework/kubernetesversions" + "sigs.k8s.io/cluster-api/util" +) + +// KCPUpgradeCIArtifactsSpecInput is the input for KCPUpgradeSpec. +type KCPUpgradeCIArtifactsSpecInput struct { + E2EConfig *clusterctl.E2EConfig + ClusterctlConfigPath string + BootstrapClusterProxy framework.ClusterProxy + ArtifactFolder string + SkipCleanup bool +} + +// KCPUpgradeCIArtifactsSpec implements a test that verifies KCP to properly upgrade a control plane with 3 machines. +func KCPUpgradeCIArtifactsSpec(ctx context.Context, inputGetter func() KCPUpgradeCIArtifactsSpecInput) { + var ( + specName = "kcp-upgrade-ci-artifacts" + input KCPUpgradeCIArtifactsSpecInput + namespace *corev1.Namespace + cancelWatches context.CancelFunc + cluster *clusterv1.Cluster + controlPlane *controlplanev1.KubeadmControlPlane + ciVersion string + lastVersion string + ) + + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + os.Setenv("USE_CI_ARTIFACTS", "false") + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + var err error + ciVersion, err = kubernetesversions.LatestCIRelease() + Expect(err).NotTo(HaveOccurred()) + lastVersion, err = kubernetesversions.PreviousMinorRelease(ciVersion) + os.Setenv("DOCKER_NODE_VERSION", lastVersion) + Expect(err).NotTo(HaveOccurred()) + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) + }) + + It("Should successfully upgrade Kubernetes to the latest main branch version", func() { + clusterName := fmt.Sprintf("cluster-%s", util.RandomString(6)) + By("Creating a workload cluster") + cluster, controlPlane, _ = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "with-ci-artifacts", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: lastVersion, + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + + os.Setenv("USE_CI_ARTIFACTS", "true") + + By("Upgrading Kubernetes") + cluster, _, _ = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "with-ci-artifacts", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: ciVersion, + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), + }) + + By("Waiting for control plane to be up to date") + framework.WaitForControlPlaneMachinesToBeUpgraded(ctx, framework.WaitForControlPlaneMachinesToBeUpgradedInput{ + Cluster: cluster, + MachineCount: int(*controlPlane.Spec.Replicas), + KubernetesUpgradeVersion: ciVersion, + Lister: input.BootstrapClusterProxy.GetClient(), + }, + input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade")..., + ) + + By("PASSED!") + }) + + It("Should successfully upgrade Kubernetes to the latest main branch version in a HA cluster", func() { + clusterName := fmt.Sprintf("clusterha-upgrade-%s", util.RandomString(6)) + By("Creating a workload cluster") + + cluster, _, _ = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "with-ci-artifacts", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: lastVersion, + ControlPlaneMachineCount: pointer.Int64Ptr(3), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + + By("Upgrading Kubernetes") + + cluster, _, _ = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "with-ci-artifacts", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: ciVersion, + ControlPlaneMachineCount: pointer.Int64Ptr(3), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + + By("Waiting for control plane to be up to date") + framework.WaitForControlPlaneMachinesToBeUpgraded(ctx, framework.WaitForControlPlaneMachinesToBeUpgradedInput{ + Cluster: cluster, + MachineCount: int(*controlPlane.Spec.Replicas), + KubernetesUpgradeVersion: ciVersion, + Lister: input.BootstrapClusterProxy.GetClient(), + }, + "20m", + ) + + By("PASSED!") + }) + + AfterEach(func() { + // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) + }) +} diff --git a/test/e2e/kcp_upgrade_ci_artifacts_test.go b/test/e2e/kcp_upgrade_ci_artifacts_test.go new file mode 100644 index 000000000000..61f20c7a9ff5 --- /dev/null +++ b/test/e2e/kcp_upgrade_ci_artifacts_test.go @@ -0,0 +1,39 @@ +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo" +) + +var _ = Describe("When testing KCP upgrade with CI artifacts", func() { + + KCPUpgradeCIArtifactsSpec(context.TODO(), func() KCPUpgradeCIArtifactsSpecInput { + return KCPUpgradeCIArtifactsSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + } + }) + +}) diff --git a/test/e2e/md_upgrades.go b/test/e2e/md_upgrades.go index d33bcf7398f4..9e895b089ae7 100644 --- a/test/e2e/md_upgrades.go +++ b/test/e2e/md_upgrades.go @@ -28,6 +28,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" @@ -65,7 +66,14 @@ func MachineDeploymentUpgradesSpec(ctx context.Context, inputGetter func() Machi Expect(input.E2EConfig.Variables).To(HaveValidVersion(input.E2EConfig.GetVariable(KubernetesVersionUpgradeFrom))) // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. - namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) }) It("Should successfully upgrade Machines upon changes in relevant MachineDeployment fields", func() { @@ -113,6 +121,18 @@ func MachineDeploymentUpgradesSpec(ctx context.Context, inputGetter func() Machi AfterEach(func() { // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. - dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) }) } diff --git a/test/e2e/mhc_remediations.go b/test/e2e/mhc_remediations.go index 0d5b76dc72fd..c19ffc5dbb06 100644 --- a/test/e2e/mhc_remediations.go +++ b/test/e2e/mhc_remediations.go @@ -28,6 +28,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" @@ -62,7 +63,14 @@ func MachineRemediationSpec(ctx context.Context, inputGetter func() MachineRemed Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. - namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) }) It("Should successfully remediate unhealthy machines with MachineHealthCheck", func() { @@ -100,6 +108,18 @@ func MachineRemediationSpec(ctx context.Context, inputGetter func() MachineRemed AfterEach(func() { // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. - dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) }) } diff --git a/test/e2e/quick_start.go b/test/e2e/quick_start.go index 5faea5972c6c..cf8b5dc6dac9 100644 --- a/test/e2e/quick_start.go +++ b/test/e2e/quick_start.go @@ -28,6 +28,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" "sigs.k8s.io/cluster-api/util" @@ -65,7 +66,14 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput) Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. - namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) }) It("Should create a workload cluster", func() { @@ -96,6 +104,18 @@ func QuickStartSpec(ctx context.Context, inputGetter func() QuickStartSpecInput) AfterEach(func() { // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. - dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) }) } diff --git a/test/e2e/self_hosted.go b/test/e2e/self_hosted.go index 06bfc173fe62..1f12a33f6ee0 100644 --- a/test/e2e/self_hosted.go +++ b/test/e2e/self_hosted.go @@ -28,7 +28,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" - "sigs.k8s.io/cluster-api/test/e2e/internal/log" + "sigs.k8s.io/cluster-api/test/e2e/internal/setup" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/bootstrap" "sigs.k8s.io/cluster-api/test/framework/clusterctl" @@ -69,7 +69,14 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput) Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. - namespace, cancelWatches = setupSpecNamespace(context.TODO(), specName, input.BootstrapClusterProxy, input.ArtifactFolder) + namespace, cancelWatches = setup.CreateSpecNamespace( + ctx, + setup.CreateSpecNamespaceInput{ + ArtifactsDirectory: input.ArtifactFolder, + ClusterProxy: input.BootstrapClusterProxy, + SpecName: specName, + }, + ) }) It("Should pivot the bootstrap cluster to a self-hosted cluster", func() { @@ -137,7 +144,7 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput) Namespace: namespace.Name, }) - log.Logf("Waiting for the cluster infrastructure to be provisioned") + By("Waiting for the cluster infrastructure to be provisioned") selfHostedCluster = framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{ Getter: selfHostedClusterProxy.GetClient(), Namespace: selfHostedNamespace.Name, @@ -174,7 +181,7 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput) Namespace: selfHostedNamespace.Name, }) - log.Logf("Waiting for the cluster infrastructure to be provisioned") + By("Waiting for the cluster infrastructure to be provisioned") cluster = framework.DiscoveryAndWaitForCluster(ctx, framework.DiscoveryAndWaitForClusterInput{ Getter: input.BootstrapClusterProxy.GetClient(), Namespace: namespace.Name, @@ -186,6 +193,18 @@ func SelfHostedSpec(ctx context.Context, inputGetter func() SelfHostedSpecInput) } // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. - dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + setup.DumpSpecResourcesAndCleanup( + ctx, + setup.DumpSpecResourcesAndCleanupInput{ + SpecName: specName, + ClusterProxy: input.BootstrapClusterProxy, + ArtifactsDirectory: input.ArtifactFolder, + Namespace: namespace, + CancelWatches: cancelWatches, + Cluster: cluster, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + }, + ) }) } diff --git a/test/framework/alltypes_helpers.go b/test/framework/alltypes_helpers.go index b947a22b2cc6..fecae7e26c11 100644 --- a/test/framework/alltypes_helpers.go +++ b/test/framework/alltypes_helpers.go @@ -22,11 +22,14 @@ import ( "fmt" "io/ioutil" "os" + "os/exec" "path" "path/filepath" + "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -182,3 +185,16 @@ func PrettyPrint(v interface{}) string { } return string(b) } + +// CompleteCommand prints a command before running it. Acts as a helper function. +// privateArgs when true will not print arguments. +func CompleteCommand(cmd *exec.Cmd, desc string, privateArgs bool) *exec.Cmd { + cmd.Stderr = ginkgo.GinkgoWriter + cmd.Stdout = ginkgo.GinkgoWriter + if privateArgs { + Byf("%s: dir=%s, command=%s", desc, cmd.Dir, cmd) + } else { + Byf("%s: dir=%s, command=%s", desc, cmd.Dir, cmd.String()) + } + return cmd +} diff --git a/test/framework/bootstrap/kind_provider.go b/test/framework/bootstrap/kind_provider.go index 4f766b5ef334..4bbb6b21af1f 100644 --- a/test/framework/bootstrap/kind_provider.go +++ b/test/framework/bootstrap/kind_provider.go @@ -23,7 +23,7 @@ import ( . "github.com/onsi/gomega" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" kindv1 "sigs.k8s.io/kind/pkg/apis/config/v1alpha4" kind "sigs.k8s.io/kind/pkg/cluster" ) diff --git a/test/framework/bootstrap/kind_util.go b/test/framework/bootstrap/kind_util.go index 65288e5dce60..957e187d0438 100644 --- a/test/framework/bootstrap/kind_util.go +++ b/test/framework/bootstrap/kind_util.go @@ -27,7 +27,7 @@ import ( "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/exec" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" kind "sigs.k8s.io/kind/pkg/cluster" kindnodes "sigs.k8s.io/kind/pkg/cluster/nodes" kindnodesutils "sigs.k8s.io/kind/pkg/cluster/nodeutils" diff --git a/test/framework/cluster_helpers.go b/test/framework/cluster_helpers.go index da1514f9938f..3f9cc178152f 100644 --- a/test/framework/cluster_helpers.go +++ b/test/framework/cluster_helpers.go @@ -26,7 +26,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" "sigs.k8s.io/cluster-api/test/framework/options" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" @@ -41,13 +41,13 @@ type CreateClusterInput struct { // CreateCluster will create the Cluster and InfraCluster objects. func CreateCluster(ctx context.Context, input CreateClusterInput, intervals ...interface{}) { - By("creating an InfrastructureCluster resource") + By("Creating an InfrastructureCluster resource") Expect(input.Creator.Create(ctx, input.InfraCluster)).To(Succeed()) // This call happens in an eventually because of a race condition with the // webhook server. If the latter isn't fully online then this call will // fail. - By("creating a Cluster resource linked to the InfrastructureCluster resource") + By("Creating a Cluster resource linked to the InfrastructureCluster resource") Eventually(func() error { if err := input.Creator.Create(ctx, input.Cluster); err != nil { log.Logf("Failed to create a cluster: %+v", err) @@ -122,7 +122,7 @@ type WaitForClusterToProvisionInput struct { // WaitForClusterToProvision will wait for a cluster to have a phase status of provisioned. func WaitForClusterToProvision(ctx context.Context, input WaitForClusterToProvisionInput, intervals ...interface{}) { - By("waiting for cluster to enter the provisioned phase") + By("Waiting for cluster to enter the provisioned phase") Eventually(func() (string, error) { cluster := &clusterv1.Cluster{} key := client.ObjectKey{ @@ -162,7 +162,7 @@ func WaitForClusterDeleted(ctx context.Context, input WaitForClusterDeletedInput if options.SkipResourceCleanup { return } - By(fmt.Sprintf("waiting for cluster %s to be deleted", input.Cluster.GetName())) + By(fmt.Sprintf("Waiting for cluster %s to be deleted", input.Cluster.GetName())) Eventually(func() bool { cluster := &clusterv1.Cluster{} key := client.ObjectKey{ diff --git a/test/framework/cluster_proxy.go b/test/framework/cluster_proxy.go index 48d2ce145b8d..ede8be30e37b 100644 --- a/test/framework/cluster_proxy.go +++ b/test/framework/cluster_proxy.go @@ -36,7 +36,7 @@ import ( "k8s.io/client-go/tools/clientcmd/api" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/test/framework/exec" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/test/framework/clusterctl/client.go b/test/framework/clusterctl/client.go index 2e5cd0741355..3b93b3f010c7 100644 --- a/test/framework/clusterctl/client.go +++ b/test/framework/clusterctl/client.go @@ -28,7 +28,7 @@ import ( clusterctlclient "sigs.k8s.io/cluster-api/cmd/clusterctl/client" clusterctllog "sigs.k8s.io/cluster-api/cmd/clusterctl/log" "sigs.k8s.io/cluster-api/test/framework/clusterctl/logger" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" ) // Provide E2E friendly wrappers for the clusterctl client library. diff --git a/test/framework/clusterctl/clusterctl_helpers.go b/test/framework/clusterctl/clusterctl_helpers.go index c33cbc9912f3..c56b5c3399c9 100644 --- a/test/framework/clusterctl/clusterctl_helpers.go +++ b/test/framework/clusterctl/clusterctl_helpers.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/cluster-api/cmd/clusterctl/client/config" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3" "sigs.k8s.io/cluster-api/test/framework" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" ) // InitManagementClusterAndWatchControllerLogsInput is the input type for InitManagementClusterAndWatchControllerLogs. diff --git a/test/framework/clusterctl/e2e_config.go b/test/framework/clusterctl/e2e_config.go index 6efd49a56c19..44e1a1bafcff 100644 --- a/test/framework/clusterctl/e2e_config.go +++ b/test/framework/clusterctl/e2e_config.go @@ -385,9 +385,18 @@ func (c *E2EConfig) GetIntervals(spec, key string) []interface{} { // GetVariable returns a variable from the e2e config file. func (c *E2EConfig) GetVariable(varName string) string { - version, ok := c.Variables[varName] + variable, ok := c.Variables[varName] Expect(ok).NotTo(BeFalse()) - return version + return variable +} + +// GetStringVariableWithDefault returns a variable from the e2e config file, but can return a default string. +func (c *E2EConfig) GetStringVariableWithDefault(varName string, defaultValue string) string { + variable, ok := c.Variables[varName] + if !ok { + return defaultValue + } + return variable } // GetVariable returns an Int64Ptr variable from the e2e config file. diff --git a/test/framework/config.go b/test/framework/config.go index 99ee1f7994a6..2d2ef7c5fcdb 100644 --- a/test/framework/config.go +++ b/test/framework/config.go @@ -22,6 +22,8 @@ import ( "io" "io/ioutil" "net/http" + "os" + "path" "regexp" "github.com/pkg/errors" @@ -363,3 +365,15 @@ func (g componentSourceGenerator) GetName() string { func (g componentSourceGenerator) Manifests(ctx context.Context) ([]byte, error) { return YAMLForComponentSource(ctx, g.ComponentSource) } + +// DumpToFile will dump the running e2e config to a file +func (c *Config) DumpToFile(filename string) error { + yaml, err := yaml.Marshal(c) + if err != nil { + return err + } + if err := os.MkdirAll(path.Dir(filename), 0o700); err != nil { + return err + } + return ioutil.WriteFile(filename, yaml, 0o600) +} diff --git a/test/framework/control_plane.go b/test/framework/control_plane.go index 3607e123b533..941c426891ec 100644 --- a/test/framework/control_plane.go +++ b/test/framework/control_plane.go @@ -34,7 +34,7 @@ type WaitForControlPlaneToBeUpToDateInput struct { // WaitForControlPlaneToBeUpToDate will wait for a control plane to be fully up-to-date. func WaitForControlPlaneToBeUpToDate(ctx context.Context, input WaitForControlPlaneToBeUpToDateInput, intervals ...interface{}) { - By("waiting for the control plane to be ready") + By("Waiting for the control plane to be ready") Eventually(func() (int32, error) { controlplane := &controlplanev1.KubeadmControlPlane{} key := client.ObjectKey{ diff --git a/test/framework/controlplane_helpers.go b/test/framework/controlplane_helpers.go index 521ba96452b2..1c23e5dbc479 100644 --- a/test/framework/controlplane_helpers.go +++ b/test/framework/controlplane_helpers.go @@ -27,7 +27,7 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -41,10 +41,10 @@ type CreateKubeadmControlPlaneInput struct { // CreateKubeadmControlPlane creates the control plane object and necessary dependencies. func CreateKubeadmControlPlane(ctx context.Context, input CreateKubeadmControlPlaneInput, intervals ...interface{}) { - By("creating the machine template") + By("Creating the machine template") Expect(input.Creator.Create(ctx, input.MachineTemplate)).To(Succeed()) - By("creating a KubeadmControlPlane") + By("Creating a KubeadmControlPlane") Eventually(func() error { err := input.Creator.Create(ctx, input.ControlPlane) if err != nil { @@ -120,7 +120,7 @@ func WaitForOneKubeadmControlPlaneMachineToExist(ctx context.Context, input Wait Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Getter can't be nil when calling WaitForOneKubeadmControlPlaneMachineToExist") Expect(input.ControlPlane).ToNot(BeNil(), "Invalid argument. input.ControlPlane can't be nil when calling WaitForOneKubeadmControlPlaneMachineToExist") - By("waiting for one control plane node to exist") + By("Waiting for one control plane node to exist") inClustersNamespaceListOption := client.InNamespace(input.Cluster.Namespace) // ControlPlane labels matchClusterListOption := client.MatchingLabels{ @@ -152,7 +152,7 @@ type WaitForControlPlaneToBeReadyInput struct { // WaitForControlPlaneToBeReady will wait for a control plane to be ready. func WaitForControlPlaneToBeReady(ctx context.Context, input WaitForControlPlaneToBeReadyInput, intervals ...interface{}) { - By("waiting for the control plane to be ready") + By("Waiting for the control plane to be ready") Eventually(func() (bool, error) { controlplane := &controlplanev1.KubeadmControlPlane{} key := client.ObjectKey{ diff --git a/test/framework/convenience.go b/test/framework/convenience.go index 3819601261e4..2032b34c9602 100644 --- a/test/framework/convenience.go +++ b/test/framework/convenience.go @@ -17,8 +17,16 @@ limitations under the License. package framework import ( + "fmt" + "os" + "os/exec" + "path" + "path/filepath" "reflect" + "strings" + "github.com/onsi/ginkgo/config" + "github.com/onsi/ginkgo/reporters" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -77,3 +85,25 @@ func TypeToKind(i interface{}) string { func ObjectToKind(i runtime.Object) string { return reflect.ValueOf(i).Elem().Type().Name() } + +func ResolveArtifactsDirectory(input string) string { + if input != "" { + return input + } + if dir, ok := os.LookupEnv("ARTIFACTS"); ok { + return dir + } + + findRootCmd := exec.Command("git", "rev-parse", "--show-toplevel") + out, err := findRootCmd.Output() + if err != nil { + return "_artifacts" + } + rootDir := strings.TrimSpace(string(out)) + return path.Join(rootDir, "_artifacts") +} + +func CreateJUnitReporterForProw(artifactsDirectory string) *reporters.JUnitReporter { + junitPath := filepath.Join(artifactsDirectory, fmt.Sprintf("junit.e2e_suite.%d.xml", config.GinkgoConfig.ParallelNode)) + return reporters.NewJUnitReporter(junitPath) +} diff --git a/test/framework/daemonset_helpers.go b/test/framework/daemonset_helpers.go index 7466165e8f16..364b08a270fb 100644 --- a/test/framework/daemonset_helpers.go +++ b/test/framework/daemonset_helpers.go @@ -35,7 +35,7 @@ type WaitForKubeProxyUpgradeInput struct { // WaitForKubeProxyUpgrade waits until kube-proxy version matches with the kubernetes version. This is called during KCP upgrade. func WaitForKubeProxyUpgrade(ctx context.Context, input WaitForKubeProxyUpgradeInput, intervals ...interface{}) { - By("ensuring kube-proxy has the correct image") + By("Ensuring kube-proxy has the correct image") Eventually(func() (bool, error) { ds := &appsv1.DaemonSet{} diff --git a/test/framework/deployment_helpers.go b/test/framework/deployment_helpers.go index 7e9b9bbc36ea..348d2e78f8cc 100644 --- a/test/framework/deployment_helpers.go +++ b/test/framework/deployment_helpers.go @@ -35,7 +35,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -49,7 +49,7 @@ type WaitForDeploymentsAvailableInput struct { // all the desired replicas are in place. // This can be used to check if Cluster API controllers installed in the management cluster are working. func WaitForDeploymentsAvailable(ctx context.Context, input WaitForDeploymentsAvailableInput, intervals ...interface{}) { - By(fmt.Sprintf("waiting for deployment %s/%s to be available", input.Deployment.GetNamespace(), input.Deployment.GetName())) + By(fmt.Sprintf("Waiting for deployment %s/%s to be available", input.Deployment.GetNamespace(), input.Deployment.GetName())) deployment := &appsv1.Deployment{} Eventually(func() bool { key := client.ObjectKey{ @@ -233,7 +233,7 @@ type WaitForDNSUpgradeInput struct { // WaitForDNSUpgrade waits until CoreDNS version matches with the CoreDNS upgrade version. This is called during KCP upgrade. func WaitForDNSUpgrade(ctx context.Context, input WaitForDNSUpgradeInput, intervals ...interface{}) { - By("ensuring CoreDNS has the correct image") + By("Ensuring CoreDNS has the correct image") Eventually(func() (bool, error) { d := &appsv1.Deployment{} diff --git a/test/framework/deprecated.go b/test/framework/deprecated.go index 6ab9fd8f55ad..d82103ea4d5a 100644 --- a/test/framework/deprecated.go +++ b/test/framework/deprecated.go @@ -33,7 +33,7 @@ import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/client-go/kubernetes" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" "sigs.k8s.io/cluster-api/test/framework/management/kind" "sigs.k8s.io/cluster-api/test/framework/options" diff --git a/test/framework/internal/log/log.go b/test/framework/ginkgoextensions/output.go similarity index 65% rename from test/framework/internal/log/log.go rename to test/framework/ginkgoextensions/output.go index c672fcb413d6..88e4e4c16105 100644 --- a/test/framework/internal/log/log.go +++ b/test/framework/ginkgoextensions/output.go @@ -14,14 +14,24 @@ See the License for the specific language governing permissions and limitations under the License. */ -package log +package ginkgoextensions import ( "fmt" - . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo" + "k8s.io/klog" + "sigs.k8s.io/cluster-api/test/framework/log" ) -func Logf(format string, a ...interface{}) { - fmt.Fprintf(GinkgoWriter, "INFO: "+format+"\n", a...) +var Log log.Logger + +func init() { + klog.InitFlags(nil) + Log = log.Logger{} + klog.SetOutput(ginkgo.GinkgoWriter) +} + +func Byf(format string, a ...interface{}) { + ginkgo.By(fmt.Sprintf(format, a...)) } diff --git a/test/framework/kubernetesversions/versions.go b/test/framework/kubernetesversions/versions.go new file mode 100644 index 000000000000..231a329be780 --- /dev/null +++ b/test/framework/kubernetesversions/versions.go @@ -0,0 +1,78 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetesversions + +import ( + "fmt" + "io/ioutil" + "net/http" + "strings" + + "github.com/blang/semver" +) + +const ( + ciVersionURL = "https://dl.k8s.io/ci/latest.txt" + stableVersionURL = "https://storage.googleapis.com/kubernetes-release/release/stable-%d.%d.txt" + tagPrefix = "v" +) + +// LatestCIRelease fetches the latest main branch Kubernetes version +func LatestCIRelease() (string, error) { + resp, err := http.Get(ciVersionURL) + if err != nil { + return "", err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return strings.TrimSpace(string(b)), nil +} + +// LatestPatchRelease returns the latest patch release matching +func LatestPatchRelease(searchVersion string) (string, error) { + searchSemVer, err := semver.Make(strings.TrimPrefix(searchVersion, tagPrefix)) + if err != nil { + return "", err + } + resp, err := http.Get(fmt.Sprintf(stableVersionURL, searchSemVer.Major, searchSemVer.Minor)) + if err != nil { + return "", err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return strings.TrimSpace(string(b)), nil +} + +// PreviousMinorRelease returns the latest patch release for the previous version +// of Kubernetes, e.g. v1.19.1 returns v1.18.8 as of Sep 2020. +func PreviousMinorRelease(searchVersion string) (string, error) { + semVer, err := semver.Make(strings.TrimPrefix(searchVersion, tagPrefix)) + if err != nil { + return "", err + } + semVer.Minor-- + + return LatestPatchRelease(semVer.String()) +} diff --git a/test/e2e/internal/log/log.go b/test/framework/kubetest/bindata.go similarity index 68% rename from test/e2e/internal/log/log.go rename to test/framework/kubetest/bindata.go index c672fcb413d6..3702181f05a9 100644 --- a/test/e2e/internal/log/log.go +++ b/test/framework/kubetest/bindata.go @@ -14,14 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -package log +package kubetest -import ( - "fmt" - - . "github.com/onsi/ginkgo" -) - -func Logf(format string, a ...interface{}) { - fmt.Fprintf(GinkgoWriter, "INFO: "+format+"\n", a...) -} +//go:generate sh -c "go-bindata -nometadata -pkg kubetest -o zz_generated.bindata.go.tmp data && cat ../../../hack/boilerplate/boilerplate.generatego.txt zz_generated.bindata.go.tmp > zz_generated.bindata.go && rm zz_generated.bindata.go.tmp" diff --git a/test/framework/kubetest/data/debian_injection_script.envsubst.sh b/test/framework/kubetest/data/debian_injection_script.envsubst.sh new file mode 100644 index 000000000000..6763815822ea --- /dev/null +++ b/test/framework/kubetest/data/debian_injection_script.envsubst.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Please note that this file needs to be escaped for envsubst to function + +# shellcheck disable=SC1083,SC2034,SC2066,SC2193 + +set -o nounset +set -o pipefail +set -o errexit + +[[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + +USE_CI_ARTIFACTS=${USE_CI_ARTIFACTS:=false} + +if [ ! "${USE_CI_ARTIFACTS}" = true ]; then + echo "No CI Artifacts installation, exiting" + exit 0 +fi + +GSUTIL=gsutil + +if ! command -v $${GSUTIL} >/dev/null; then + apt-get update + apt-get install -y apt-transport-https ca-certificates gnupg curl + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | $${SUDO} tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | $${SUDO} apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + apt-get update + apt-get install -y google-cloud-sdk +fi + +$${GSUTIL} version + +# This test installs release packages or binaries that are a result of the CI and release builds. +# It runs '... --version' commands to verify that the binaries are correctly installed +# and finally uninstalls the packages. +# For the release packages it tests all versions in the support skew. +LINE_SEPARATOR="*************************************************" +echo "$${LINE_SEPARATOR}" + +## Clusterctl set variables +## +# $${KUBERNETES_VERSION} will be replaced by clusterctl +KUBERNETES_VERSION=${KUBERNETES_VERSION} +## +## End clusterctl set variables + +if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="gs://kubernetes-release/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' >/etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + DEBIAN_FRONTEND=noninteractive apt-get install -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + done + else + CI_URL="gs://kubernetes-release-dev/ci/$${KUBERNETES_VERSION}-bazel/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + $${GSUTIL} cp "$${CI_URL}/$${CI_PACKAGE}" "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${GSUTIL} cp "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "k8s.gcr.io/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/kubernetes-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done +fi +echo "* checking binary versions" +echo "ctr version: " "$(ctr version)" +echo "kubeadm version: " "$(kubeadm version -o=short)" +echo "kubectl version: " "$(kubectl version --client=true --short=true)" +echo "kubelet version: " "$(kubelet --version)" +echo "$${LINE_SEPARATOR}" diff --git a/test/framework/kubetest/data/kustomization.yaml b/test/framework/kubetest/data/kustomization.yaml new file mode 100644 index 000000000000..e5db20f339ab --- /dev/null +++ b/test/framework/kubetest/data/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: + - ci-artifacts-source-template.yaml +patchesStrategicMerge: + - kustomizeversions.yaml + - platform-kustomization.yaml diff --git a/test/framework/kubetest/run.go b/test/framework/kubetest/run.go new file mode 100644 index 000000000000..ed5a7e86e91f --- /dev/null +++ b/test/framework/kubetest/run.go @@ -0,0 +1,211 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubetest + +import ( + "io/ioutil" + "os" + "os/exec" + "os/user" + "path" + "runtime" + "strconv" + "strings" + + "github.com/pkg/errors" + corev1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/cluster-api/test/framework" +) + +const ( + standardImage = "us.gcr.io/k8s-artifacts-prod/conformance" + ciArtifactImage = "gcr.io/kubernetes-ci-images/conformance" +) + +const ( + DefaultGinkgoNodes = 1 + DefaultGinkoSlowSpecThreshold = 120 +) + +type RunInput struct { + // ClusterProxy is a clusterctl test framework proxy for the workload cluster + // for which to run kubetest against + ClusterProxy framework.ClusterProxy + // NumberOfNodes is the number of cluster nodes that exist for kubetest + // to be aware of + NumberOfNodes int + // UseCIArtifacts will fetch the latest build from the main branch of kubernetes/kubernetes + UseCIArtifacts bool + // ArtifactsDirectory is where conformance suite output will go + ArtifactsDirectory string + // Path to the kubetest e2e config file + ConfigFilePath string + // GinkgoNodes is the number of Ginkgo nodes to use + GinkgoNodes int + // GinkgoSlowSpecThreshold is time in s before spec is marked as slow + GinkgoSlowSpecThreshold int + // KubernetesVersion is the version of Kubernetes to test + KubernetesVersion string + // ConformanceImage is an optional field to specify an exact conformance image + ConformanceImage string +} + +// Run executes kube-test given an artifact directory, and sets settings +// required for kubetest to work with Cluster API. JUnit files are +// also gathered for inclusion in Prow. +func Run(input RunInput) error { + if input.GinkgoNodes == 0 { + input.GinkgoNodes = DefaultGinkgoNodes + } + if input.GinkgoSlowSpecThreshold == 0 { + input.GinkgoSlowSpecThreshold = 120 + } + if input.NumberOfNodes == 0 { + numNodes, err := countClusterNodes(input.ClusterProxy) + if err != nil { + return err + } + input.NumberOfNodes = numNodes + } + input.ArtifactsDirectory = framework.ResolveArtifactsDirectory(input.ArtifactsDirectory) + reportDir := path.Join(input.ArtifactsDirectory, "kubetest") + outputDir := path.Join(reportDir, "e2e-output") + kubetestConfigDir := path.Join(reportDir, "config") + if err := os.MkdirAll(outputDir, 0o750); err != nil { + return err + } + if err := os.MkdirAll(kubetestConfigDir, 0o750); err != nil { + return err + } + ginkgoVars := map[string]string{ + "nodes": strconv.Itoa(input.GinkgoNodes), + "slowSpecThreshold": strconv.Itoa(input.GinkgoSlowSpecThreshold), + } + + // Copy configuration files for kubetest into the artifacts directory + // to avoid issues with volume mounts on MacOS + tmpConfigFilePath := path.Join(kubetestConfigDir, "viper-config.yaml") + if err := copy(input.ConfigFilePath, tmpConfigFilePath); err != nil { + return err + } + tmpKubeConfigPath, err := dockeriseKubeconfig(kubetestConfigDir, input.ClusterProxy.GetKubeconfigPath()) + if err != nil { + return err + } + + e2eVars := map[string]string{ + "kubeconfig": "/tmp/kubeconfig", + "provider": "skeleton", + "report-dir": "/output", + "e2e-output-dir": "/output/e2e-output", + "dump-logs-on-failure": "false", + "report-prefix": "kubetest.", + "num-nodes": strconv.FormatInt(int64(input.NumberOfNodes), 10), + "viper-config": "/tmp/viper-config.yaml", + } + ginkgoArgs := buildArgs(ginkgoVars, "-") + e2eArgs := buildArgs(e2eVars, "--") + if input.ConformanceImage == "" { + input.ConformanceImage = versionToConformanceImage(input.KubernetesVersion, input.UseCIArtifacts) + } + kubeConfigVolumeMount := volumeArg(tmpKubeConfigPath, "/tmp/kubeconfig") + outputVolumeMount := volumeArg(reportDir, "/output") + viperVolumeMount := volumeArg(tmpConfigFilePath, "/tmp/viper-config.yaml") + user, err := user.Current() + if err != nil { + return errors.Wrap(err, "unable to determine current user") + } + userArg := user.Uid + ":" + user.Gid + e2eCmd := exec.Command("docker", "run", "--user", userArg, kubeConfigVolumeMount, outputVolumeMount, viperVolumeMount, "-t", input.ConformanceImage) + e2eCmd.Args = append(e2eCmd.Args, "/usr/local/bin/ginkgo") + e2eCmd.Args = append(e2eCmd.Args, ginkgoArgs...) + e2eCmd.Args = append(e2eCmd.Args, "/usr/local/bin/e2e.test") + e2eCmd.Args = append(e2eCmd.Args, "--") + e2eCmd.Args = append(e2eCmd.Args, e2eArgs...) + e2eCmd = framework.CompleteCommand(e2eCmd, "Running e2e test", false) + if err := e2eCmd.Run(); err != nil { + return errors.Wrap(err, "Unable to run conformance tests") + } + if err := framework.GatherJUnitReports(reportDir, input.ArtifactsDirectory); err != nil { + return err + } + return nil +} + +func dockeriseKubeconfig(kubetestConfigDir string, kubeConfigPath string) (string, error) { + kubeConfig, err := clientcmd.LoadFromFile(kubeConfigPath) + if err != nil { + return "", err + } + newPath := path.Join(kubetestConfigDir, "kubeconfig") + + // On CAPD, if not running on Linux, we need to use Docker's proxy to connect back to the host + // to the CAPD cluster. Moby on Linux doesn't use the host.docker.internal DNS name. + if runtime.GOOS != "linux" { + for i := range kubeConfig.Clusters { + kubeConfig.Clusters[i].Server = strings.ReplaceAll(kubeConfig.Clusters[i].Server, "127.0.0.1", "host.docker.internal") + } + } + if err := clientcmd.WriteToFile(*kubeConfig, newPath); err != nil { + return "", err + } + return newPath, nil +} + +func countClusterNodes(proxy framework.ClusterProxy) (int, error) { + nodeList, err := proxy.GetClientSet().CoreV1().Nodes().List(corev1.ListOptions{}) + if err != nil { + return 0, errors.Wrap(err, "Unable to count nodes") + } + return len(nodeList.Items), nil +} + +func isSELinuxEnforcing() bool { + dat, err := ioutil.ReadFile("/sys/fs/selinux/enforce") + if err != nil { + return false + } + return string(dat) == "1" +} + +func volumeArg(src, dest string) string { + volumeArg := "-v" + src + ":" + dest + if isSELinuxEnforcing() { + return volumeArg + ":z" + } + return volumeArg +} + +func versionToConformanceImage(kubernetesVersion string, usingCIArtifacts bool) string { + k8sVersion := strings.ReplaceAll(kubernetesVersion, "+", "_") + if usingCIArtifacts { + return ciArtifactImage + ":" + k8sVersion + } + return standardImage + ":" + k8sVersion +} + +// buildArgs converts a string map to the format --key=value +func buildArgs(kv map[string]string, flagMarker string) []string { + args := make([]string, len(kv)) + i := 0 + for k, v := range kv { + args[i] = flagMarker + k + "=" + v + i++ + } + return args +} diff --git a/test/framework/kubetest/setup.go b/test/framework/kubetest/setup.go new file mode 100644 index 000000000000..365e87669894 --- /dev/null +++ b/test/framework/kubetest/setup.go @@ -0,0 +1,63 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubetest + +import ( + "io" + "io/ioutil" + "net/http" + "os" + "path" + "strings" +) + +const ( + ciVersionURL = "https://dl.k8s.io/ci/latest.txt" +) + +// FetchKubernetesCIVersion fetches the latest main branch Kubernetes version +func FetchKubernetesCIVersion() (string, error) { + resp, err := http.Get(ciVersionURL) + if err != nil { + return "", err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + return strings.TrimSpace(string(b)), nil +} + +func copy(src, dest string) error { + err := os.MkdirAll(path.Dir(dest), 0o750) + if err != nil { + return err + } + srcFile, err := os.Open(src) + if err != nil { + return err + } + destFile, err := os.Create(dest) + if err != nil { + return err + } + if _, err := io.Copy(destFile, srcFile); err != nil { + return err + } + return nil +} diff --git a/test/framework/kubetest/template.go b/test/framework/kubetest/template.go new file mode 100644 index 000000000000..116d98cb7515 --- /dev/null +++ b/test/framework/kubetest/template.go @@ -0,0 +1,177 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubetest + +import ( + "errors" + "io/ioutil" + "os" + "os/exec" + "path" + + "sigs.k8s.io/yaml" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3" + kcpv1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3" + "sigs.k8s.io/cluster-api/test/framework" +) + +type GenerateCIArtifactsInjectedTemplateForDebianInput struct { + ArtifactsDirectory string + SourceTemplate []byte + PlatformKustomization []byte + KubeadmConfigTemplateName string + KubeadmControlPlaneName string +} + +func GenerateCIArtifactsInjectedTemplateForDebian(input GenerateCIArtifactsInjectedTemplateForDebianInput) (string, error) { + if input.SourceTemplate == nil { + return "", errors.New("SourceTemplate must be provided") + } + input.ArtifactsDirectory = framework.ResolveArtifactsDirectory(input.ArtifactsDirectory) + if input.KubeadmConfigTemplateName == "" { + input.KubeadmConfigTemplateName = "${ CLUSTER_NAME }-md-0" + } + if input.KubeadmControlPlaneName == "" { + input.KubeadmControlPlaneName = "${ CLUSTER_NAME }-control-plane" + } + templateDir := path.Join(input.ArtifactsDirectory, "templates") + overlayDir := path.Join(input.ArtifactsDirectory, "overlay") + + if err := os.MkdirAll(templateDir, 0o750); err != nil { + return "", err + } + if err := os.MkdirAll(overlayDir, 0o750); err != nil { + return "", err + } + + kustomizedTemplate := path.Join(templateDir, "cluster-template-conformance-ci-artifacts.yaml") + + kustomization, err := dataKustomizationYamlBytes() + if err != nil { + return "", err + } + + if err := ioutil.WriteFile(path.Join(overlayDir, "kustomization.yaml"), kustomization, 0o600); err != nil { + return "", err + } + + kustomizeVersions, err := generateKustomizeVersionsYaml(input.KubeadmControlPlaneName, input.KubeadmConfigTemplateName) + if err != nil { + return "", err + } + + if err := ioutil.WriteFile(path.Join(overlayDir, "kustomizeversions.yaml"), kustomizeVersions, 0o600); err != nil { + return "", err + } + if err := ioutil.WriteFile(path.Join(overlayDir, "ci-artifacts-source-template.yaml"), input.SourceTemplate, 0o600); err != nil { + return "", err + } + if err := ioutil.WriteFile(path.Join(overlayDir, "platform-kustomization.yaml"), input.PlatformKustomization, 0o600); err != nil { + return "", err + } + cmd := exec.Command("kustomize", "build", overlayDir) + data, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + if err := ioutil.WriteFile(kustomizedTemplate, data, 0o600); err != nil { + return "", err + } + return kustomizedTemplate, nil +} + +func generateKustomizeVersionsYaml(kcpName, kubeadmName string) ([]byte, error) { + kcp, err := generateKubeadmControlPlane(kcpName) + if err != nil { + return nil, err + } + kubeadm, err := generateKubeadmConfigTemplate(kubeadmName) + if err != nil { + return nil, err + } + kcpYaml, err := yaml.Marshal(kcp) + if err != nil { + return nil, err + } + kubeadmYaml, err := yaml.Marshal(kubeadm) + if err != nil { + return nil, err + } + fileStr := string(kcpYaml) + "\n---\n" + string(kubeadmYaml) + return []byte(fileStr), nil +} + +func generateKubeadmConfigTemplate(name string) (*cabpkv1.KubeadmConfigTemplate, error) { + kubeadmSpec, err := generateKubeadmConfigSpec() + if err != nil { + return nil, err + } + return &cabpkv1.KubeadmConfigTemplate{ + TypeMeta: metav1.TypeMeta{ + Kind: "KubeadmConfigTemplate", + APIVersion: cabpkv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: cabpkv1.KubeadmConfigTemplateSpec{ + Template: cabpkv1.KubeadmConfigTemplateResource{ + Spec: *kubeadmSpec, + }, + }, + }, nil +} + +func generateKubeadmControlPlane(name string) (*kcpv1.KubeadmControlPlane, error) { + kubeadmSpec, err := generateKubeadmConfigSpec() + if err != nil { + return nil, err + } + return &kcpv1.KubeadmControlPlane{ + TypeMeta: metav1.TypeMeta{ + Kind: "KubeadmControlPlane", + APIVersion: kcpv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: kcpv1.KubeadmControlPlaneSpec{ + KubeadmConfigSpec: *kubeadmSpec, + Version: "${ KUBERNETES_VERSION }", + }, + }, nil +} + +func generateKubeadmConfigSpec() (*cabpkv1.KubeadmConfigSpec, error) { + data, err := dataDebian_injection_scriptEnvsubstShBytes() + if err != nil { + return nil, err + } + return &cabpkv1.KubeadmConfigSpec{ + Files: []cabpkv1.File{ + { + Path: "/usr/local/bin/ci-artifacts.sh", + Content: string(data), + Owner: "root:root", + Permissions: "0750", + }, + }, + PreKubeadmCommands: []string{"/usr/local/bin/ci-artifacts.sh"}, + }, nil +} diff --git a/test/framework/kubetest/zz_generated.bindata.go b/test/framework/kubetest/zz_generated.bindata.go new file mode 100644 index 000000000000..b1115bc9f385 --- /dev/null +++ b/test/framework/kubetest/zz_generated.bindata.go @@ -0,0 +1,285 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated for package kubetest by go-bindata DO NOT EDIT. (@generated) +// sources: +// data/debian_injection_script.envsubst.sh +// data/kustomization.yaml +package kubetest + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" +) + +func bindataRead(data []byte, name string) ([]byte, error) { + gz, err := gzip.NewReader(bytes.NewBuffer(data)) + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + + var buf bytes.Buffer + _, err = io.Copy(&buf, gz) + clErr := gz.Close() + + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + if clErr != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +type asset struct { + bytes []byte + info os.FileInfo +} + +type bindataFileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time +} + +// Name return file name +func (fi bindataFileInfo) Name() string { + return fi.name +} + +// Size return file size +func (fi bindataFileInfo) Size() int64 { + return fi.size +} + +// Mode return file mode +func (fi bindataFileInfo) Mode() os.FileMode { + return fi.mode +} + +// Mode return file modify time +func (fi bindataFileInfo) ModTime() time.Time { + return fi.modTime +} + +// IsDir return file whether a directory +func (fi bindataFileInfo) IsDir() bool { + return fi.mode&os.ModeDir != 0 +} + +// Sys return file is sys mode +func (fi bindataFileInfo) Sys() interface{} { + return nil +} + +var _dataDebian_injection_scriptEnvsubstSh = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xbc\x57\x6d\x6f\xdb\x38\x12\xfe\xae\x5f\x31\x95\x83\x4d\xb3\x0d\xa5\x34\x5d\x14\xdd\x14\x5e\x9c\xeb\xa8\x3d\xa1\x81\x5d\xd8\xce\xbe\x20\xcd\x19\x34\x35\x96\x09\xd3\xa4\x8e\xa4\x9c\xf8\x1a\xdf\x6f\x3f\x90\x92\x1c\x3b\x71\xbb\xe9\x1e\xb0\xf9\xe2\x88\xe2\xbc\xf0\x19\x3e\xcf\x8c\x5a\xcf\xe2\x09\x97\xf1\x84\x9a\x59\x10\xb4\xa0\xab\x8a\x95\xe6\xf9\xcc\xc2\xe9\xc9\xe9\x09\x8c\x66\x08\x1f\xcb\x09\x6a\x89\x16\x0d\x74\x4a\x3b\x53\xda\x44\x41\x2b\x68\xc1\x05\x67\x28\x0d\x66\x50\xca\x0c\x35\xd8\x19\x42\xa7\xa0\x6c\x86\xcd\x9b\x63\xf8\x15\xb5\xe1\x4a\xc2\x69\x74\x02\xcf\xdd\x86\xb0\x7e\x15\x1e\xbd\x0d\x5a\xb0\x52\x25\x2c\xe8\x0a\xa4\xb2\x50\x1a\x04\x3b\xe3\x06\xa6\x5c\x20\xe0\x2d\xc3\xc2\x02\x97\xc0\xd4\xa2\x10\x9c\x4a\x86\x70\xc3\xed\xcc\x87\xa9\x9d\x44\x41\x0b\xfe\xa8\x5d\xa8\x89\xa5\x5c\x02\x05\xa6\x8a\x15\xa8\xe9\xf6\x3e\xa0\xd6\x27\xec\xfe\x66\xd6\x16\x67\x71\x7c\x73\x73\x13\x51\x9f\x6c\xa4\x74\x1e\x8b\x6a\xa3\x89\x2f\xd2\x6e\xd2\x1b\x26\xe4\x34\x3a\xf1\x26\x97\x52\xa0\x31\xa0\xf1\xdf\x25\xd7\x98\xc1\x64\x05\xb4\x28\x04\x67\x74\x22\x10\x04\xbd\x01\xa5\x81\xe6\x1a\x31\x03\xab\x5c\xbe\x37\x9a\x5b\x2e\xf3\x63\x30\x6a\x6a\x6f\xa8\xc6\xa0\x05\x19\x37\x56\xf3\x49\x69\x77\xc0\x6a\xb2\xe3\x66\x67\x83\x92\x40\x25\x84\x9d\x21\xa4\xc3\x10\xde\x75\x86\xe9\xf0\x38\x68\xc1\x6f\xe9\xe8\x9f\xfd\xcb\x11\xfc\xd6\x19\x0c\x3a\xbd\x51\x9a\x0c\xa1\x3f\x80\x6e\xbf\x77\x9e\x8e\xd2\x7e\x6f\x08\xfd\xf7\xd0\xe9\xfd\x01\x1f\xd3\xde\xf9\x31\x20\xb7\x33\xd4\x80\xb7\x85\x76\xf9\x2b\x0d\xdc\xc1\x88\x99\xc3\x6c\x88\xb8\x93\xc0\x54\x55\x09\x99\x02\x19\x9f\x72\x06\x82\xca\xbc\xa4\x39\x42\xae\x96\xa8\x25\x97\x39\x14\xa8\x17\xdc\xb8\x62\x1a\xa0\x32\x0b\x5a\x20\xf8\x82\x5b\x6a\xfd\xca\xa3\x43\x45\x41\xd0\x6a\xc1\x27\x81\xd4\xa0\x2b\xaf\x0b\x48\xed\x56\x81\x25\x62\x66\x1c\x64\x13\x04\x34\x8c\x16\x98\xf9\x3c\x50\x2e\x4d\x39\x31\xd6\xbd\x9a\x96\x92\x39\xff\xee\x5e\x9a\x19\x0a\xc1\x66\xc8\xe6\x0e\x2c\x87\x7e\x7b\xd8\x7d\x79\xf2\xe6\xd5\xf1\xb0\x7b\x7a\xf2\xea\x27\xff\xf3\xfa\xb5\xfb\x79\xf9\xf3\xab\x20\x30\x68\x81\x28\x90\xaa\x94\x06\x6d\xf3\x58\xf0\x02\xa7\x94\x8b\xe6\x19\xb5\xc6\x5b\x6e\x83\xe0\xea\x0a\x0e\x9e\xf3\x0c\x48\x79\x04\xcf\xda\x70\x02\xd7\xd7\xf0\xc3\x0f\x30\xbc\x3c\xef\xb7\x43\x53\x66\x2a\x84\xbb\xbb\xfa\x31\x0c\x82\xcb\x61\x32\xee\xa6\xe3\xce\x60\x94\xbe\xef\x74\x47\xc3\xf6\xc1\x97\x87\x4b\x67\xed\x29\x15\x06\xd7\x41\xc0\xa7\x70\x05\xcf\x20\x7c\xbc\x67\x1d\x42\x1b\xac\x2e\x11\xae\xdf\x3a\xf4\x64\x00\x80\x6c\xa6\x20\xec\x29\xe8\xa6\xd0\xd1\x96\x4f\x29\xb3\x06\xb8\x34\x96\x0a\xe1\xe1\x3e\x06\x97\x33\x97\x79\xe8\xb6\xdf\x72\x0b\x27\xc1\x94\x07\xc1\x87\xe1\xe5\x28\xbd\x68\xe7\xa6\xb4\x5c\xf8\xb0\xcf\x1c\x7b\x16\x54\x66\x40\x96\x70\x70\xf0\xa5\xda\xb1\x86\x5f\xe2\x0c\x97\xb1\x2c\x85\xd8\x84\xa5\x85\x25\x39\x5a\x28\x8b\x8c\x5a\xdc\x5a\xa8\x23\x03\x59\xf9\x25\xab\xa9\x34\x85\xd2\x96\x38\x26\x19\x60\x94\x30\x74\x69\x72\x46\x9d\x42\xe4\xb2\x2c\x72\x60\xa5\x16\x9b\xb3\x64\x38\x81\x2b\xc3\x73\x89\x19\x99\xac\xda\x71\x69\x74\x6c\x66\x54\x63\x3c\xc7\x95\xe6\x32\x37\x31\x13\xaa\xcc\xa2\x5c\xa9\x5c\x60\x94\x17\xf9\xb5\xe7\xa9\x39\x8b\xe3\x82\xb2\x39\xcd\xd1\x44\x3b\x5b\x98\x5a\xc4\xb4\xb0\xe0\x17\x89\xc9\xe6\xb0\xa0\x5c\x86\x70\xe7\x4e\xe9\xaa\xb4\x06\x8b\x08\x84\x42\x8c\x96\xb9\xad\xb1\x51\xa5\x66\x68\x22\xc1\x8d\x8d\xb2\xb8\x72\x44\x36\x0e\xfc\x7a\x00\x3e\xf3\xa7\x05\x8f\x33\xe5\x3d\x93\x39\xae\x5c\xce\xdb\xc1\xeb\x65\x20\xa4\x3e\x22\x3c\xe5\xd4\x40\xb3\x0c\xc8\x13\x8b\xf1\xf0\x00\xfe\x0a\x6c\xd5\x78\x59\x49\xaf\xe3\xce\xc8\xb1\xce\xa2\xd9\xd8\x3b\x41\xab\xa8\xd9\x9c\xd0\x29\xc4\x84\x4b\xaa\x39\x9a\x8a\xaa\x54\x23\x50\xd0\x68\x4a\x61\x1b\x3d\xed\xa6\x8e\xfb\x1b\xe3\x49\xc9\x45\xe6\x9a\x01\xa4\x16\x74\x29\x0d\x1c\x46\x51\x04\x84\xd4\xb1\x0f\x9b\xeb\xe7\x89\xbe\x44\xcd\xa7\xab\x46\x07\xf0\x3e\x9c\x8b\xc4\x94\xd6\xc8\xac\x58\x35\x29\xa2\x93\x18\x17\x6c\xca\x25\x15\x62\x05\xa5\xdc\x24\xef\xac\x37\xa5\x09\x5a\xf0\xbe\xd6\xaf\x47\x87\xe2\xd6\x1f\xdb\x80\xc3\xac\x4e\xca\x71\xa9\x52\xbb\xb2\x70\xf7\x18\xcc\x1c\x6f\xa2\xe0\x22\xed\x25\xe3\x61\xf2\xa9\x33\xe8\x8c\xfa\x83\x76\xf8\xe3\xf7\xfe\x85\x41\x75\xdb\x0f\x0e\xbe\xec\xfa\x5a\x87\x5e\x0b\xbb\xa2\x34\x16\x35\xb3\x02\x9c\xf8\x2c\xa9\xe6\x4e\xc5\x4c\xd0\x72\x7d\xe6\xe0\xe0\xcb\xc7\xcb\x77\xc9\xa0\x97\x8c\x92\xe1\xf8\xd7\x64\x30\x4c\xfb\xbd\x35\xdc\x70\x21\x9c\x42\x6a\x2c\x04\x65\x55\xff\x61\x1b\x47\xc1\x63\x93\xf6\x5e\x3f\x3e\x46\x0b\x12\x99\x6d\x59\x3f\x48\xc3\xcb\xd4\x95\xcf\x7f\x8f\x87\xd0\xa9\x62\x18\xc2\xf5\xbd\x50\x75\xd3\xf1\x79\x3a\x68\xc7\x76\x51\xc4\xf3\x37\x86\x30\x1e\x00\x2c\xe6\x19\xd7\x40\x0a\xef\xa7\xda\xb1\x76\x32\x95\x21\x13\xae\xce\x84\xc2\xa7\x4e\xf7\x63\xe7\x43\x32\x1c\x8f\xfa\xe3\x51\x32\x1c\xb5\x9f\x87\xf3\x72\xe2\x8a\x1f\x82\xff\x4f\xa0\xad\xff\xa3\xd9\x22\x3c\xda\xb5\xee\xf6\x7b\xa3\x4e\xda\x4b\x06\x0f\xed\x09\x2d\xb8\x41\xbd\x44\x5d\x1b\x13\xa6\xa4\xd5\x4a\x08\xd4\x64\x41\x25\xcd\xef\xdf\x14\x5a\xdd\xae\x9a\x07\xc3\x66\x98\x95\x02\xb5\x0f\xb5\xf1\x3f\x4e\x7e\x1f\xb5\x43\x4b\x75\xb8\x11\xb2\x1f\xfd\x6d\x72\x6c\xee\xa6\xcd\x75\xfa\x4a\xe5\x9c\x51\x0b\xba\xbe\x5f\xb9\xb6\x66\x70\xb1\x44\x1d\x00\xfc\x09\xca\xed\xff\xc2\xbf\x96\x57\x27\xe4\xe7\xeb\x17\x9f\xa3\xdd\xdf\x83\x6d\xf0\x3d\xfc\x97\x83\x8b\x76\x98\x3b\xa1\x9a\x6f\x46\x34\x52\x93\x20\x6e\x7e\xf7\x47\xf2\x33\x9f\xe0\xb2\xbc\x8d\xe9\x22\x7b\xfd\x53\xe8\x7d\xd6\x6f\xc7\xf5\xac\x31\xfe\x34\x48\xde\xa7\xbf\xb7\xf7\x67\xdb\x5a\xae\x2b\xab\xf3\xe4\x5d\xda\xe9\x8d\xdf\x0f\xfa\xbd\x51\xd2\x3b\x6f\x4b\x25\xb9\xb4\xa8\x29\xb3\x7c\x89\x4f\x6e\x24\x55\xcb\xa8\x15\x98\x98\xbf\x2c\xc2\x8d\xf6\x36\x62\x5a\x57\xef\xd0\xb5\xa1\xc6\x27\x2d\x6c\x74\x8f\x59\xc4\x55\x0c\x5b\x10\xde\xa2\xe4\x54\xf8\x96\x72\x08\xbf\x7c\xad\x83\x6c\xd9\xd7\xbd\x63\x8f\x70\xbb\x5b\x50\x93\x17\xa2\x6a\x80\xfd\x1c\xed\x60\x3d\x48\x3e\x24\x15\xc4\xfb\xd1\x8f\xe3\x28\xfe\xfc\x39\xaa\xb1\xae\xc9\xb3\xa1\x7b\x78\xf0\xdc\xc5\x64\x7e\xec\x5e\xd0\x8c\x1b\x25\xa1\x26\x11\xdc\x41\xae\xb1\xa2\xe2\x4e\xb4\x35\x71\x9d\x72\x86\x34\x03\x22\x5f\xc2\x1d\xb0\xd2\x02\xc9\xe0\xf0\xee\x10\xc8\x14\x4e\xe1\x0e\xac\xf6\x0b\x57\x67\xa6\xa0\x0c\xcf\xae\x0f\x8f\xaa\xf8\xee\x2e\x77\xd3\x71\x9d\x86\x93\x52\xe7\xfd\x21\xa5\xaf\xfe\x71\xbd\x0e\xdf\x42\xa6\xbc\xcd\x3d\x7d\xea\x2b\xe0\xe7\xc9\xaa\xaa\x67\x50\xe9\x44\xed\x61\x0d\xf7\xde\x76\xb8\xf4\xd7\xee\x59\xb8\xeb\xbc\xfd\x55\xe7\x99\x92\xae\x5a\x28\x0c\x3e\x85\x5c\xc4\x8d\x4f\x8c\x7f\x85\x5b\x64\x42\xff\x83\x62\x3f\xc3\xfe\x3f\xfc\x32\x75\x23\x85\xa2\x99\x03\xd0\xb7\xcf\x55\x83\xdf\xe5\xe0\x62\x1d\xef\x9e\xb6\x81\x6d\x6b\x26\x60\x1b\x5d\xde\xb7\x7f\x5b\xb3\xbf\xe2\x8b\xcd\x16\x2a\x83\x17\xb7\x4f\xd8\xba\x58\x7e\x6b\x13\x84\x7e\x20\x72\x18\xed\x33\xaf\xeb\x01\x60\x56\xc6\xe2\xc2\x75\x2b\x8d\xc6\x52\x6d\x9b\xcb\x1d\x00\x4c\x5d\xc3\xa9\x01\xdd\xc8\x76\x03\xe9\xe3\x3e\xb1\x0b\xea\x3e\x48\x1f\xdc\xc9\x2d\x8c\x36\xde\xd6\xd1\xb6\x6f\xd7\x23\xea\x8c\xff\x0c\xe5\x6f\x7a\xd8\x83\xd4\x93\x22\x56\xc3\x26\x73\x64\x95\x30\x7f\xe3\x84\x0c\xf8\xa2\x1a\x7d\x16\x7e\xb8\xf9\x4e\xc7\xee\x23\x67\x43\xd6\x5c\x2a\x3f\xba\xe2\x6d\x81\xcc\x7d\x99\x1e\xba\x48\x3b\xfe\x0f\xeb\x01\xf1\x29\x19\x59\x9a\x43\xe8\x96\x72\xa6\x9d\xe2\x3e\x4c\x87\x78\x9a\x9c\xed\xe5\x54\x1c\xbf\x88\xc7\x0e\xa7\x6f\xd8\x7f\xd3\xf2\xef\xca\xaf\xb6\xdd\x92\x0c\xc6\x49\x15\xe0\xbb\x13\xf6\x24\x98\xf2\xa0\x29\x88\xff\xfa\xbd\xa7\xfe\x66\xa0\x6d\x26\x4f\x77\xaa\x7a\xed\x0c\xdc\x9d\x7a\xbe\xb5\x72\xd4\xec\xaa\xe7\xaa\x07\x3b\x1f\xac\x02\x51\x6d\x33\x53\xda\xee\x98\x39\x1a\x3e\x36\xdb\x5a\x05\x42\x98\xe0\x28\x6d\xdb\x7f\xd4\x12\xe2\x7d\xf8\x87\x1d\x47\xae\x39\x3d\x76\xe4\x56\x37\xdf\x0e\x47\xdf\x9a\xa7\xff\x17\x00\x00\xff\xff\x3f\x8e\x7f\x9b\xb4\x12\x00\x00") + +func dataDebian_injection_scriptEnvsubstShBytes() ([]byte, error) { + return bindataRead( + _dataDebian_injection_scriptEnvsubstSh, + "data/debian_injection_script.envsubst.sh", + ) +} + +func dataDebian_injection_scriptEnvsubstSh() (*asset, error) { + bytes, err := dataDebian_injection_scriptEnvsubstShBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "data/debian_injection_script.envsubst.sh", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _dataKustomizationYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x4c\x8e\x4d\x0a\xc2\x30\x10\x46\xf7\x39\x45\x2e\x90\x48\x77\x92\x2b\x88\x2b\xc1\xfd\x98\x4e\xea\x90\xe6\x87\x99\x69\x41\x4f\x2f\x25\x22\xae\xdf\xf7\x3e\x1e\x74\xba\x23\x0b\xb5\x1a\x6c\xde\x44\x5b\xa1\x37\xfa\xd8\x6a\xa2\xc5\xe7\xb3\x78\x6a\xa7\x7d\x7a\xa0\xc2\x64\x32\xd5\x39\xd8\xcb\x77\x05\x4a\xad\x9a\x0a\x05\xa5\x43\xc4\x60\x67\x4c\xb0\xad\x6a\x18\xa5\x6d\x1c\x51\x82\xb1\xd6\xd9\x48\x0e\x58\x29\x41\x54\x71\x83\x38\xc5\xd2\x57\x50\xf4\x2f\x28\xab\xe9\xa0\xf1\x89\x72\x53\x06\xc5\x85\xe2\x15\x79\xc1\x21\xff\x92\xf6\x11\x29\xc3\x38\xd0\x71\x90\x1a\x17\x97\xff\x83\x06\xff\x04\x00\x00\xff\xff\x16\x92\x86\x00\xd6\x00\x00\x00") + +func dataKustomizationYamlBytes() ([]byte, error) { + return bindataRead( + _dataKustomizationYaml, + "data/kustomization.yaml", + ) +} + +func dataKustomizationYaml() (*asset, error) { + bytes, err := dataKustomizationYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "data/kustomization.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +// Asset loads and returns the asset for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func Asset(name string) ([]byte, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err) + } + return a.bytes, nil + } + return nil, fmt.Errorf("Asset %s not found", name) +} + +// MustAsset is like Asset but panics when Asset would return an error. +// It simplifies safe initialization of global variables. +func MustAsset(name string) []byte { + a, err := Asset(name) + if err != nil { + panic("asset: Asset(" + name + "): " + err.Error()) + } + + return a +} + +// AssetInfo loads and returns the asset info for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func AssetInfo(name string) (os.FileInfo, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err) + } + return a.info, nil + } + return nil, fmt.Errorf("AssetInfo %s not found", name) +} + +// AssetNames returns the names of the assets. +func AssetNames() []string { + names := make([]string, 0, len(_bindata)) + for name := range _bindata { + names = append(names, name) + } + return names +} + +// _bindata is a table, holding each asset generator, mapped to its name. +var _bindata = map[string]func() (*asset, error){ + "data/debian_injection_script.envsubst.sh": dataDebian_injection_scriptEnvsubstSh, + "data/kustomization.yaml": dataKustomizationYaml, +} + +// AssetDir returns the file names below a certain +// directory embedded in the file by go-bindata. +// For example if you run go-bindata on data/... and data contains the +// following hierarchy: +// data/ +// foo.txt +// img/ +// a.png +// b.png +// then AssetDir("data") would return []string{"foo.txt", "img"} +// AssetDir("data/img") would return []string{"a.png", "b.png"} +// AssetDir("foo.txt") and AssetDir("notexist") would return an error +// AssetDir("") will return []string{"data"}. +func AssetDir(name string) ([]string, error) { + node := _bintree + if len(name) != 0 { + cannonicalName := strings.Replace(name, "\\", "/", -1) + pathList := strings.Split(cannonicalName, "/") + for _, p := range pathList { + node = node.Children[p] + if node == nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + } + } + if node.Func != nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + rv := make([]string, 0, len(node.Children)) + for childName := range node.Children { + rv = append(rv, childName) + } + return rv, nil +} + +type bintree struct { + Func func() (*asset, error) + Children map[string]*bintree +} + +var _bintree = &bintree{nil, map[string]*bintree{ + "data": &bintree{nil, map[string]*bintree{ + "debian_injection_script.envsubst.sh": &bintree{dataDebian_injection_scriptEnvsubstSh, map[string]*bintree{}}, + "kustomization.yaml": &bintree{dataKustomizationYaml, map[string]*bintree{}}, + }}, +}} + +// RestoreAsset restores an asset under the given directory +func RestoreAsset(dir, name string) error { + data, err := Asset(name) + if err != nil { + return err + } + info, err := AssetInfo(name) + if err != nil { + return err + } + err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) + if err != nil { + return err + } + err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) + if err != nil { + return err + } + err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) + if err != nil { + return err + } + return nil +} + +// RestoreAssets restores an asset under the given directory recursively +func RestoreAssets(dir, name string) error { + children, err := AssetDir(name) + // File + if err != nil { + return RestoreAsset(dir, name) + } + // Dir + for _, child := range children { + err = RestoreAssets(dir, filepath.Join(name, child)) + if err != nil { + return err + } + } + return nil +} + +func _filePath(dir, name string) string { + cannonicalName := strings.Replace(name, "\\", "/", -1) + return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) +} diff --git a/test/framework/log/log.go b/test/framework/log/log.go new file mode 100644 index 000000000000..60ca4378d5a1 --- /dev/null +++ b/test/framework/log/log.go @@ -0,0 +1,167 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package log + +import ( + "fmt" + "strings" + "time" + + "bytes" + "encoding/json" + "sort" + + "github.com/onsi/ginkgo" +) + +func nowStamp() string { + return time.Now().UTC().Format(time.RFC3339) +} + +func log(level string, format string, args ...interface{}) { + timeStr := flatten("time", nowStamp()) + fmt.Fprintf(ginkgo.GinkgoWriter, level+": "+format+" "+timeStr+"\n", args...) +} + +// Logf logs the info. +func Logf(format string, args ...interface{}) { + log("INFO", format, args...) +} + +type Logger struct { + level int + prefix string + values []interface{} +} + +func (l Logger) clone() Logger { + return Logger{ + level: l.level, + prefix: l.prefix, + values: copySlice(l.values), + } +} + +func copySlice(in []interface{}) []interface{} { + out := make([]interface{}, len(in)) + copy(out, in) + return out +} + +// trimDuplicates will deduplicates elements provided in multiple KV tuple +// slices, whilst maintaining the distinction between where the items are +// contained. +func trimDuplicates(kvLists ...[]interface{}) [][]interface{} { + // maintain a map of all seen keys + seenKeys := map[interface{}]struct{}{} + // build the same number of output slices as inputs + outs := make([][]interface{}, len(kvLists)) + // iterate over the input slices backwards, as 'later' kv specifications + // of the same key will take precedence over earlier ones + for i := len(kvLists) - 1; i >= 0; i-- { + // initialise this output slice + outs[i] = []interface{}{} + // obtain a reference to the kvList we are processing + kvList := kvLists[i] + + // start iterating at len(kvList) - 2 (i.e. the 2nd last item) for + // slices that have an even number of elements. + // We add (len(kvList) % 2) here to handle the case where there is an + // odd number of elements in a kvList. + // If there is an odd number, then the last element in the slice will + // have the value 'null'. + for i2 := len(kvList) - 2 + (len(kvList) % 2); i2 >= 0; i2 -= 2 { + k := kvList[i2] + // if we have already seen this key, do not include it again + if _, ok := seenKeys[k]; ok { + continue + } + // make a note that we've observed a new key + seenKeys[k] = struct{}{} + // attempt to obtain the value of the key + var v interface{} + // i2+1 should only ever be out of bounds if we handling the first + // iteration over a slice with an odd number of elements + if i2+1 < len(kvList) { + v = kvList[i2+1] + } + // add this KV tuple to the *start* of the output list to maintain + // the original order as we are iterating over the slice backwards + outs[i] = append([]interface{}{k, v}, outs[i]...) + } + } + return outs +} + +func flatten(kvList ...interface{}) string { + keys := make([]string, 0, len(kvList)) + vals := make(map[string]interface{}, len(kvList)) + for i := 0; i < len(kvList); i += 2 { + k, ok := kvList[i].(string) + if !ok { + panic(fmt.Sprintf("key is not a string: %s", pretty(kvList[i]))) + } + var v interface{} + if i+1 < len(kvList) { + v = kvList[i+1] + } + keys = append(keys, k) + vals[k] = v + } + sort.Strings(keys) + buf := bytes.Buffer{} + for i, k := range keys { + v := vals[k] + if i > 0 { + buf.WriteRune(' ') + } + buf.WriteString(pretty(k)) + buf.WriteString("=") + buf.WriteString(pretty(v)) + } + return buf.String() +} + +func pretty(value interface{}) string { + jb, _ := json.Marshal(value) + return string(jb) +} + +func (l Logger) Info(msg string, kvList ...interface{}) { + trimmed := trimDuplicates(l.values, kvList) + fixedStr := flatten(trimmed[0]...) + userStr := flatten(trimmed[1]...) + log("INFO", l.prefix+strings.Join([]string{msg + ":", fixedStr, userStr}, " ")) +} + +func (l Logger) Error(err error, msg string, kvList ...interface{}) { + var loggableErr interface{} + if err != nil { + loggableErr = err.Error() + } + errStr := flatten("error", loggableErr) + trimmed := trimDuplicates(l.values, kvList) + fixedStr := flatten(trimmed[0]...) + userStr := flatten(trimmed[1]...) + log("ERROR", l.prefix+strings.Join([]string{msg + ":", errStr, fixedStr, userStr}, " ")) +} + +func (l Logger) WithValues(kvList ...interface{}) Logger { + new := l.clone() + new.values = append(new.values, kvList...) + return new +} diff --git a/test/framework/machine_helpers.go b/test/framework/machine_helpers.go index 62234eafab75..d9be430c8bb9 100644 --- a/test/framework/machine_helpers.go +++ b/test/framework/machine_helpers.go @@ -26,7 +26,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -120,7 +120,7 @@ func WaitForControlPlaneMachinesToBeUpgraded(ctx context.Context, input WaitForC Expect(input.KubernetesUpgradeVersion).ToNot(BeEmpty(), "Invalid argument. input.KubernetesUpgradeVersion can't be empty when calling WaitForControlPlaneMachinesToBeUpgraded") Expect(input.MachineCount).To(BeNumerically(">", 0), "Invalid argument. input.MachineCount can't be smaller than 1 when calling WaitForControlPlaneMachinesToBeUpgraded") - By("ensuring all machines have upgraded kubernetes version") + By("Ensuring all machines have upgraded kubernetes version") log.Logf("Ensuring all MachineDeployment Machines have upgraded kubernetes version %s", input.KubernetesUpgradeVersion) Eventually(func() (int, error) { diff --git a/test/framework/machinedeployment_helpers.go b/test/framework/machinedeployment_helpers.go index 31a1a1f09d66..982bbb523b55 100644 --- a/test/framework/machinedeployment_helpers.go +++ b/test/framework/machinedeployment_helpers.go @@ -29,7 +29,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/test/framework/machines.go b/test/framework/machines.go index b19c2230140e..8c12109dc692 100644 --- a/test/framework/machines.go +++ b/test/framework/machines.go @@ -35,7 +35,7 @@ type WaitForClusterMachineNodeRefsInput struct { // WaitForClusterMachineNodesRefs waits until all nodes associated with a machine deployment exist. func WaitForClusterMachineNodeRefs(ctx context.Context, input WaitForClusterMachineNodeRefsInput, intervals ...interface{}) { - By("waiting for the machines' nodes to exist") + By("Waiting for the machines' nodes to exist") machines := &clusterv1.MachineList{} Expect(input.GetLister.List(ctx, machines, byClusterOptions(input.Cluster.Name, input.Cluster.Namespace)...)).To(Succeed(), "Failed to get Cluster machines %s/%s", input.Cluster.Namespace, input.Cluster.Name) @@ -61,7 +61,7 @@ type WaitForClusterMachinesReadyInput struct { } func WaitForClusterMachinesReady(ctx context.Context, input WaitForClusterMachinesReadyInput, intervals ...interface{}) { - By("waiting for the machines' nodes to be ready") + By("Waiting for the machines' nodes to be ready") machines := &clusterv1.MachineList{} Expect(input.GetLister.List(ctx, machines, byClusterOptions(input.Cluster.Name, input.Cluster.Namespace)...)).To(Succeed(), "Failed to get Cluster machines %s/%s", input.Cluster.Namespace, input.Cluster.Name) diff --git a/test/framework/namespace_helpers.go b/test/framework/namespace_helpers.go index be46ba2d18e9..6b4069b070e3 100644 --- a/test/framework/namespace_helpers.go +++ b/test/framework/namespace_helpers.go @@ -33,7 +33,7 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" - "sigs.k8s.io/cluster-api/test/framework/internal/log" + "sigs.k8s.io/cluster-api/test/framework/log" "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/controller-runtime/pkg/client" ) diff --git a/test/framework/pod_helpers.go b/test/framework/pod_helpers.go index 70ec68a140fc..79151c24da6e 100644 --- a/test/framework/pod_helpers.go +++ b/test/framework/pod_helpers.go @@ -55,7 +55,7 @@ func WaitForPodListCondition(ctx context.Context, input WaitForPodListConditionI } return true, nil }, intervals...).Should(BeTrue()) - By("pod condition satisfied") + By("Pod condition satisfied") } // EtcdImageTagCondition returns a podListCondition that ensures the pod image diff --git a/test/framework/suite_helpers.go b/test/framework/suite_helpers.go new file mode 100644 index 000000000000..48e12e73f5b1 --- /dev/null +++ b/test/framework/suite_helpers.go @@ -0,0 +1,46 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "os" + "path" + "path/filepath" + "strings" +) + +func GatherJUnitReports(srcDir string, destDir string) error { + if err := os.MkdirAll(srcDir, 0o700); err != nil { + return err + } + return filepath.Walk(srcDir, func(p string, info os.FileInfo, err error) error { + if info.IsDir() && p != srcDir { + return filepath.SkipDir + } + if filepath.Ext(p) != ".xml" { + return nil + } + base := filepath.Base(p) + if strings.HasPrefix(base, "junit") { + newName := strings.ReplaceAll(base, "_", ".") + if err := os.Rename(p, path.Join(destDir, newName)); err != nil { + return err + } + } + return nil + }) +} diff --git a/test/infrastructure/docker/.gitignore b/test/infrastructure/docker/.gitignore index e93330e1dcb4..9650bd390831 100644 --- a/test/infrastructure/docker/.gitignore +++ b/test/infrastructure/docker/.gitignore @@ -18,3 +18,5 @@ manager *.xml e2e/resources/** + +*.sentinel diff --git a/test/infrastructure/docker/Makefile b/test/infrastructure/docker/Makefile index 0d75fec84584..26ad2ac0a621 100644 --- a/test/infrastructure/docker/Makefile +++ b/test/infrastructure/docker/Makefile @@ -15,33 +15,15 @@ # If you update this file, please follow: # https://suva.sh/posts/well-documented-makefiles/ -ROOT = ../../.. +ROOT_DIR_RELATIVE := ../../.. +include $(ROOT_DIR_RELATIVE)/common.mk +include $(ROOT_DIR_RELATIVE)/hack/tools/tools.mk -.DEFAULT_GOAL:=help - -# Use GOPROXY environment variable if set -GOPROXY := $(shell go env GOPROXY) -ifeq ($(GOPROXY),) -GOPROXY := https://proxy.golang.org -endif -export GOPROXY - -# Active module mode, as we use go modules to manage dependencies -export GO111MODULE=on - -# This option is for running docker manifest command -export DOCKER_CLI_EXPERIMENTAL := enabled +GO_SRCS := $(call rwildcard,.,*.go) # Directories. -TOOLS_DIR := hack/tools -TOOLS_BIN_DIR := $(TOOLS_DIR)/bin BIN_DIR := bin -# Binaries. -CONTROLLER_GEN := $(TOOLS_BIN_DIR)/controller-gen -CONVERSION_GEN := $(TOOLS_BIN_DIR)/conversion-gen -GOLANGCI_LINT := $(TOOLS_BIN_DIR)/golangci-lint - # Define Docker related variables. Releases should modify and double check these vars. REGISTRY ?= gcr.io/$(shell gcloud config get-value project) STAGING_REGISTRY := gcr.io/k8s-staging-cluster-api @@ -56,9 +38,6 @@ PULL_POLICY ?= Always all: test manager clusterctl -help: ## Display this help - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - ## -------------------------------------- ## Testing ## -------------------------------------- @@ -75,25 +54,18 @@ test: ## Run tests manager: ## Build manager binary go build -o $(BIN_DIR)/manager sigs.k8s.io/cluster-api/test/infrastructure/docker -$(CONTROLLER_GEN): $(TOOLS_DIR)/go.mod # Build controller-gen from tools folder. - cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/controller-gen sigs.k8s.io/controller-tools/cmd/controller-gen - -$(CONVERSION_GEN): $(TOOLS_DIR)/go.mod - cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/conversion-gen k8s.io/code-generator/cmd/conversion-gen - -$(GOLANGCI_LINT): $(TOOLS_DIR)/go.mod # Build golangci-lint from tools folder. - cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/golangci-lint github.com/golangci/golangci-lint/cmd/golangci-lint - ## -------------------------------------- ## Linting ## -------------------------------------- +LINT_ARGS ?= + .PHONY: lint lint-full lint: $(GOLANGCI_LINT) ## Lint codebase - $(GOLANGCI_LINT) run -v + $(GOLANGCI_LINT) run -v $(LINT_ARGS) -lint-full: $(GOLANGCI_LINT) ## Run slower linters to detect possible issues - $(GOLANGCI_LINT) run -v --fast=false +lint-full: ## Run slower linters to detect possible issues + $(MAKE) lint LINT_ARGS=--fast=false ## -------------------------------------- ## Generate / Manifests @@ -107,12 +79,12 @@ generate: $(CONTROLLER_GEN) ## Generate code .PHONY: generate-go generate-go: $(CONTROLLER_GEN) $(CONVERSION_GEN) ## Runs Go related generate targets $(CONTROLLER_GEN) \ - object:headerFile=$(ROOT)/hack/boilerplate/boilerplate.generatego.txt \ + object:headerFile=$(ROOT_DIR_RELATIVE)/hack/boilerplate/boilerplate.generatego.txt \ paths=./api/... $(CONVERSION_GEN) \ --input-dirs=./api/v1alpha3 \ --output-file-base=zz_generated.conversion \ - --go-header-file=$(ROOT)/hack/boilerplate/boilerplate.generatego.txt + --go-header-file=$(ROOT_DIR_RELATIVE)/hack/boilerplate/boilerplate.generatego.txt .PHONY: generate-manifests generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc. @@ -135,10 +107,12 @@ modules: ## Runs go mod to ensure modules are up to date. ## -------------------------------------- .PHONY: docker-build -docker-build: ## Build the docker image for controller-manager +docker-build: .docker-build.sentinel ## Build the docker image for controller-manager + +.docker-build.sentinel: $(GO_SRCS) DOCKER_BUILDKIT=1 docker build --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) ../../.. -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) --file Dockerfile - MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) $(MAKE) set-manifest-image - $(MAKE) set-manifest-pull-policy + touch $@ + .PHONY: docker-push docker-push: ## Push the docker image @@ -226,6 +200,8 @@ release-alias-tag: # Adds the tag to the last build tag. .PHONY: clean clean: ## Remove all generated files $(MAKE) clean-bin + $(MAKE) clean-release + $(MAKE) clean-sentinels .PHONY: clean-bin clean-bin: ## Remove all generated binaries @@ -236,6 +212,10 @@ clean-bin: ## Remove all generated binaries clean-release: ## Remove the release folder rm -rf $(RELEASE_DIR) +.PHONY: clean-sentinels +clean-sentinels: ## Delete docker build sentinels + -rm -f .docker-build.sentinel + .PHONY: verify verify: ./hack/verify-all.sh diff --git a/test/infrastructure/docker/api/v1alpha3/dockermachine_types.go b/test/infrastructure/docker/api/v1alpha3/dockermachine_types.go index 8fb81a1187c1..801d0a47f265 100644 --- a/test/infrastructure/docker/api/v1alpha3/dockermachine_types.go +++ b/test/infrastructure/docker/api/v1alpha3/dockermachine_types.go @@ -52,6 +52,11 @@ type DockerMachineSpec struct { // against this machine // +optional Bootstrapped bool `json:"bootstrapped,omitempty"` + + // BootstrapTimeout is how long to wait for the machine to bootstrap + // +optional + // +kubebuilder:default:="3m" + BootstrapTimeout string `json:"bootstrapTimeout,omitempty"` } // Mount specifies a host volume to mount into a container. diff --git a/test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachines.yaml b/test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachines.yaml index fee2052298de..bb2d041d9907 100644 --- a/test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachines.yaml +++ b/test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachines.yaml @@ -38,6 +38,11 @@ spec: spec: description: DockerMachineSpec defines the desired state of DockerMachine properties: + bootstrapTimeout: + default: 3m + description: BootstrapTimeout is how long to wait for the machine + to bootstrap + type: string bootstrapped: description: Bootstrapped is true when the kubeadm bootstrapping has been run against this machine diff --git a/test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachinetemplates.yaml b/test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachinetemplates.yaml index 9758443bfedb..b09c407cf7a8 100644 --- a/test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachinetemplates.yaml +++ b/test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockermachinetemplates.yaml @@ -47,6 +47,11 @@ spec: description: Spec is the specification of the desired behavior of the machine. properties: + bootstrapTimeout: + default: 3m + description: BootstrapTimeout is how long to wait for the + machine to bootstrap + type: string bootstrapped: description: Bootstrapped is true when the kubeadm bootstrapping has been run against this machine diff --git a/test/infrastructure/docker/controllers/dockermachine_controller.go b/test/infrastructure/docker/controllers/dockermachine_controller.go index 7b0b733948aa..7e2555b3f5ec 100644 --- a/test/infrastructure/docker/controllers/dockermachine_controller.go +++ b/test/infrastructure/docker/controllers/dockermachine_controller.go @@ -68,6 +68,7 @@ func (r *DockerMachineReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, re if apierrors.IsNotFound(err) { return ctrl.Result{}, nil } + return ctrl.Result{}, err } @@ -78,6 +79,7 @@ func (r *DockerMachineReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, re } if machine == nil { log.Info("Waiting for Machine Controller to set OwnerRef on DockerMachine") + return ctrl.Result{}, nil } @@ -87,10 +89,12 @@ func (r *DockerMachineReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, re cluster, err := util.GetClusterFromMetadata(ctx, r.Client, machine.ObjectMeta) if err != nil { log.Info("DockerMachine owner Machine is missing cluster label or cluster does not exist") + return ctrl.Result{}, err } if cluster == nil { log.Info(fmt.Sprintf("Please associate this machine with a cluster using the label %s: ", clusterv1.ClusterLabelName)) + return ctrl.Result{}, nil } @@ -104,6 +108,7 @@ func (r *DockerMachineReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, re } if err := r.Client.Get(ctx, dockerClusterName, dockerCluster); err != nil { log.Info("DockerCluster is not available yet") + return ctrl.Result{}, nil } @@ -191,6 +196,7 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, machine * // This is required after move, because status is not moved to the target cluster. dockerMachine.Status.Ready = true conditions.MarkTrue(dockerMachine, infrav1.ContainerProvisionedCondition) + return ctrl.Result{}, nil } @@ -198,6 +204,7 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, machine * if machine.Spec.Bootstrap.DataSecretName == nil { log.Info("Waiting for the Bootstrap provider controller to set bootstrap data") conditions.MarkFalse(dockerMachine, infrav1.ContainerProvisionedCondition, infrav1.WaitingForBootstrapDataReason, clusterv1.ConditionSeverityInfo, "") + return ctrl.Result{}, nil } @@ -273,14 +280,26 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, machine * bootstrapData, err := r.getBootstrapData(ctx, machine) if err != nil { r.Log.Error(err, "failed to get bootstrap data") + + return ctrl.Result{}, err + } + + timeout, err := time.ParseDuration(dockerMachine.Spec.BootstrapTimeout) + if err != nil { + r.Log.Error(err, "failed to parse bootstrapTimeout value") + conditions.MarkFalse(dockerMachine, infrav1.BootstrapExecSucceededCondition, infrav1.BootstrappingReason, clusterv1.ConditionSeverityInfo, "") + if err := patchDockerMachine(ctx, patchHelper, dockerMachine); err != nil { + return ctrl.Result{}, errors.Wrap(err, "failed to patch DockerMachine") + } return ctrl.Result{}, err } - timeoutctx, cancel := context.WithTimeout(ctx, 3*time.Minute) + timeoutctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() // Run the bootstrap script. Simulates cloud-init. if err := externalMachine.ExecBootstrap(timeoutctx, bootstrapData); err != nil { conditions.MarkFalse(dockerMachine, infrav1.BootstrapExecSucceededCondition, infrav1.BootstrapFailedReason, clusterv1.ConditionSeverityWarning, "Repeating bootstrap") + return ctrl.Result{}, errors.Wrap(err, "failed to exec DockerMachine bootstrap") } dockerMachine.Spec.Bootstrapped = true @@ -316,6 +335,7 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, machine * // state changes during control plane provisioning. if err := externalMachine.SetNodeProviderID(ctx); err != nil { r.Log.Error(err, "failed to patch the Kubernetes node with the machine providerID") + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } // Set ProviderID so the Cluster API Machine Controller can pull it @@ -401,6 +421,7 @@ func (r *DockerMachineReconciler) DockerClusterToDockerMachines(o handler.MapObj c, ok := o.Object.(*infrav1.DockerCluster) if !ok { r.Log.Error(errors.Errorf("expected a DockerCluster but got a %T", o.Object), "failed to get DockerMachine for DockerCluster") + return nil } log := r.Log.WithValues("DockerCluster", c.Name, "Namespace", c.Namespace) @@ -411,6 +432,7 @@ func (r *DockerMachineReconciler) DockerClusterToDockerMachines(o handler.MapObj return result case err != nil: log.Error(err, "failed to get owning cluster") + return result } @@ -418,6 +440,7 @@ func (r *DockerMachineReconciler) DockerClusterToDockerMachines(o handler.MapObj machineList := &clusterv1.MachineList{} if err := r.Client.List(context.TODO(), machineList, client.InNamespace(c.Namespace), client.MatchingLabels(labels)); err != nil { log.Error(err, "failed to list DockerMachines") + return nil } for _, m := range machineList.Items { diff --git a/versions.mk b/versions.mk new file mode 100644 index 000000000000..8f80a914439f --- /dev/null +++ b/versions.mk @@ -0,0 +1,16 @@ +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GOLANG_VERSION := 1.13.15 +CERT_MANAGER_VERSION := v0.16.1