From 103f1db29595d4c90c3771ff07d5eadda265351b Mon Sep 17 00:00:00 2001 From: Tzu-Hua Lan Date: Thu, 18 Jul 2024 10:42:47 +0800 Subject: [PATCH 01/12] [YUNIKORN-2760] `make tools` should check the version of tools From 7ea1b0bc88feac8c620cf552159769462444db13 Mon Sep 17 00:00:00 2001 From: Tzu-Hua Lan Date: Thu, 18 Jul 2024 12:17:34 +0800 Subject: [PATCH 02/12] build: update `shellcheck` --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 030a68b1c..e207521d8 100644 --- a/Makefile +++ b/Makefile @@ -132,7 +132,8 @@ endif # shellcheck SHELLCHECK_VERSION=v0.9.0 -SHELLCHECK_BIN=${TOOLS_DIR}/shellcheck +SHELLCHECK_PATH=${TOOLS_DIR}/shellcheck-$(SHELLCHECK_VERSION) +SHELLCHECK_BIN=${SHELLCHECK_PATH}/shellcheck SHELLCHECK_ARCHIVE := shellcheck-$(SHELLCHECK_VERSION).$(OS).$(HOST_ARCH).tar.xz ifeq (darwin, $(OS)) ifeq (arm64, $(HOST_ARCH)) @@ -229,9 +230,9 @@ tools: $(SHELLCHECK_BIN) $(GOLANGCI_LINT_BIN) $(KUBECTL_BIN) $(KIND_BIN) $(HELM_ # Install shellcheck $(SHELLCHECK_BIN): @echo "installing shellcheck $(SHELLCHECK_VERSION)" - @mkdir -p "$(TOOLS_DIR)" + @mkdir -p "$(SHELLCHECK_PATH)" @curl -sSfL "https://github.com/koalaman/shellcheck/releases/download/$(SHELLCHECK_VERSION)/$(SHELLCHECK_ARCHIVE)" \ - | tar -x -J --strip-components=1 -C "$(TOOLS_DIR)" "shellcheck-$(SHELLCHECK_VERSION)/shellcheck" + | tar -x -J --strip-components=1 -C "$(SHELLCHECK_PATH)" "shellcheck-$(SHELLCHECK_VERSION)/shellcheck" # Install golangci-lint $(GOLANGCI_LINT_BIN): From 03e1fed9341c85ba5c54b6399b498607d3208f86 Mon Sep 17 00:00:00 2001 From: Tzu-Hua Lan Date: Thu, 18 Jul 2024 12:24:26 +0800 Subject: [PATCH 03/12] build: update `golangci-lint` --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e207521d8..384f852d2 100644 --- a/Makefile +++ b/Makefile @@ -147,7 +147,8 @@ endif # golangci-lint GOLANGCI_LINT_VERSION=1.57.2 -GOLANGCI_LINT_BIN=$(TOOLS_DIR)/golangci-lint +GOLANGCI_LINT_PATH=$(TOOLS_DIR)/golangci-lint-$(GOLANGCI_LINT_VERSION) +GOLANGCI_LINT_BIN=$(GOLANGCI_LINT_PATH)/golangci-lint GOLANGCI_LINT_ARCHIVE=golangci-lint-$(GOLANGCI_LINT_VERSION)-$(OS)-$(EXEC_ARCH).tar.gz GOLANGCI_LINT_ARCHIVEBASE=golangci-lint-$(GOLANGCI_LINT_VERSION)-$(OS)-$(EXEC_ARCH) @@ -237,9 +238,9 @@ $(SHELLCHECK_BIN): # Install golangci-lint $(GOLANGCI_LINT_BIN): @echo "installing golangci-lint v$(GOLANGCI_LINT_VERSION)" - @mkdir -p "$(TOOLS_DIR)" + @mkdir -p "$(GOLANGCI_LINT_PATH)" @curl -sSfL "https://github.com/golangci/golangci-lint/releases/download/v$(GOLANGCI_LINT_VERSION)/$(GOLANGCI_LINT_ARCHIVE)" \ - | tar -x -z --strip-components=1 -C "$(TOOLS_DIR)" "$(GOLANGCI_LINT_ARCHIVEBASE)/golangci-lint" + | tar -x -z --strip-components=1 -C "$(GOLANGCI_LINT_PATH)" "$(GOLANGCI_LINT_ARCHIVEBASE)/golangci-lint" # Install kubectl $(KUBECTL_BIN): From 2c2665df7bed176fd948f426640e3b3706ecb48f Mon Sep 17 00:00:00 2001 From: Tzu-Hua Lan Date: Thu, 18 Jul 2024 12:28:58 +0800 Subject: [PATCH 04/12] build: update `kubectl` --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 384f852d2..51eec02ed 100644 --- a/Makefile +++ b/Makefile @@ -154,7 +154,8 @@ GOLANGCI_LINT_ARCHIVEBASE=golangci-lint-$(GOLANGCI_LINT_VERSION)-$(OS)-$(EXEC_AR # kubectl KUBECTL_VERSION=v1.27.7 -KUBECTL_BIN=$(TOOLS_DIR)/kubectl +KUBECTL_PATH=$(TOOLS_DIR)/kubectl-$(KUBECTL_VERSION) +KUBECTL_BIN=$(KUBECTL_PATH)/kubectl # kind KIND_VERSION=v0.23.0 @@ -245,7 +246,7 @@ $(GOLANGCI_LINT_BIN): # Install kubectl $(KUBECTL_BIN): @echo "installing kubectl $(KUBECTL_VERSION)" - @mkdir -p "$(TOOLS_DIR)" + @mkdir -p "$(KUBECTL_PATH)" @curl -sSfL -o "$(KUBECTL_BIN)" \ "https://storage.googleapis.com/kubernetes-release/release/$(KUBECTL_VERSION)/bin/$(OS)/$(EXEC_ARCH)/kubectl" && \ chmod +x "$(KUBECTL_BIN)" From 9d9a95aa163ff7766423b68916c4ca89a774addc Mon Sep 17 00:00:00 2001 From: Tzu-Hua Lan Date: Thu, 18 Jul 2024 12:31:25 +0800 Subject: [PATCH 05/12] build: update `kind` --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 51eec02ed..f1a08c1b7 100644 --- a/Makefile +++ b/Makefile @@ -159,7 +159,8 @@ KUBECTL_BIN=$(KUBECTL_PATH)/kubectl # kind KIND_VERSION=v0.23.0 -KIND_BIN=$(TOOLS_DIR)/kind +KIND_PATH=$(TOOLS_DIR)/kind-$(KIND_VERSION) +KIND_BIN=$(KIND_PATH)/kind # helm HELM_VERSION=v3.12.1 @@ -254,7 +255,7 @@ $(KUBECTL_BIN): # Install kind $(KIND_BIN): @echo "installing kind $(KIND_VERSION)" - @mkdir -p "$(TOOLS_DIR)" + @mkdir -p "$(KIND_PATH)" @curl -sSfL -o "$(KIND_BIN)" \ "https://kind.sigs.k8s.io/dl/$(KIND_VERSION)/kind-$(OS)-$(EXEC_ARCH)" && \ chmod +x "$(KIND_BIN)" From 453a13d0abf002276b5886afed4605413132cbc8 Mon Sep 17 00:00:00 2001 From: Tzu-Hua Lan Date: Thu, 18 Jul 2024 12:33:15 +0800 Subject: [PATCH 06/12] build: update `helm` --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f1a08c1b7..4f581f70e 100644 --- a/Makefile +++ b/Makefile @@ -164,7 +164,8 @@ KIND_BIN=$(KIND_PATH)/kind # helm HELM_VERSION=v3.12.1 -HELM_BIN=$(TOOLS_DIR)/helm +HELM_PATH=$(TOOLS_DIR)/helm-$(HELM_VERSION) +HELM_BIN=$(HELM_PATH)/helm HELM_ARCHIVE=helm-$(HELM_VERSION)-$(OS)-$(EXEC_ARCH).tar.gz HELM_ARCHIVE_BASE=$(OS)-$(EXEC_ARCH) @@ -263,9 +264,9 @@ $(KIND_BIN): # Install helm $(HELM_BIN): @echo "installing helm $(HELM_VERSION)" - @mkdir -p "$(TOOLS_DIR)" + @mkdir -p "$(HELM_PATH)" @curl -sSfL "https://get.helm.sh/$(HELM_ARCHIVE)" \ - | tar -x -z --strip-components=1 -C "$(TOOLS_DIR)" "$(HELM_ARCHIVE_BASE)/helm" + | tar -x -z --strip-components=1 -C "$(HELM_PATH)" "$(HELM_ARCHIVE_BASE)/helm" # Install spark $(SPARK_SUBMIT_CMD): From 4288c50fa76842e99a7ab5db5cdd04e3a156de97 Mon Sep 17 00:00:00 2001 From: Tzu-Hua Lan Date: Thu, 18 Jul 2024 12:37:20 +0800 Subject: [PATCH 07/12] build: update `spark` --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4f581f70e..2c3801b82 100644 --- a/Makefile +++ b/Makefile @@ -173,7 +173,7 @@ HELM_ARCHIVE_BASE=$(OS)-$(EXEC_ARCH) export SPARK_VERSION=3.3.3 # sometimes the image is not avaiable with $SPARK_VERSION, the minor version must match export SPARK_PYTHON_VERSION=3.3.1 -export SPARK_HOME=$(BASE_DIR)$(TOOLS_DIR)/spark +export SPARK_HOME=$(BASE_DIR)$(TOOLS_DIR)/spark-v$(SPARK_VERSION) export SPARK_SUBMIT_CMD=$(SPARK_HOME)/bin/spark-submit export SPARK_PYTHON_IMAGE=docker.io/apache/spark-py:v$(SPARK_PYTHON_VERSION) @@ -270,7 +270,7 @@ $(HELM_BIN): # Install spark $(SPARK_SUBMIT_CMD): - @echo "installing spark $(SPARK_VERSION)" + @echo "installing spark v$(SPARK_VERSION)" @rm -rf "$(SPARK_HOME)" "$(SPARK_HOME).tmp" @mkdir -p "$(SPARK_HOME).tmp" @curl -sSfL "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz" \ From 9ab3cd0f2a69bcda6fc2677a2022d506c3d3ca08 Mon Sep 17 00:00:00 2001 From: Tzu-Hua Lan Date: Thu, 18 Jul 2024 12:42:44 +0800 Subject: [PATCH 08/12] build: update `go-licenses` --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 2c3801b82..864a96598 100644 --- a/Makefile +++ b/Makefile @@ -179,7 +179,8 @@ export SPARK_PYTHON_IMAGE=docker.io/apache/spark-py:v$(SPARK_PYTHON_VERSION) # go-licenses GO_LICENSES_VERSION=v1.6.0 -GO_LICENSES_BIN=$(TOOLS_DIR)/go-licenses +GO_LICENSES_PATH=$(TOOLS_DIR)/go-licenses-$(GO_LICENSES_VERSION) +GO_LICENSES_BIN=$(GO_LICENSES_PATH)/go-licenses # ginkgo GINKGO_BIN=$(TOOLS_DIR)/ginkgo @@ -280,8 +281,8 @@ $(SPARK_SUBMIT_CMD): # Install go-licenses $(GO_LICENSES_BIN): @echo "installing go-licenses $(GO_LICENSES_VERSION)" - @mkdir -p "$(TOOLS_DIR)" - @GOBIN="$(BASE_DIR)/$(TOOLS_DIR)" "$(GO)" install "github.com/google/go-licenses@$(GO_LICENSES_VERSION)" + @mkdir -p "$(GO_LICENSES_PATH)" + @GOBIN="$(BASE_DIR)/$(GO_LICENSES_PATH)" "$(GO)" install "github.com/google/go-licenses@$(GO_LICENSES_VERSION)" $(GINKGO_BIN): @echo "installing ginkgo" From 2efed1c2cedf62bf37d2047a306136ea9a84b4d2 Mon Sep 17 00:00:00 2001 From: Tzu-Hua Lan Date: Thu, 18 Jul 2024 13:46:32 +0800 Subject: [PATCH 09/12] build: update `ginkgo` --- Makefile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 864a96598..b0a92f21b 100644 --- a/Makefile +++ b/Makefile @@ -183,7 +183,9 @@ GO_LICENSES_PATH=$(TOOLS_DIR)/go-licenses-$(GO_LICENSES_VERSION) GO_LICENSES_BIN=$(GO_LICENSES_PATH)/go-licenses # ginkgo -GINKGO_BIN=$(TOOLS_DIR)/ginkgo +GINKGO_VERSION=v2.19.0 +GINKGO_PATH=$(TOOLS_DIR)/ginkgo-$(GINKGO_VERSION) +GINKGO_BIN=$(GINKGO_PATH)/ginkgo FLAG_PREFIX=github.com/apache/yunikorn-k8shim/pkg/conf @@ -285,9 +287,9 @@ $(GO_LICENSES_BIN): @GOBIN="$(BASE_DIR)/$(GO_LICENSES_PATH)" "$(GO)" install "github.com/google/go-licenses@$(GO_LICENSES_VERSION)" $(GINKGO_BIN): - @echo "installing ginkgo" - @mkdir -p "$(TOOLS_DIR)" - @GOBIN="$(BASE_DIR)/$(TOOLS_DIR)" "$(GO)" install "github.com/onsi/ginkgo/v2/ginkgo" + @echo "installing ginkgo $(GINKGO_VERSION)" + @mkdir -p "$(GINKGO_PATH)" + @GOBIN="$(BASE_DIR)/$(GINKGO_PATH)" "$(GO)" install "github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION)" # Run lint against the previous commit for PR and branch build # In dev setup look at all changes on top of master From 9d1b13d9c63d458f887afb2408b8d8f52aec3f8b Mon Sep 17 00:00:00 2001 From: Tzu-Hua Lan Date: Thu, 18 Jul 2024 15:12:05 +0800 Subject: [PATCH 10/12] build: update `run-e2e-tests.sh` --- Makefile | 11 +++++++++++ scripts/run-e2e-tests.sh | 9 ++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b0a92f21b..01831fbc5 100644 --- a/Makefile +++ b/Makefile @@ -230,6 +230,17 @@ init: conf/scheduler-config-local.yaml conf/scheduler-config-local.yaml: conf/scheduler-config.yaml ./scripts/plugin-conf-gen.sh $(KUBECONFIG) "conf/scheduler-config.yaml" "conf/scheduler-config-local.yaml" +# Print tools version +.PHONY: print_kubectl_version +print_kubectl_version: + @echo $(KUBECTL_VERSION) +.PHONY: print_kind_version +print_kind_version: + @echo $(KIND_VERSION) +.PHONY: print_helm_version +print_helm_version: + @echo $(HELM_VERSION) + # Install tools .PHONY: tools tools: $(SHELLCHECK_BIN) $(GOLANGCI_LINT_BIN) $(KUBECTL_BIN) $(KIND_BIN) $(HELM_BIN) $(SPARK_SUBMIT_CMD) $(GO_LICENSES_BIN) $(GINKGO_BIN) diff --git a/scripts/run-e2e-tests.sh b/scripts/run-e2e-tests.sh index 9ac477ed1..a9ea909d4 100755 --- a/scripts/run-e2e-tests.sh +++ b/scripts/run-e2e-tests.sh @@ -17,9 +17,12 @@ #limitations under the License. TOOLS_DIRECTORY=tools -HELM=$TOOLS_DIRECTORY/helm -KIND=$TOOLS_DIRECTORY/kind -KUBECTL=$TOOLS_DIRECTORY/kubectl +HELM_VERSION=$(make -s print_helm_version) +KIND_VERSION=$(make -s print_kind_version) +KUBECTL_VERSION=$(make -s print_kubectl_version) +HELM=$TOOLS_DIRECTORY/helm-$HELM_VERSION/helm +KIND=$TOOLS_DIRECTORY/kind-$KIND_VERSION/kind +KUBECTL=$TOOLS_DIRECTORY/kubectl-$KUBECTL_VERSION/kubectl GO="${GO:-go}" export GO From dd37b16537dc1ac71b0ac399d0b4bdbf0e6a48eb Mon Sep 17 00:00:00 2001 From: Tzu-Hua Lan Date: Thu, 18 Jul 2024 18:40:05 +0800 Subject: [PATCH 11/12] build: add `v` in `golangci-lint` path --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 01831fbc5..125afba26 100644 --- a/Makefile +++ b/Makefile @@ -147,7 +147,7 @@ endif # golangci-lint GOLANGCI_LINT_VERSION=1.57.2 -GOLANGCI_LINT_PATH=$(TOOLS_DIR)/golangci-lint-$(GOLANGCI_LINT_VERSION) +GOLANGCI_LINT_PATH=$(TOOLS_DIR)/golangci-lint-v$(GOLANGCI_LINT_VERSION) GOLANGCI_LINT_BIN=$(GOLANGCI_LINT_PATH)/golangci-lint GOLANGCI_LINT_ARCHIVE=golangci-lint-$(GOLANGCI_LINT_VERSION)-$(OS)-$(EXEC_ARCH).tar.gz GOLANGCI_LINT_ARCHIVEBASE=golangci-lint-$(GOLANGCI_LINT_VERSION)-$(OS)-$(EXEC_ARCH) From 1a28948339c2f10fae6d6db0c8aaf38e6f8db0c7 Mon Sep 17 00:00:00 2001 From: Tzu-Hua Lan Date: Sat, 27 Jul 2024 19:39:11 +0800 Subject: [PATCH 12/12] build: export tools to `PATH` --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index 125afba26..848eb4f74 100644 --- a/Makefile +++ b/Makefile @@ -144,6 +144,7 @@ ifeq (armv7l, $(HOST_ARCH)) SHELLCHECK_ARCHIVE := shellcheck-$(SHELLCHECK_VERSION).$(OS).armv6hf.tar.xz endif endif +export PATH := $(BASE_DIR)/$(SHELLCHECK_PATH):$(PATH) # golangci-lint GOLANGCI_LINT_VERSION=1.57.2 @@ -151,16 +152,19 @@ GOLANGCI_LINT_PATH=$(TOOLS_DIR)/golangci-lint-v$(GOLANGCI_LINT_VERSION) GOLANGCI_LINT_BIN=$(GOLANGCI_LINT_PATH)/golangci-lint GOLANGCI_LINT_ARCHIVE=golangci-lint-$(GOLANGCI_LINT_VERSION)-$(OS)-$(EXEC_ARCH).tar.gz GOLANGCI_LINT_ARCHIVEBASE=golangci-lint-$(GOLANGCI_LINT_VERSION)-$(OS)-$(EXEC_ARCH) +export PATH := $(BASE_DIR)/$(GOLANGCI_LINT_PATH):$(PATH) # kubectl KUBECTL_VERSION=v1.27.7 KUBECTL_PATH=$(TOOLS_DIR)/kubectl-$(KUBECTL_VERSION) KUBECTL_BIN=$(KUBECTL_PATH)/kubectl +export PATH := $(BASE_DIR)/$(KUBECTL_PATH):$(PATH) # kind KIND_VERSION=v0.23.0 KIND_PATH=$(TOOLS_DIR)/kind-$(KIND_VERSION) KIND_BIN=$(KIND_PATH)/kind +export PATH := $(BASE_DIR)/$(KIND_PATH):$(PATH) # helm HELM_VERSION=v3.12.1 @@ -168,6 +172,7 @@ HELM_PATH=$(TOOLS_DIR)/helm-$(HELM_VERSION) HELM_BIN=$(HELM_PATH)/helm HELM_ARCHIVE=helm-$(HELM_VERSION)-$(OS)-$(EXEC_ARCH).tar.gz HELM_ARCHIVE_BASE=$(OS)-$(EXEC_ARCH) +export PATH := $(BASE_DIR)/$(HELM_PATH):$(PATH) # spark export SPARK_VERSION=3.3.3 @@ -176,16 +181,19 @@ export SPARK_PYTHON_VERSION=3.3.1 export SPARK_HOME=$(BASE_DIR)$(TOOLS_DIR)/spark-v$(SPARK_VERSION) export SPARK_SUBMIT_CMD=$(SPARK_HOME)/bin/spark-submit export SPARK_PYTHON_IMAGE=docker.io/apache/spark-py:v$(SPARK_PYTHON_VERSION) +export PATH := $(SPARK_HOME):$(PATH) # go-licenses GO_LICENSES_VERSION=v1.6.0 GO_LICENSES_PATH=$(TOOLS_DIR)/go-licenses-$(GO_LICENSES_VERSION) GO_LICENSES_BIN=$(GO_LICENSES_PATH)/go-licenses +export PATH := $(BASE_DIR)/$(GO_LICENSES_PATH):$(PATH) # ginkgo GINKGO_VERSION=v2.19.0 GINKGO_PATH=$(TOOLS_DIR)/ginkgo-$(GINKGO_VERSION) GINKGO_BIN=$(GINKGO_PATH)/ginkgo +export PATH := $(BASE_DIR)/$(GINKGO_PATH):$(PATH) FLAG_PREFIX=github.com/apache/yunikorn-k8shim/pkg/conf