Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[YUNIKORN-2760] make tools should check the version of tools #880

Closed
wants to merge 12 commits into from
+55 −25
Closed

[YUNIKORN-2760] make tools should check the version of tools #880

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
103f1db
[YUNIKORN-2760] `make tools` should check the version of tools
blueBlue0102 Jul 18, 2024
7ea1b0b
build: update `shellcheck`
blueBlue0102 Jul 18, 2024
03e1fed
build: update `golangci-lint`
blueBlue0102 Jul 18, 2024
2c2665d
build: update `kubectl`
blueBlue0102 Jul 18, 2024
9d9a95a
build: update `kind`
blueBlue0102 Jul 18, 2024
453a13d
build: update `helm`
blueBlue0102 Jul 18, 2024
4288c50
build: update `spark`
blueBlue0102 Jul 18, 2024
9ab3cd0
build: update `go-licenses`
blueBlue0102 Jul 18, 2024
2efed1c
build: update `ginkgo`
blueBlue0102 Jul 18, 2024
9d1b13d
build: update `run-e2e-tests.sh`
blueBlue0102 Jul 18, 2024
dd37b16
build: add `v` in `golangci-lint` path
blueBlue0102 Jul 18, 2024
1a28948
build: export tools to `PATH`
blueBlue0102 Jul 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 49 additions & 22 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,8 @@ endif

# shellcheck
SHELLCHECK_VERSION=v0.9.0
SHELLCHECK_BIN=${TOOLS_DIR}/shellcheck
SHELLCHECK_PATH=${TOOLS_DIR}/shellcheck-$(SHELLCHECK_VERSION)
SHELLCHECK_BIN=${SHELLCHECK_PATH}/shellcheck
SHELLCHECK_ARCHIVE := shellcheck-$(SHELLCHECK_VERSION).$(OS).$(HOST_ARCH).tar.xz
ifeq (darwin, $(OS))
ifeq (arm64, $(HOST_ARCH))
Expand All @@ -143,41 +144,56 @@ ifeq (armv7l, $(HOST_ARCH))
SHELLCHECK_ARCHIVE := shellcheck-$(SHELLCHECK_VERSION).$(OS).armv6hf.tar.xz
endif
endif
export PATH := $(BASE_DIR)/$(SHELLCHECK_PATH):$(PATH)

# golangci-lint
GOLANGCI_LINT_VERSION=1.57.2
GOLANGCI_LINT_BIN=$(TOOLS_DIR)/golangci-lint
GOLANGCI_LINT_PATH=$(TOOLS_DIR)/golangci-lint-v$(GOLANGCI_LINT_VERSION)
GOLANGCI_LINT_BIN=$(GOLANGCI_LINT_PATH)/golangci-lint
GOLANGCI_LINT_ARCHIVE=golangci-lint-$(GOLANGCI_LINT_VERSION)-$(OS)-$(EXEC_ARCH).tar.gz
GOLANGCI_LINT_ARCHIVEBASE=golangci-lint-$(GOLANGCI_LINT_VERSION)-$(OS)-$(EXEC_ARCH)
export PATH := $(BASE_DIR)/$(GOLANGCI_LINT_PATH):$(PATH)

# kubectl
KUBECTL_VERSION=v1.27.7
KUBECTL_BIN=$(TOOLS_DIR)/kubectl
KUBECTL_PATH=$(TOOLS_DIR)/kubectl-$(KUBECTL_VERSION)
KUBECTL_BIN=$(KUBECTL_PATH)/kubectl
export PATH := $(BASE_DIR)/$(KUBECTL_PATH):$(PATH)

# kind
KIND_VERSION=v0.23.0
KIND_BIN=$(TOOLS_DIR)/kind
KIND_PATH=$(TOOLS_DIR)/kind-$(KIND_VERSION)
KIND_BIN=$(KIND_PATH)/kind
export PATH := $(BASE_DIR)/$(KIND_PATH):$(PATH)

# helm
HELM_VERSION=v3.12.1
HELM_BIN=$(TOOLS_DIR)/helm
HELM_PATH=$(TOOLS_DIR)/helm-$(HELM_VERSION)
HELM_BIN=$(HELM_PATH)/helm
HELM_ARCHIVE=helm-$(HELM_VERSION)-$(OS)-$(EXEC_ARCH).tar.gz
HELM_ARCHIVE_BASE=$(OS)-$(EXEC_ARCH)
export PATH := $(BASE_DIR)/$(HELM_PATH):$(PATH)

# spark
export SPARK_VERSION=3.3.3
# sometimes the image is not avaiable with $SPARK_VERSION, the minor version must match
export SPARK_PYTHON_VERSION=3.3.1
export SPARK_HOME=$(BASE_DIR)$(TOOLS_DIR)/spark
export SPARK_HOME=$(BASE_DIR)$(TOOLS_DIR)/spark-v$(SPARK_VERSION)
export SPARK_SUBMIT_CMD=$(SPARK_HOME)/bin/spark-submit
export SPARK_PYTHON_IMAGE=docker.io/apache/spark-py:v$(SPARK_PYTHON_VERSION)
export PATH := $(SPARK_HOME):$(PATH)

# go-licenses
GO_LICENSES_VERSION=v1.6.0
GO_LICENSES_BIN=$(TOOLS_DIR)/go-licenses
GO_LICENSES_PATH=$(TOOLS_DIR)/go-licenses-$(GO_LICENSES_VERSION)
GO_LICENSES_BIN=$(GO_LICENSES_PATH)/go-licenses
export PATH := $(BASE_DIR)/$(GO_LICENSES_PATH):$(PATH)

# ginkgo
GINKGO_BIN=$(TOOLS_DIR)/ginkgo
GINKGO_VERSION=v2.19.0
GINKGO_PATH=$(TOOLS_DIR)/ginkgo-$(GINKGO_VERSION)
GINKGO_BIN=$(GINKGO_PATH)/ginkgo
export PATH := $(BASE_DIR)/$(GINKGO_PATH):$(PATH)

FLAG_PREFIX=github.com/apache/yunikorn-k8shim/pkg/conf

Expand Down Expand Up @@ -222,50 +238,61 @@ init: conf/scheduler-config-local.yaml
conf/scheduler-config-local.yaml: conf/scheduler-config.yaml
./scripts/plugin-conf-gen.sh $(KUBECONFIG) "conf/scheduler-config.yaml" "conf/scheduler-config-local.yaml"

# Print tools version
.PHONY: print_kubectl_version
print_kubectl_version:
@echo $(KUBECTL_VERSION)
.PHONY: print_kind_version
print_kind_version:
@echo $(KIND_VERSION)
.PHONY: print_helm_version
print_helm_version:
@echo $(HELM_VERSION)

# Install tools
.PHONY: tools
tools: $(SHELLCHECK_BIN) $(GOLANGCI_LINT_BIN) $(KUBECTL_BIN) $(KIND_BIN) $(HELM_BIN) $(SPARK_SUBMIT_CMD) $(GO_LICENSES_BIN) $(GINKGO_BIN)

# Install shellcheck
$(SHELLCHECK_BIN):
@echo "installing shellcheck $(SHELLCHECK_VERSION)"
@mkdir -p "$(TOOLS_DIR)"
@mkdir -p "$(SHELLCHECK_PATH)"
@curl -sSfL "https://github.com/koalaman/shellcheck/releases/download/$(SHELLCHECK_VERSION)/$(SHELLCHECK_ARCHIVE)" \
| tar -x -J --strip-components=1 -C "$(TOOLS_DIR)" "shellcheck-$(SHELLCHECK_VERSION)/shellcheck"
| tar -x -J --strip-components=1 -C "$(SHELLCHECK_PATH)" "shellcheck-$(SHELLCHECK_VERSION)/shellcheck"

# Install golangci-lint
$(GOLANGCI_LINT_BIN):
@echo "installing golangci-lint v$(GOLANGCI_LINT_VERSION)"
@mkdir -p "$(TOOLS_DIR)"
@mkdir -p "$(GOLANGCI_LINT_PATH)"
@curl -sSfL "https://github.com/golangci/golangci-lint/releases/download/v$(GOLANGCI_LINT_VERSION)/$(GOLANGCI_LINT_ARCHIVE)" \
| tar -x -z --strip-components=1 -C "$(TOOLS_DIR)" "$(GOLANGCI_LINT_ARCHIVEBASE)/golangci-lint"
| tar -x -z --strip-components=1 -C "$(GOLANGCI_LINT_PATH)" "$(GOLANGCI_LINT_ARCHIVEBASE)/golangci-lint"

# Install kubectl
$(KUBECTL_BIN):
@echo "installing kubectl $(KUBECTL_VERSION)"
@mkdir -p "$(TOOLS_DIR)"
@mkdir -p "$(KUBECTL_PATH)"
@curl -sSfL -o "$(KUBECTL_BIN)" \
"https://storage.googleapis.com/kubernetes-release/release/$(KUBECTL_VERSION)/bin/$(OS)/$(EXEC_ARCH)/kubectl" && \
chmod +x "$(KUBECTL_BIN)"

# Install kind
$(KIND_BIN):
@echo "installing kind $(KIND_VERSION)"
@mkdir -p "$(TOOLS_DIR)"
@mkdir -p "$(KIND_PATH)"
@curl -sSfL -o "$(KIND_BIN)" \
"https://kind.sigs.k8s.io/dl/$(KIND_VERSION)/kind-$(OS)-$(EXEC_ARCH)" && \
chmod +x "$(KIND_BIN)"

# Install helm
$(HELM_BIN):
@echo "installing helm $(HELM_VERSION)"
@mkdir -p "$(TOOLS_DIR)"
@mkdir -p "$(HELM_PATH)"
@curl -sSfL "https://get.helm.sh/$(HELM_ARCHIVE)" \
| tar -x -z --strip-components=1 -C "$(TOOLS_DIR)" "$(HELM_ARCHIVE_BASE)/helm"
| tar -x -z --strip-components=1 -C "$(HELM_PATH)" "$(HELM_ARCHIVE_BASE)/helm"

# Install spark
$(SPARK_SUBMIT_CMD):
@echo "installing spark $(SPARK_VERSION)"
@echo "installing spark v$(SPARK_VERSION)"
@rm -rf "$(SPARK_HOME)" "$(SPARK_HOME).tmp"
@mkdir -p "$(SPARK_HOME).tmp"
@curl -sSfL "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz" \
Expand All @@ -275,13 +302,13 @@ $(SPARK_SUBMIT_CMD):
# Install go-licenses
$(GO_LICENSES_BIN):
@echo "installing go-licenses $(GO_LICENSES_VERSION)"
@mkdir -p "$(TOOLS_DIR)"
@GOBIN="$(BASE_DIR)/$(TOOLS_DIR)" "$(GO)" install "github.com/google/go-licenses@$(GO_LICENSES_VERSION)"
@mkdir -p "$(GO_LICENSES_PATH)"
@GOBIN="$(BASE_DIR)/$(GO_LICENSES_PATH)" "$(GO)" install "github.com/google/go-licenses@$(GO_LICENSES_VERSION)"

$(GINKGO_BIN):
@echo "installing ginkgo"
@mkdir -p "$(TOOLS_DIR)"
@GOBIN="$(BASE_DIR)/$(TOOLS_DIR)" "$(GO)" install "github.com/onsi/ginkgo/v2/ginkgo"
@echo "installing ginkgo $(GINKGO_VERSION)"
@mkdir -p "$(GINKGO_PATH)"
@GOBIN="$(BASE_DIR)/$(GINKGO_PATH)" "$(GO)" install "github.com/onsi/ginkgo/v2/ginkgo@$(GINKGO_VERSION)"

# Run lint against the previous commit for PR and branch build
# In dev setup look at all changes on top of master
Expand Down
9 changes: 6 additions & 3 deletions scripts/run-e2e-tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,12 @@
#limitations under the License.

TOOLS_DIRECTORY=tools
HELM=$TOOLS_DIRECTORY/helm
KIND=$TOOLS_DIRECTORY/kind
KUBECTL=$TOOLS_DIRECTORY/kubectl
HELM_VERSION=$(make -s print_helm_version)
KIND_VERSION=$(make -s print_kind_version)
KUBECTL_VERSION=$(make -s print_kubectl_version)
HELM=$TOOLS_DIRECTORY/helm-$HELM_VERSION/helm
KIND=$TOOLS_DIRECTORY/kind-$KIND_VERSION/kind
KUBECTL=$TOOLS_DIRECTORY/kubectl-$KUBECTL_VERSION/kubectl
GO="${GO:-go}"
export GO

Expand Down
Loading