Skip to content

Commit

Permalink
Merge branch 'master' into YUNIKORN-2563
Browse files Browse the repository at this point in the history
  • Loading branch information
@pbacsko
pbacsko authored Apr 22, 2024
2 parents 8131831 + 04daf62 commit 6ff47cb
Show file tree
Hide file tree
Showing 4 changed files with 35 additions and 32 deletions.
10 changes: 9 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,9 @@ export SPARK_PYTHON_IMAGE=docker.io/apache/spark-py:v$(SPARK_PYTHON_VERSION)
GO_LICENSES_VERSION=v1.6.0
GO_LICENSES_BIN=$(TOOLS_DIR)/go-licenses

# ginkgo
GINKGO_BIN=$(TOOLS_DIR)/ginkgo

FLAG_PREFIX=github.com/apache/yunikorn-k8shim/pkg/conf

# Image hashes
Expand Down Expand Up @@ -219,7 +222,7 @@ conf/scheduler-config-local.yaml: conf/scheduler-config.yaml

# Install tools
.PHONY: tools
tools: $(SHELLCHECK_BIN) $(GOLANGCI_LINT_BIN) $(KUBECTL_BIN) $(KIND_BIN) $(HELM_BIN) $(SPARK_SUBMIT_CMD) $(GO_LICENSES_BIN)
tools: $(SHELLCHECK_BIN) $(GOLANGCI_LINT_BIN) $(KUBECTL_BIN) $(KIND_BIN) $(HELM_BIN) $(SPARK_SUBMIT_CMD) $(GO_LICENSES_BIN) $(GINKGO_BIN)

# Install shellcheck
$(SHELLCHECK_BIN):
Expand Down Expand Up @@ -273,6 +276,11 @@ $(GO_LICENSES_BIN):
@mkdir -p "$(TOOLS_DIR)"
@GOBIN="$(BASE_DIR)/$(TOOLS_DIR)" "$(GO)" install "github.com/google/go-licenses@$(GO_LICENSES_VERSION)"

$(GINKGO_BIN):
@echo "installing ginkgo"
@mkdir -p "$(TOOLS_DIR)"
@GOBIN="$(BASE_DIR)/$(TOOLS_DIR)" "$(GO)" install "github.com/onsi/ginkgo/v2/ginkgo"

# Run lint against the previous commit for PR and branch build
# In dev setup look at all changes on top of master
.PHONY: lint
Expand Down
16 changes: 8 additions & 8 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ module github.com/apache/yunikorn-k8shim
go 1.21

require (
github.com/apache/yunikorn-core v0.0.0-20240417112924-e89458fa3cc4
github.com/apache/yunikorn-scheduler-interface v0.0.0-20240402211642-e7421a4261fd
github.com/apache/yunikorn-core v0.0.0-20240422073303-db13a4fa8585
github.com/apache/yunikorn-scheduler-interface v0.0.0-20240422062544-b70081933c38
github.com/google/go-cmp v0.6.0
github.com/google/uuid v1.6.0
github.com/looplab/fsm v1.0.1
Expand Down Expand Up @@ -122,13 +122,13 @@ require (
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.19.0 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/oauth2 v0.12.0 // indirect
golang.org/x/sync v0.6.0 // indirect
golang.org/x/sys v0.17.0 // indirect
golang.org/x/term v0.17.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/term v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.17.0 // indirect
Expand Down Expand Up @@ -164,10 +164,10 @@ require (
replace (
github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.12
github.com/petermattis/goid => github.com/petermattis/goid v0.0.0-20240327183114-c42a807a84ba
golang.org/x/crypto => golang.org/x/crypto v0.19.0
golang.org/x/crypto => golang.org/x/crypto v0.21.0
golang.org/x/lint => golang.org/x/lint v0.0.0-20210508222113-6edffad5e616
golang.org/x/net => golang.org/x/net v0.21.0
golang.org/x/sys => golang.org/x/sys v0.17.0
golang.org/x/net => golang.org/x/net v0.23.0
golang.org/x/sys => golang.org/x/sys v0.18.0
golang.org/x/text => golang.org/x/text v0.14.0
golang.org/x/tools => golang.org/x/tools v0.17.0
google.golang.org/protobuf => google.golang.org/protobuf v1.33.0
Expand Down
24 changes: 12 additions & 12 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@ github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cq
github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18=
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM=
github.com/apache/yunikorn-core v0.0.0-20240417112924-e89458fa3cc4 h1:nnZKWKycGVcIBVmQsIM4MHO0TwLfZ6wS09dMI4Tvl7o=
github.com/apache/yunikorn-core v0.0.0-20240417112924-e89458fa3cc4/go.mod h1:DnScYvh1qQ7v89tebVH43LcuEDoUXLy2wm8aE4Co75Y=
github.com/apache/yunikorn-scheduler-interface v0.0.0-20240402211642-e7421a4261fd h1:uNOijHkCotZLUZ+A85NSftEJGfP50Opf7ms6Daj6pco=
github.com/apache/yunikorn-scheduler-interface v0.0.0-20240402211642-e7421a4261fd/go.mod h1:0f4l3ManMROX60xU7GbhejCEYYyMksH275oY2xIVkbM=
github.com/apache/yunikorn-core v0.0.0-20240422073303-db13a4fa8585 h1:R8ugyStoQzlZra+up+UVxLlxW59O/bp7jw1S97xo61U=
github.com/apache/yunikorn-core v0.0.0-20240422073303-db13a4fa8585/go.mod h1:Zh5AYAA1Sskzak8F4xikX0gp8nub6rhuh8Trfnr1Es8=
github.com/apache/yunikorn-scheduler-interface v0.0.0-20240422062544-b70081933c38 h1:/02cjuc0xpQPZIGezL45QZ6muGI7dfesu9l38U9fbx0=
github.com/apache/yunikorn-scheduler-interface v0.0.0-20240422062544-b70081933c38/go.mod h1:WuHJpVk34t8N5+1ErYGj/5Qq33/cRzL4YtuoAsbMtWc=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
Expand Down Expand Up @@ -268,23 +268,23 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA=
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
Expand Down
17 changes: 6 additions & 11 deletions scripts/run-e2e-tests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,12 @@ function install_tools() {
}

function install_cluster() {
echo "step 1/7: checking required configuration"
echo "step 1/6: checking required configuration"
if [ ! -r "${KIND_CONFIG}" ]; then
exit_on_error "kind config not found: ${KIND_CONFIG}"
fi

echo "step 2/7: install tools"
echo "step 2/6: install tools"
install_tools

# use latest helm charts from the release repo to install yunikorn unless path is provided
Expand All @@ -95,20 +95,15 @@ function install_cluster() {
fi

# build docker images from latest code, so that we can install yunikorn with these latest images
echo "step 3/7: building docker images from latest code"
echo "step 3/6: building docker images from latest code"
check_docker
QUIET="--quiet" REGISTRY=local VERSION=latest make image
exit_on_error "build docker images failed"
QUIET="--quiet" REGISTRY=local VERSION=latest make webtest_image
exit_on_error "build test web images failed"

# install ginkgo and gomega for e2e tests.
echo "step 4/7: installing Ginkgo & Gomega at $("${GO}" env GOPATH)/bin"
"${GO}" install github.com/onsi/ginkgo/v2/ginkgo
check_cmd "ginkgo"

# create K8s cluster
echo "step 5/7: installing K8s cluster using kind"
echo "step 4/6: installing K8s cluster using kind"
"${KIND}" create cluster --name "${CLUSTER_NAME}" --image "${CLUSTER_VERSION}" --config="${KIND_CONFIG}"
exit_on_error "install K8s cluster failed"
"${KUBECTL}" cluster-info --context kind-"${CLUSTER_NAME}"
Expand All @@ -119,15 +114,15 @@ function install_cluster() {
"${KUBECTL}" describe nodes

# pre-load yunikorn docker images to kind
echo "step 6/7: pre-load yunikorn images"
echo "step 5/6: pre-load yunikorn images"
"${KIND}" load docker-image "local/yunikorn:${SCHEDULER_IMAGE}" --name "${CLUSTER_NAME}"
exit_on_error "pre-load scheduler image failed: ${SCHEDULER_IMAGE}"
"${KIND}" load docker-image "local/yunikorn:${ADMISSION_IMAGE}" --name "${CLUSTER_NAME}"
exit_on_error "pre-load admission controller image failed: ${ADMISSION_IMAGE}"
"${KIND}" load docker-image "local/yunikorn:${WEBTEST_IMAGE}" --name "${CLUSTER_NAME}"
exit_on_error "pre-load web image failed: ${WEBTEST_IMAGE}"

echo "step 7/7: installing yunikorn"
echo "step 6/6: installing yunikorn"
"${HELM}" install yunikorn "${CHART_PATH}" --namespace yunikorn \
--set image.repository=local/yunikorn \
--set image.tag="${SCHEDULER_IMAGE}" \
Expand Down

0 comments on commit 6ff47cb

Please sign in to comment.