[YUNIKORN-2135] Add integration test code coverage

Signed-off-by: PoAn Yang <[email protected]>

Makefile

@@ -44,6 +44,14 @@ endif
 # Make sure we are in the same directory as the Makefile
 BASE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
+# Set cover flags for go build and uid/gid for dockerfile
+ifeq ($(ENABLE_GO_COVER_DIR), TRUE)
+	DOCKERFILE_UID_ARG := --build-arg UID=0
+	DOCKERFILE_GID_ARG := --build-arg GID=0
+	COVER_FLAGS := -cover -coverpkg=github.com/apache/yunikorn-k8shim/...,github.com/apache/yunikorn-core/...
+	ADMISSION_COVER_FLAGS := -cover -coverpkg=github.com/apache/yunikorn-k8shim/...
+endif
+
 # Output directories
 OUTPUT=build
 DEV_BIN_DIR=${OUTPUT}/dev
@@ -338,7 +346,7 @@ $(RELEASE_BIN_DIR)/$(SCHEDULER_BINARY): go.mod go.sum $(shell find pkg)
 	-ldflags '-extldflags "-static" -X ${FLAG_PREFIX}.buildVersion=${VERSION} -X ${FLAG_PREFIX}.buildDate=${DATE} -X ${FLAG_PREFIX}.isPluginVersion=false -X ${FLAG_PREFIX}.goVersion=${GO_VERSION} -X ${FLAG_PREFIX}.arch=${EXEC_ARCH} -X ${FLAG_PREFIX}.coreSHA=${CORE_SHA} -X ${FLAG_PREFIX}.siSHA=${SI_SHA} -X ${FLAG_PREFIX}.shimSHA=${SHIM_SHA}' \
 	-tags netgo \
 	-installsuffix netgo \
-	./pkg/cmd/shim/
+	${COVER_FLAGS} ./pkg/cmd/shim/
 
 # Build plugin binary in a production ready version
 .PHONY: plugin
@@ -354,7 +362,7 @@ $(RELEASE_BIN_DIR)/$(PLUGIN_BINARY): go.mod go.sum $(shell find pkg)
 	-ldflags '-extldflags "-static" -X ${FLAG_PREFIX}.buildVersion=${VERSION} -X ${FLAG_PREFIX}.buildDate=${DATE} -X ${FLAG_PREFIX}.isPluginVersion=true -X ${FLAG_PREFIX}.goVersion=${GO_VERSION} -X ${FLAG_PREFIX}.arch=${EXEC_ARCH} -X ${FLAG_PREFIX}.coreSHA=${CORE_SHA} -X ${FLAG_PREFIX}.siSHA=${SI_SHA} -X ${FLAG_PREFIX}.shimSHA=${SHIM_SHA}' \
 	-tags netgo \
 	-installsuffix netgo \
-	./pkg/cmd/schedulerplugin/
+	${COVER_FLAGS} ./pkg/cmd/schedulerplugin/
 
 # Build a scheduler image based on the production ready version
 .PHONY: sched_image
@@ -364,7 +372,7 @@ sched_image: scheduler docker/scheduler
 	@mkdir -p "$(DOCKER_DIR)/scheduler"
 	@cp -a "docker/scheduler/." "$(DOCKER_DIR)/scheduler/."
 	@cp "$(RELEASE_BIN_DIR)/$(SCHEDULER_BINARY)" "$(DOCKER_DIR)/scheduler/."
-	DOCKER_BUILDKIT=1 docker build \
+	DOCKER_BUILDKIT=1 docker build ${DOCKERFILE_UID_ARG} ${DOCKERFILE_GID_ARG} \
 	"$(DOCKER_DIR)/scheduler" \
 	-t "$(SCHEDULER_TAG)" \
 	--platform "linux/${DOCKER_ARCH}" \
@@ -384,7 +392,7 @@ plugin_image: plugin docker/plugin conf/scheduler-config.yaml
 	@cp -a "docker/plugin/." "$(DOCKER_DIR)/plugin/."
 	@cp "$(RELEASE_BIN_DIR)/$(PLUGIN_BINARY)" "$(DOCKER_DIR)/plugin/."
 	@cp conf/scheduler-config.yaml "$(DOCKER_DIR)/plugin/scheduler-config.yaml"
-	DOCKER_BUILDKIT=1 docker build \
+	DOCKER_BUILDKIT=1 docker build ${DOCKERFILE_UID_ARG} ${DOCKERFILE_GID_ARG} \
 	"$(DOCKER_DIR)/plugin" \
 	-t "$(PLUGIN_TAG)" \
 	--platform "linux/${DOCKER_ARCH}" \
@@ -409,7 +417,7 @@ $(RELEASE_BIN_DIR)/$(ADMISSION_CONTROLLER_BINARY): go.mod go.sum $(shell find pk
 	-ldflags '-extldflags "-static" -X ${FLAG_PREFIX}.buildVersion=${VERSION} -X ${FLAG_PREFIX}.buildDate=${DATE} -X ${FLAG_PREFIX}.goVersion=${GO_VERSION} -X ${FLAG_PREFIX}.arch=${EXEC_ARCH}' \
 	-tags netgo \
 	-installsuffix netgo \
-	./pkg/cmd/admissioncontroller
+	${ADMISSION_COVER_FLAGS} ./pkg/cmd/admissioncontroller
 
 # Build an admission controller image based on the production ready version
 .PHONY: adm_image
@@ -419,7 +427,7 @@ adm_image: admission docker/admission
 	@mkdir -p "$(DOCKER_DIR)/admission"
 	@cp -a "docker/admission/." "$(DOCKER_DIR)/admission/."
 	@cp "$(RELEASE_BIN_DIR)/$(ADMISSION_CONTROLLER_BINARY)" "$(DOCKER_DIR)/admission/."
-	DOCKER_BUILDKIT=1 docker build \
+	DOCKER_BUILDKIT=1 docker build ${DOCKERFILE_UID_ARG} ${DOCKERFILE_GID_ARG} \
 	"$(DOCKER_DIR)/admission" \
 	-t "$(ADMISSION_TAG)" \
 	--platform "linux/${DOCKER_ARCH}" \

docker/admission/Dockerfile

@@ -14,7 +14,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+ARG UID=4444
+ARG GID=4444
+
 FROM --platform=$TARGETPLATFORM scratch
 COPY --chown=0:0 yunikorn-admission-controller /
-USER 4444:4444
+USER ${UID}:${GID}
 ENTRYPOINT [ "/yunikorn-admission-controller" ]

docker/plugin/Dockerfile

@@ -14,7 +14,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+ARG UID=4444
+ARG GID=4444
+
 FROM --platform=$TARGETPLATFORM scratch
 COPY --chown=0:0 yunikorn-scheduler-plugin scheduler-config.yaml /
-USER 4444:4444
+USER ${UID}:${GID}
 ENTRYPOINT [ "/yunikorn-scheduler-plugin", "--bind-address=0.0.0.0", "--config=/scheduler-config.yaml" ]

docker/scheduler/Dockerfile

@@ -14,7 +14,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+ARG UID=4444
+ARG GID=4444
+
 FROM --platform=$TARGETPLATFORM scratch
 COPY --chown=0:0 yunikorn-scheduler /
-USER 4444:4444
+USER ${UID}:${GID}
 ENTRYPOINT [ "/yunikorn-scheduler" ]

scripts/run-e2e-tests.sh

@@ -87,7 +87,8 @@ function install_cluster() {
   if [ "${GIT_CLONE}" = "true" ]; then
     check_cmd "git"
     rm -rf ./build/yunikorn-release
-    git clone --depth 1 https://github.com/apache/yunikorn-release.git ./build/yunikorn-release
+    # TODO: update following branch if PR in yunikorn-release is merged
+    git clone --depth 1 https://github.com/FrankYang0529/yunikorn-release.git -b YUNIKORN-2135 ./build/yunikorn-release
   fi
   if [ ! -d "${CHART_PATH}" ]; then
     exit_on_error "helm charts not found in path: ${CHART_PATH}"
@@ -96,7 +97,7 @@ function install_cluster() {
   # build docker images from latest code, so that we can install yunikorn with these latest images
   echo "step 3/7: building docker images from latest code"
   check_docker
-  QUIET="--quiet" REGISTRY=local VERSION=latest make image
+  QUIET="--quiet" REGISTRY=local VERSION=latest ENABLE_GO_COVER_DIR=TRUE make image
   exit_on_error "build docker images failed"
   QUIET="--quiet" REGISTRY=local VERSION=latest make webtest_image
   exit_on_error "build test web images failed"
@@ -136,14 +137,43 @@ function install_cluster() {
     --set admissionController.image.pullPolicy=IfNotPresent \
     --set web.image.repository=local/yunikorn \
     --set web.image.tag="${WEBTEST_IMAGE}" \
-    --set web.image.pullPolicy=IfNotPresent
+    --set web.image.pullPolicy=IfNotPresent \
+    --set enableGoCoverDir=true
   exit_on_error "failed to install yunikorn"
   "${KUBECTL}" wait --for=condition=available --timeout=300s deployment/yunikorn-scheduler -n yunikorn
   exit_on_error "failed to wait for yunikorn scheduler deployment being deployed"
   "${KUBECTL}" wait --for=condition=ready --timeout=300s pod -l app=yunikorn -n yunikorn
   exit_on_error "failed to wait for yunikorn scheduler pods being deployed"
 }
 
+function uninstall_yunikorn() {
+  echo "uninstall yunikorn"
+  install_tools
+  "${HELM}" uninstall yunikorn --namespace yunikorn --wait --cascade foreground
+  exit_on_error "failed to uninstall yunikorn"
+}
+
+function copy_go_cover_dir() {
+  echo "copy coverage profile files from kind containers to local go-cover-dir-merged directory"
+  install_tools
+  mkdir -p go-cover-dir-merged
+  GO_COVER_DIR_FOLDERS=()
+  for i in $("${KIND}" get nodes --name "${CLUSTER_NAME}"); do
+    if docker exec -it $i test -d /go-cover-dir
+    then
+      docker cp $i:/go-cover-dir $i
+      GO_COVER_DIR_FOLDERS+=($i)
+    fi
+  done
+
+  if [ ${#GO_COVER_DIR_FOLDERS[@]} -ne 0 ]; then
+    GO_COVER_DIR_STR=$(printf ",%s" "${GO_COVER_DIR_FOLDERS[@]}")
+    GO_COVER_DIR_STR=${GO_COVER_DIR_STR:1}
+    go tool covdata merge -i=${GO_COVER_DIR_STR} -o go-cover-dir-merged
+    rm -rf $(printf " %s" "${GO_COVER_DIR_FOLDERS[@]}")
+  fi
+}
+
 function delete_cluster() {
   echo "deleting K8s cluster: ${CLUSTER_NAME}"
   install_tools
@@ -269,6 +299,8 @@ if [ "${ACTION}" == "test" ]; then
   fi
   make e2e_test
   exit_on_error "e2e tests failed"
+  uninstall_yunikorn
+  copy_go_cover_dir
 elif [ "${ACTION}" == "install" ]; then
   check_cmd "${GO}"
   check_opt "kind-node-image-version" "${CLUSTER_VERSION}"