From 3ab5b6aafd635e47ae07e63f9035dd3f0fcdff32 Mon Sep 17 00:00:00 2001 From: "tao.yang" Date: Mon, 13 May 2024 09:57:41 +0800 Subject: [PATCH] optimizate upgrade Signed-off-by: ty-dc --- .github/workflows/auto-upgrade-ci.yaml | 63 ++++++++++++++++++++------ test/Makefile | 44 +++++++++++++----- test/e2e/annotation/annotation_test.go | 8 ++-- test/scripts/debugEnv.sh | 6 +-- 4 files changed, 89 insertions(+), 32 deletions(-) diff --git a/.github/workflows/auto-upgrade-ci.yaml b/.github/workflows/auto-upgrade-ci.yaml index 95aefe7eff..a99d420f05 100644 --- a/.github/workflows/auto-upgrade-ci.yaml +++ b/.github/workflows/auto-upgrade-ci.yaml @@ -9,6 +9,11 @@ env: on: schedule: - cron: "0 20 * * *" + pull_request_target: + types: + - opened + - synchronize + - reopened workflow_call: inputs: dest_tag: @@ -74,14 +79,33 @@ jobs: echo "LATEST_RELEASE_VERISON: ${LATEST_RELEASE_VERISON} " echo "OLD_VERSION=${LATEST_RELEASE_VERISON}" >> $GITHUB_ENV fi + elif ${{ github.event_name == 'push' }} ; then + echo "trigger by push" + echo "NEW_VERSION=${{ github.sha }}" >> $GITHUB_ENV + # for PR scenarios, the latest version of the main branch will be used as old-version + echo "OLD_VERSION=main" >> $GITHUB_ENV + echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV + elif ${{ github.event_name == 'pull_request_target' }} ; then + echo "trigger by pull_request_target" + echo "NEW_VERSION=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV + # for PR scenarios, the latest version of the main branch will be used as old-version + echo "OLD_VERSION=main" >> $GITHUB_ENV + echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV else # schedule event # use main sha for ci image tag echo "trigger by schedule" echo "RUN_TAG=main" >> $GITHUB_ENV echo "NEW_VERSION=main" >> $GITHUB_ENV - LATEST_RELEASE_VERISON=$(curl -s https://api.github.com/repos/spidernet-io/spiderpool/releases | grep '"tag_name":' | grep -Eo "v([0-9]+\.[0-9]+\.[0-9])" | sort -r | head -n 1) - echo "OLD_VERSION=${LATEST_RELEASE_VERISON}" >> $GITHUB_ENV + LATEST_RELEASE_VERISON=$(curl -s https://api.github.com/repos/spidernet-io/spiderpool/releases | grep '"tag_name":' | sort -r | head -n 1) + rc_version=$(grep -Eo 'v[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]' <<< ${LATEST_RELEASE_VERISON}) + if [ -z ${rc_version} ]; then + echo "the latest released version is not an 'rc: ${LATEST_RELEASE_VERISON}' version." + echo "OLD_VERSION=$(grep -Eo "v([0-9]+\.[0-9]+\.[0-9])" <<< ${LATEST_RELEASE_VERISON})" >> $GITHUB_ENV + else + echo "The latest released version is an 'rc: ${LATEST_RELEASE_VERISON}' version." + echo "OLD_VERSION=${rc_version}" >> $GITHUB_ENV + fi echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV fi @@ -89,6 +113,7 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false + fetch-depth: 0 ref: ${{ env.RUN_TAG }} - name: Result Ref @@ -139,14 +164,18 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - ref: ${{ needs.get_ref.outputs.old_version }} + ref: ${{ needs.get_ref.outputs.new_version }} - name: Prepare id: prepare run: | echo "ref: ${{ inputs.ref }} " echo "===== image " - echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-controller:${{ needs.get_ref.outputs.old_version }}" + if ${{ needs.get_ref.outputs.old_version == 'main' }} ; then + echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-controller:latest" + else + echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-controller:${{ needs.get_ref.outputs.old_version }}" + fi TMP=` date +%m%d%H%M%S ` E2E_CLUSTER_NAME="spiderpool${TMP}" echo "E2E_CLUSTER_NAME=${E2E_CLUSTER_NAME}" >> $GITHUB_ENV @@ -177,9 +206,20 @@ jobs: echo "restart docker before trying again" systemctl restart docker command: | + E2E_SPIDERPOOL_TAG="" + if ${{ needs.get_ref.outputs.old_version == 'main' }} ; then + E2E_SPIDERPOOL_TAG=latest + else + E2E_SPIDERPOOL_TAG=${{ needs.get_ref.outputs.old_version }} + fi make e2e_init_underlay -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }} \ - -e E2E_SPIDERPOOL_TAG=${{ needs.get_ref.outputs.old_version }} \ - -e PYROSCOPE_LOCAL_PORT="" + -e E2E_SPIDERPOOL_TAG=${E2E_SPIDERPOOL_TAG} \ + -e PYROSCOPE_LOCAL_PORT="" \ + -e INSTALL_KUBEVIRT=true \ + -e INSTALL_KRUISE=true \ + -e INSTALL_KDOCTOR=true \ + -e INSTALL_RDMA=true \ + -e INSTALL_SRIOV=true - name: backup kubeconfig from olderVersion ${{ needs.get_ref.outputs.old_version }} run: | @@ -223,15 +263,10 @@ jobs: id: upgrade continue-on-error: true run: | - make upgrade_e2e_spiderpool -e E2E_SPIDERPOOL_TAG=${{ needs.call_build_ci_image.outputs.imageTag }} \ + make upgrade_e2e_spiderpool -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }} \ + -e E2E_SPIDERPOOL_TAG=${{ needs.call_build_ci_image.outputs.imageTag }} \ -e SPIDERPOOL_AGENT_IMAGE_NAME=spiderpool-agent-race \ - -e SPIDERPOOL_CONTROLLER_IMAGE_NAME=spiderpool-controller-race \ - -e INSTALL_KUBEVIRT=true \ - -e INSTALL_KRUISE=true \ - -e INSTALL_KDOCTOR=true \ - -e INSTALL_OVS=${INSTALL_OVS_VALUE} \ - -e INSTALL_RDMA=true \ - -e INSTALL_SRIOV=true + -e SPIDERPOOL_CONTROLLER_IMAGE_NAME=spiderpool-controller-race - name: Run e2e Test on ${{ needs.get_ref.outputs.ref }} id: run_e2e diff --git a/test/Makefile b/test/Makefile index bd931e3c0c..1e4f36a187 100644 --- a/test/Makefile +++ b/test/Makefile @@ -403,20 +403,42 @@ helm_upgrade_spiderpool: kubectl delete po -n $(RELEASE_NAMESPACE) spiderpool-init --kubeconfig $(E2E_KUBECONFIG) || true ;\ HELM_OPTION="";\ HELM_OPTION+=" --set spiderpoolController.replicas=1 " ; \ + if [ "$(INSTALL_OVERLAY_CNI)" == "true" ]; then \ + HELM_OPTION+=" --set multus.multusCNI.defaultCniCRName= " ; \ + else \ + HELM_OPTION+=" --set multus.multusCNI.defaultCniCRName=$(MULTUS_DEFAULT_CNI_VLAN0) " ; \ + fi ; \ HELM_OPTION+=" --set spiderpoolAgent.image.registry="" \ - --set spiderpoolAgent.image.repository=$(SPIDERPOOL_AGENT_IMAGE_NAME) \ - --set spiderpoolAgent.image.tag=$(E2E_SPIDERPOOL_TAG) \ - --set spiderpoolController.image.registry="" \ - --set spiderpoolController.image.repository=$(SPIDERPOOL_CONTROLLER_IMAGE_NAME) \ - --set spiderpoolController.image.tag=$(E2E_SPIDERPOOL_TAG) \ - --set spiderpoolInit.image.registry="" \ - --set spiderpoolInit.image.repository=$(SPIDERPOOL_CONTROLLER_IMAGE_NAME) \ - --set spiderpoolInit.image.tag=$(E2E_SPIDERPOOL_TAG) " \ - helm --kubeconfig $(E2E_KUBECONFIG) upgrade $(RELEASE_NAME) $(ROOT_DIR)/charts/spiderpool $${HELM_OPTION} \ - -n $(RELEASE_NAMESPACE) ; \ + --set spiderpoolAgent.image.repository=$(SPIDERPOOL_AGENT_IMAGE_NAME) \ + --set spiderpoolAgent.image.tag=$(E2E_SPIDERPOOL_TAG) \ + --set spiderpoolController.image.registry="" \ + --set spiderpoolController.image.repository=$(SPIDERPOOL_CONTROLLER_IMAGE_NAME) \ + --set spiderpoolController.image.tag=$(E2E_SPIDERPOOL_TAG) \ + --set spiderpoolInit.image.registry="" \ + --set spiderpoolInit.image.repository=$(SPIDERPOOL_CONTROLLER_IMAGE_NAME) \ + --set spiderpoolInit.image.tag=$(E2E_SPIDERPOOL_TAG) \ + --set multus.multusCNI.uninstall=false " \ + ALL_IMAGES=`helm template $(RELEASE_NAME) $(ROOT_DIR)/charts/spiderpool $${HELM_OPTION} | grep ' image: ' | tr -d '"' | awk -F 'image: ' '{print $$2}' | sort | uniq | tr '\n' ' '` ; \ + echo "ALL_IMAGES: $${ALL_IMAGES} " ; \ + for IMAGE in $${ALL_IMAGES}; do \ + if ! grep "$${IMAGE}" <<< `docker images | awk '{printf("%s:%s\n",$$1,$$2)}'`; then \ + echo "==> $${IMAGE} no found, pulling...." ; \ + docker pull $${IMAGE} ; \ + fi ; \ + kind load docker-image $${IMAGE} --name $(E2E_CLUSTER_NAME); \ + done ; \ + echo "upgrade spiderpool with image $(SPIDERPOOL_AGENT_IMAGE_NAME):$(E2E_SPIDERPOOL_TAG) and $(SPIDERPOOL_CONTROLLER_IMAGE_NAME):$(E2E_SPIDERPOOL_TAG) " ; \ + set -x ; \ + helm --kubeconfig $(E2E_KUBECONFIG) upgrade $(RELEASE_NAME) $(ROOT_DIR)/charts/spiderpool \ + $${HELM_OPTION} \ + -n $(RELEASE_NAMESPACE) --debug --reuse-values ; \ cd $(ROOT_DIR)/charts/spiderpool/crds ; \ ls | grep '\.yaml$$' | xargs -I {} kubectl apply -f {} --kubeconfig $(E2E_KUBECONFIG) ; \ - kubectl wait --for=condition=ready -l app.kubernetes.io/instance=spiderpool --timeout=300s pod -n kube-system --kubeconfig $(E2E_KUBECONFIG) || true; \ + kubectl wait --for=condition=ready -l app.kubernetes.io/instance=spiderpool --timeout=300s pod -n $(RELEASE_NAMESPACE) --kubeconfig $(E2E_KUBECONFIG) || true; \ + kubectl scale deploy -n $(RELEASE_NAMESPACE) -l app.kubernetes.io/component=spiderpool-controller --replicas=2 --kubeconfig $(E2E_KUBECONFIG); \ + kubectl get po -n $(RELEASE_NAMESPACE) -l app.kubernetes.io/instance=spiderpool -oyaml --kubeconfig $(E2E_KUBECONFIG) | grep image ; \ + kubectl wait --for=condition=ready -l app.kubernetes.io/component=spiderpool-controller --timeout=300s pod -n $(RELEASE_NAMESPACE) --kubeconfig $(E2E_KUBECONFIG) || true; \ + helm --kubeconfig $(E2E_KUBECONFIG) list -A ; \ .PHONY: clean clean: diff --git a/test/e2e/annotation/annotation_test.go b/test/e2e/annotation/annotation_test.go index 523f01db50..f36531701c 100644 --- a/test/e2e/annotation/annotation_test.go +++ b/test/e2e/annotation/annotation_test.go @@ -351,10 +351,10 @@ var _ = Describe("test annotation", Label("annotation"), func() { podIppoolAnnoStr = common.GeneratePodIPPoolAnnotations(frame, common.NIC1, globalDefaultV4IpoolList, globalDefaultV6IpoolList) var tmpV4PoolNameList, tmpV6PoolNameList []string if frame.Info.IpV4Enabled { - tmpV4PoolNameList = []string{fmt.Sprintf("%s*", v4PoolNameList[0])} + tmpV4PoolNameList = []string{v4PoolNameList[0]} } if frame.Info.IpV6Enabled { - tmpV6PoolNameList = []string{fmt.Sprintf("%s*", v6PoolNameList[0])} + tmpV6PoolNameList = []string{v6PoolNameList[0]} } podIppoolsAnnoStr = common.GeneratePodIPPoolsAnnotations(frame, common.NIC1, cleanGateway, tmpV4PoolNameList, tmpV6PoolNameList) GinkgoWriter.Printf("Annotation '%s' value is '%s'\n", pkgconstant.AnnoPodIPPools, podIppoolsAnnoStr) @@ -424,11 +424,11 @@ var _ = Describe("test annotation", Label("annotation"), func() { namespaceObject.Annotations = make(map[string]string) if frame.Info.IpV4Enabled { v4IppoolAnnoValue := types.AnnoNSDefautlV4PoolValue{} - common.SetNamespaceIppoolAnnotation(v4IppoolAnnoValue, namespaceObject, []string{fmt.Sprintf("%s*", v4PoolName)}, pkgconstant.AnnoNSDefautlV4Pool) + common.SetNamespaceIppoolAnnotation(v4IppoolAnnoValue, namespaceObject, []string{v4PoolName}, pkgconstant.AnnoNSDefautlV4Pool) } if frame.Info.IpV6Enabled { v6IppoolAnnoValue := types.AnnoNSDefautlV6PoolValue{} - common.SetNamespaceIppoolAnnotation(v6IppoolAnnoValue, namespaceObject, []string{fmt.Sprintf("%s*", v6PoolName)}, pkgconstant.AnnoNSDefautlV6Pool) + common.SetNamespaceIppoolAnnotation(v6IppoolAnnoValue, namespaceObject, []string{v6PoolName}, pkgconstant.AnnoNSDefautlV6Pool) } GinkgoWriter.Printf("Generate namespace objects: %v with namespace annotations \n", namespaceObject) diff --git a/test/scripts/debugEnv.sh b/test/scripts/debugEnv.sh index 385dfb5926..dece778df2 100755 --- a/test/scripts/debugEnv.sh +++ b/test/scripts/debugEnv.sh @@ -174,8 +174,8 @@ elif [ "$TYPE"x == "detail"x ] ; then kubectl get spidermultusconfig -A -o wide --kubeconfig ${E2E_KUBECONFIG} echo "" - echo "--------- kubectl get spidermultusconfig -o json" - kubectl get spidermultusconfig -o json --kubeconfig ${E2E_KUBECONFIG} + echo "--------- kubectl get spidermultusconfig -A -o json" + kubectl get spidermultusconfig -A -o json --kubeconfig ${E2E_KUBECONFIG} echo "" echo "--------- kubectl get network-attachment-definitions.k8s.cni.cncf.io -A -o wide" @@ -183,7 +183,7 @@ elif [ "$TYPE"x == "detail"x ] ; then echo "" echo "--------- kubectl get network-attachment-definitions.k8s.cni.cncf.io -A -o json" - kubectl get network-attachment-definitions.k8s.cni.cncf.io -o json --kubeconfig ${E2E_KUBECONFIG} + kubectl get network-attachment-definitions.k8s.cni.cncf.io -A -o json --kubeconfig ${E2E_KUBECONFIG} echo "" echo "--------- kubectl get configmaps -n kube-system spiderpool-conf -ojson"