Optimize upgrade CI, and use the self-built image in the release branch

.github/workflows/auto-upgrade-ci.yaml

@@ -49,6 +49,7 @@ jobs:
       e2e_enabled: ${{ env.RUN_E2E_ENABLED }}
       run_upgrade_enabled: ${{ env.RUN_UPGRADE_ENABLED }}
       skip_create_pr: ${{ env.SKIP_CREATE_PR }}
+      build_old_image_tag: ${{ env.BUILD_OLD_IMAGE_TAG }}
     steps:
       - name: Check Code Changes
         uses: dorny/[email protected]
@@ -79,6 +80,8 @@ jobs:
             echo "OLD_VERSION=${{ github.event.inputs.old_version }}" >> $GITHUB_ENV
             echo "NEW_VERSION=${{ github.event.inputs.ref }}" >> $GITHUB_ENV
             echo "SKIP_CREATE_PR=true" >> $GITHUB_ENV
+            echo "RUN_UPGRADE_ENABLED=true" >> $GITHUB_ENV
+            echo "BUILD_OLD_IMAGE_TAG=false" >> $GITHUB_ENV
             if ${{ github.event.inputs.e2e_enabled == 'true' }}; then
               echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV
             else
@@ -88,8 +91,10 @@ jobs:
               echo "call by workflow_call"
               echo "RUN_TAG=${{ inputs.dest_tag }}" >> $GITHUB_ENV
               echo "NEW_VERSION=${{ inputs.dest_tag }}" >> $GITHUB_ENV
+              echo "RUN_UPGRADE_ENABLED=true" >> $GITHUB_ENV
               echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV
               echo "SKIP_CREATE_PR=true" >> $GITHUB_ENV
+              echo "BUILD_OLD_IMAGE_TAG=false" >> $GITHUB_ENV
               YBranchName=` grep -Eo "v[0-9]+\.[0-9]+" <<< "${{ inputs.dest_tag }}" `
               # Get the previous minor version of version y
               LATEST_RELEASE_VERISON=$(curl -s https://api.github.com/repos/spidernet-io/spiderpool/releases | grep '"tag_name":' | grep -Eo "v([0-9]+\.[0-9]+\.[0-9])" | grep ${YBranchName} | sort -r | head -n 1)
@@ -107,15 +112,23 @@ jobs:
             echo "RUN_TAG=${{ github.sha }}" >> $GITHUB_ENV
             echo "NEW_VERSION=${{ github.sha }}" >> $GITHUB_ENV
             # for PR scenarios, the latest version of the main branch will be used as old-version
-            echo "OLD_VERSION=main" >> $GITHUB_ENV
+            echo "OLD_VERSION: ${{ github.event.pull_request.base.sha }}"
             echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV
             echo "SKIP_CREATE_PR=false" >> $GITHUB_ENV
+            echo "BUILD_OLD_IMAGE_TAG=true" >> $GITHUB_ENV
+            if ${{ steps.filter_pr.outputs.run_upgrade == 'true' }} ; then
+                echo "RUN_UPGRADE_ENABLED=true" >> $GITHUB_ENV
+            else
+                echo "RUN_UPGRADE_ENABLED=false" >> $GITHUB_ENV
+            fi
           elif ${{ github.event_name == 'pull_request_target' }} ; then
             echo "trigger by pull_request_target"
             echo "RUN_TAG=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
             echo "NEW_VERSION=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
             # for PR scenarios, the latest version of the main branch will be used as old-version
-            echo "OLD_VERSION=main" >> $GITHUB_ENV
+            echo "BUILD_OLD_IMAGE_TAG=true" >> $GITHUB_ENV
+            echo "OLD_VERSION=${{ github.event.pull_request.base.sha }}" >> $GITHUB_ENV
+            echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV
             echo "SKIP_CREATE_PR=true" >> $GITHUB_ENV
             if ${{ steps.filter_pr.outputs.run_upgrade == 'true' }} ; then
                 echo "RUN_UPGRADE_ENABLED=true" >> $GITHUB_ENV
@@ -128,6 +141,9 @@ jobs:
             echo "trigger by schedule"
             echo "RUN_TAG=main" >> $GITHUB_ENV
             echo "NEW_VERSION=main" >> $GITHUB_ENV
+            echo "RUN_UPGRADE_ENABLED=true" >> $GITHUB_ENV
+            echo "BUILD_OLD_IMAGE_TAG=false" >> $GITHUB_ENV
+            echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV
             LATEST_RELEASE_VERISON=$(curl -s https://api.github.com/repos/spidernet-io/spiderpool/releases | grep '"tag_name":' | sort -r | head -n 1)
             rc_version=$(grep -Eo 'v[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]' <<< ${LATEST_RELEASE_VERISON})
             echo "SKIP_CREATE_PR=false" >> $GITHUB_ENV
@@ -138,7 +154,6 @@ jobs:
               echo "The latest released version is an 'rc: ${LATEST_RELEASE_VERISON}' version."
               echo "OLD_VERSION=${rc_version}" >> $GITHUB_ENV
             fi
-            echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV
           fi
 
       - name: Checkout code
@@ -154,34 +169,28 @@ jobs:
           ref=$( git show -s --format='format:%H')
           echo "RUN_REF=${ref}" >> $GITHUB_ENV
 
-  call_build_ci_image:
+  call_build_old_ci_image:
     needs: [get_ref]
     if: ${{ needs.get_ref.outputs.run_upgrade_enabled == 'true' }}
     uses: ./.github/workflows/build-image-ci.yaml
     with:
-      ref: ${{ needs.get_ref.outputs.ref }}
+      ref: ${{ needs.get_ref.outputs.old_version }}
       push: false
+      imageTarName: old-image-tar
     secrets: inherit
 
-  lint_chart_against_release_image:
-    needs: get_ref
-    if: ${{ needs.get_ref.outputs.run_upgrade_enabled == 'true' }}
-    uses: ./.github/workflows/call-lint-chart.yaml
-    with:
-      ref: ${{ needs.get_ref.outputs.ref }}
-    secrets: inherit
-
-  trivy_scan_images:
-    needs: [call_build_ci_image, get_ref]
+  call_build_new_ci_image:
+    needs: [get_ref]
     if: ${{ needs.get_ref.outputs.run_upgrade_enabled == 'true' }}
-    uses: ./.github/workflows/trivy-scan-image.yaml
+    uses: ./.github/workflows/build-image-ci.yaml
     with:
-      image_tag: ${{ needs.call_build_ci_image.outputs.imageTag }}
-      ref: ${{ needs.get_ref.outputs.ref }}
+      ref: ${{ needs.get_ref.outputs.new_version }}
+      push: false
+      imageTarName: new-image-tar
     secrets: inherit
 
   run_upgrade_tests_on_release_version:
-    needs: [call_build_ci_image, get_ref]
+    needs: [get_ref, call_build_old_ci_image, call_build_new_ci_image]
     if: ${{ needs.get_ref.outputs.run_upgrade_enabled == 'true' }}
     runs-on: ubuntu-latest
     steps:
@@ -200,21 +209,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           persist-credentials: false
-          ref: ${{ needs.get_ref.outputs.new_version }}
-
-      - name: Prepare
-        id: prepare
-        run: |
-          echo "ref: ${{ inputs.ref }} "
-          echo "===== image "
-          if ${{ needs.get_ref.outputs.old_version == 'main' }} ; then
-            echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-controller:latest"
-          else
-            echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-controller:${{ needs.get_ref.outputs.old_version }}"
-          fi
-          TMP=` date +%m%d%H%M%S `
-          E2E_CLUSTER_NAME="spiderpool${TMP}"
-          echo "E2E_CLUSTER_NAME=${E2E_CLUSTER_NAME}" >> $GITHUB_ENV
+          ref: ${{ needs.get_ref.outputs.old_version }}
 
       - name: Setup Golang
         uses: actions/setup-go@v5
@@ -230,6 +225,41 @@ jobs:
         run: |
           bash ./test/scripts/install-tools.sh
 
+      - name: Download old spiderpool-agent image with tag ${{ needs.call_build_old_ci_image.outputs.imageTag }}
+        if: ${{ needs.get_ref.outputs.build_old_image_tag == 'true' }}
+        uses: actions/download-artifact@v3
+        with:
+          name: old-image-tar-spiderpool-agent
+          path: test/.download
+
+      - name: Download old spiderpool-controller image with tag ${{ needs.call_build_old_ci_image.outputs.imageTag }}
+        if: ${{ needs.get_ref.outputs.build_old_image_tag == 'true' }}
+        uses: actions/download-artifact@v3
+        with:
+          name: old-image-tar-spiderpool-controller
+          path: test/.download
+
+      - name: Load Images
+        if: ${{ needs.get_ref.outputs.build_old_image_tag == 'true' }}
+        run: |
+          TAR_FILES=` ls test/.download `
+          echo $TAR_FILES
+          for ITEM in $TAR_FILES ; do
+              IMAGE_NAME=${ITEM%*.tar}
+              echo ${IMAGE_NAME}
+              cat test/.download/${ITEM} | docker import - ${IMAGE_NAME}:${{ needs.call_build_old_ci_image.outputs.imageTag }}
+          done
+
+      - name: Prepare
+        id: prepare
+        run: |
+          echo "ref: ${{ inputs.ref }} "
+          echo "===== image "
+          echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-controller:${{ needs.get_ref.outputs.old_version }}"
+          TMP=` date +%m%d%H%M%S `
+          E2E_CLUSTER_NAME="spiderpool${TMP}"
+          echo "E2E_CLUSTER_NAME=${E2E_CLUSTER_NAME}" >> $GITHUB_ENV
+
       - name: Setup Kind Cluster on oldVersion ${{ needs.get_ref.outputs.old_version }}
         uses: nick-invision/retry@v2
         with:
@@ -242,13 +272,17 @@ jobs:
             echo "restart docker before trying again"
             systemctl restart docker
           command: |
-            E2E_SPIDERPOOL_TAG=""
-            if ${{ needs.get_ref.outputs.old_version == 'main' }} ; then
-              E2E_SPIDERPOOL_TAG=latest
-            else
-              E2E_SPIDERPOOL_TAG=${{ needs.get_ref.outputs.old_version }}
+            E2E_SPIDERPOOL_TAG=${{ needs.get_ref.outputs.old_version }}
+            SPIDERPOOL_AGENT_IMAGE_NAME=ghcr.io/spidernet-io/spiderpool/spiderpool-agent
+            SPIDERPOOL_CONTROLLER_IMAGE_NAME=ghcr.io/spidernet-io/spiderpool/spiderpool-controller
+            if ${{ needs.get_ref.outputs.build_old_image_tag == 'true' }} ; then
+               SPIDERPOOL_AGENT_IMAGE_NAME=spiderpool-agent-race
+               SPIDERPOOL_CONTROLLER_IMAGE_NAME=spiderpool-controller-race
+               E2E_SPIDERPOOL_TAG=${{ needs.call_build_old_ci_image.outputs.imageTag }}
             fi
             make e2e_init_underlay -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }} \
+                -e SPIDERPOOL_AGENT_IMAGE_NAME=${SPIDERPOOL_AGENT_IMAGE_NAME} \
+                -e SPIDERPOOL_CONTROLLER_IMAGE_NAME=${SPIDERPOOL_CONTROLLER_IMAGE_NAME} \
                 -e E2E_SPIDERPOOL_TAG=${E2E_SPIDERPOOL_TAG} \
                 -e PYROSCOPE_LOCAL_PORT="" \
                 -e INSTALL_KUBEVIRT=true \
@@ -265,46 +299,46 @@ jobs:
         uses: actions/checkout@v4
         with:
           persist-credentials: false
-          ref: ${{ needs.get_ref.outputs.ref }}
+          ref: ${{ needs.get_ref.outputs.new_version }}
           fetch-depth: 0
 
       - name: Move kubeconfig from oldVersion ${{ needs.get_ref.outputs.old_version }} to the upgradeVersion ${{ needs.get_ref.outputs.new_version }}
         run: |
           mkdir -p ${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/
           cp -r /tmp/config ${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/config
 
-      - name: Download spiderpool-agent image
+      - name: Download new spiderpool-agent image with tag ${{ needs.call_build_new_ci_image.outputs.imageTag }}
         uses: actions/download-artifact@v3
         with:
-          name: image-tar-spiderpool-agent
+          name: new-image-tar-spiderpool-agent
           path: test/.download
 
-      - name: Download spiderpool-controller image
+      - name: Download new spiderpool-controller image with tag ${{ needs.call_build_new_ci_image.outputs.imageTag }}
         uses: actions/download-artifact@v3
         with:
-          name: image-tar-spiderpool-controller
+          name: new-image-tar-spiderpool-controller
           path: test/.download
 
-      - name: Load Images
+      - name: Load Images with tag ${{ needs.call_build_new_ci_image.outputs.imageTag }}
         run: |
           TAR_FILES=` ls test/.download `
           echo $TAR_FILES
           for ITEM in $TAR_FILES ; do
               IMAGE_NAME=${ITEM%*.tar}
               echo ${IMAGE_NAME}
-              cat test/.download/${ITEM} | docker import - ${IMAGE_NAME}:${{ needs.call_build_ci_image.outputs.imageTag }}
+              cat test/.download/${ITEM} | docker import - ${IMAGE_NAME}:${{ needs.call_build_new_ci_image.outputs.imageTag }}
           done
 
-      - name: Upgrade to version ${{ needs.get_ref.outputs.ref }}
+      - name: Upgrade to version ${{ needs.get_ref.outputs.new_version }}
         id: upgrade
         continue-on-error: true
         run: |
           make upgrade_e2e_spiderpool -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }} \
-                -e E2E_SPIDERPOOL_TAG=${{ needs.call_build_ci_image.outputs.imageTag }} \
+                -e E2E_SPIDERPOOL_TAG=${{ needs.call_build_new_ci_image.outputs.imageTag }} \
                 -e SPIDERPOOL_AGENT_IMAGE_NAME=spiderpool-agent-race \
                 -e SPIDERPOOL_CONTROLLER_IMAGE_NAME=spiderpool-controller-race
 
-      - name: Run e2e Test on ${{ needs.get_ref.outputs.ref }}
+      - name: Run e2e Test on ${{ needs.get_ref.outputs.new_version }}
         id: run_e2e
         continue-on-error: true
         if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }}
@@ -358,7 +392,7 @@ jobs:
 
   creat_issue:
     runs-on: ubuntu-latest
-    needs: [run_upgrade_tests_on_release_version, trivy_scan_images, get_ref]
+    needs: [run_upgrade_tests_on_release_version, get_ref]
     if: ${{ always() && needs.run_upgrade_tests_on_release_version.result == 'failure' && needs.get_ref.outputs.skip_create_pr == 'false' }}
     steps:
       - name: echo

.github/workflows/build-image-ci.yaml

@@ -24,6 +24,10 @@ on:
       push:
         required: true
         type: string
+      imageTarName:
+        required: false
+        type: string
+        default: "image-tar"
     outputs:
       imageTag:
         description: "tag of image ci"
@@ -256,7 +260,7 @@ jobs:
       - name: Upload artifact race image tar
         uses: actions/[email protected]
         with:
-          name: image-tar-${{ matrix.name }}
+          name: ${{ inputs.imageTarName }}-${{ matrix.name }}
           path: /tmp/${{ matrix.name }}-race.tar
           retention-days: 1
 

Makefile

@@ -428,6 +428,10 @@ clean: clean_e2e
 clean_e2e_spiderpool:
 	-$(QUIET) make -C test uninstall_spiderpool
 
+.PHONY: upgrade_e2e_spiderpool
+upgrade_e2e_spiderpool:
+	-$(QUIET) make -C test upgrade_spiderpool
+
 .PHONY: codegen
 codegen:
 	@echo "Generate k8s SDK with code-generator."

test/e2e/common/constant.go

@@ -26,7 +26,7 @@ const (
 	IPReclaimTimeout           = time.Minute * 5
 	ExecCommandTimeout         = time.Minute * 5
 	EventOccurTimeout          = time.Second * 30
-	ServiceAccountReadyTimeout = time.Second * 20
+	ServiceAccountReadyTimeout = time.Minute
 	NodeReadyTimeout           = time.Minute
 	ResourceDeleteTimeout      = time.Minute * 5
 	BatchCreateTimeout         = time.Minute * 5