diff --git a/.github/workflows/auto-upgrade-ci.yaml b/.github/workflows/auto-upgrade-ci.yaml index 8f62eabc6d..6892931ff5 100644 --- a/.github/workflows/auto-upgrade-ci.yaml +++ b/.github/workflows/auto-upgrade-ci.yaml @@ -49,6 +49,7 @@ jobs: e2e_enabled: ${{ env.RUN_E2E_ENABLED }} run_upgrade_enabled: ${{ env.RUN_UPGRADE_ENABLED }} skip_create_pr: ${{ env.SKIP_CREATE_PR }} + build_old_image_tag: ${{ env.BUILD_OLD_IMAGE_TAG }} steps: - name: Check Code Changes uses: dorny/paths-filter@v2.11.1 @@ -79,6 +80,8 @@ jobs: echo "OLD_VERSION=${{ github.event.inputs.old_version }}" >> $GITHUB_ENV echo "NEW_VERSION=${{ github.event.inputs.ref }}" >> $GITHUB_ENV echo "SKIP_CREATE_PR=true" >> $GITHUB_ENV + echo "RUN_UPGRADE_ENABLED=true" >> $GITHUB_ENV + echo "BUILD_OLD_IMAGE_TAG=false" >> $GITHUB_ENV if ${{ github.event.inputs.e2e_enabled == 'true' }}; then echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV else @@ -88,8 +91,10 @@ jobs: echo "call by workflow_call" echo "RUN_TAG=${{ inputs.dest_tag }}" >> $GITHUB_ENV echo "NEW_VERSION=${{ inputs.dest_tag }}" >> $GITHUB_ENV + echo "RUN_UPGRADE_ENABLED=true" >> $GITHUB_ENV echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV echo "SKIP_CREATE_PR=true" >> $GITHUB_ENV + echo "BUILD_OLD_IMAGE_TAG=false" >> $GITHUB_ENV YBranchName=` grep -Eo "v[0-9]+\.[0-9]+" <<< "${{ inputs.dest_tag }}" ` # Get the previous minor version of version y LATEST_RELEASE_VERISON=$(curl -s https://api.github.com/repos/spidernet-io/spiderpool/releases | grep '"tag_name":' | grep -Eo "v([0-9]+\.[0-9]+\.[0-9])" | grep ${YBranchName} | sort -r | head -n 1) @@ -107,15 +112,23 @@ jobs: echo "RUN_TAG=${{ github.sha }}" >> $GITHUB_ENV echo "NEW_VERSION=${{ github.sha }}" >> $GITHUB_ENV # for PR scenarios, the latest version of the main branch will be used as old-version - echo "OLD_VERSION=main" >> $GITHUB_ENV + echo "OLD_VERSION: ${{ github.event.pull_request.base.sha }}" echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV echo "SKIP_CREATE_PR=false" >> $GITHUB_ENV + echo "BUILD_OLD_IMAGE_TAG=true" >> $GITHUB_ENV + if ${{ steps.filter_pr.outputs.run_upgrade == 'true' }} ; then + echo "RUN_UPGRADE_ENABLED=true" >> $GITHUB_ENV + else + echo "RUN_UPGRADE_ENABLED=false" >> $GITHUB_ENV + fi elif ${{ github.event_name == 'pull_request_target' }} ; then echo "trigger by pull_request_target" echo "RUN_TAG=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV echo "NEW_VERSION=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV # for PR scenarios, the latest version of the main branch will be used as old-version - echo "OLD_VERSION=main" >> $GITHUB_ENV + echo "BUILD_OLD_IMAGE_TAG=true" >> $GITHUB_ENV + echo "OLD_VERSION=${{ github.event.pull_request.base.sha }}" >> $GITHUB_ENV + echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV echo "SKIP_CREATE_PR=true" >> $GITHUB_ENV if ${{ steps.filter_pr.outputs.run_upgrade == 'true' }} ; then echo "RUN_UPGRADE_ENABLED=true" >> $GITHUB_ENV @@ -128,6 +141,9 @@ jobs: echo "trigger by schedule" echo "RUN_TAG=main" >> $GITHUB_ENV echo "NEW_VERSION=main" >> $GITHUB_ENV + echo "RUN_UPGRADE_ENABLED=true" >> $GITHUB_ENV + echo "BUILD_OLD_IMAGE_TAG=false" >> $GITHUB_ENV + echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV LATEST_RELEASE_VERISON=$(curl -s https://api.github.com/repos/spidernet-io/spiderpool/releases | grep '"tag_name":' | sort -r | head -n 1) rc_version=$(grep -Eo 'v[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]' <<< ${LATEST_RELEASE_VERISON}) echo "SKIP_CREATE_PR=false" >> $GITHUB_ENV @@ -138,7 +154,6 @@ jobs: echo "The latest released version is an 'rc: ${LATEST_RELEASE_VERISON}' version." echo "OLD_VERSION=${rc_version}" >> $GITHUB_ENV fi - echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV fi - name: Checkout code @@ -154,34 +169,28 @@ jobs: ref=$( git show -s --format='format:%H') echo "RUN_REF=${ref}" >> $GITHUB_ENV - call_build_ci_image: + call_build_old_ci_image: needs: [get_ref] if: ${{ needs.get_ref.outputs.run_upgrade_enabled == 'true' }} uses: ./.github/workflows/build-image-ci.yaml with: - ref: ${{ needs.get_ref.outputs.ref }} + ref: ${{ needs.get_ref.outputs.old_version }} push: false + imageTarName: old-image-tar secrets: inherit - lint_chart_against_release_image: - needs: get_ref - if: ${{ needs.get_ref.outputs.run_upgrade_enabled == 'true' }} - uses: ./.github/workflows/call-lint-chart.yaml - with: - ref: ${{ needs.get_ref.outputs.ref }} - secrets: inherit - - trivy_scan_images: - needs: [call_build_ci_image, get_ref] + call_build_new_ci_image: + needs: [get_ref] if: ${{ needs.get_ref.outputs.run_upgrade_enabled == 'true' }} - uses: ./.github/workflows/trivy-scan-image.yaml + uses: ./.github/workflows/build-image-ci.yaml with: - image_tag: ${{ needs.call_build_ci_image.outputs.imageTag }} - ref: ${{ needs.get_ref.outputs.ref }} + ref: ${{ needs.get_ref.outputs.new_version }} + push: false + imageTarName: new-image-tar secrets: inherit run_upgrade_tests_on_release_version: - needs: [call_build_ci_image, get_ref] + needs: [get_ref, call_build_old_ci_image, call_build_new_ci_image] if: ${{ needs.get_ref.outputs.run_upgrade_enabled == 'true' }} runs-on: ubuntu-latest steps: @@ -200,21 +209,7 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - ref: ${{ needs.get_ref.outputs.new_version }} - - - name: Prepare - id: prepare - run: | - echo "ref: ${{ inputs.ref }} " - echo "===== image " - if ${{ needs.get_ref.outputs.old_version == 'main' }} ; then - echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-controller:latest" - else - echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-controller:${{ needs.get_ref.outputs.old_version }}" - fi - TMP=` date +%m%d%H%M%S ` - E2E_CLUSTER_NAME="spiderpool${TMP}" - echo "E2E_CLUSTER_NAME=${E2E_CLUSTER_NAME}" >> $GITHUB_ENV + ref: ${{ needs.get_ref.outputs.old_version }} - name: Setup Golang uses: actions/setup-go@v5 @@ -230,6 +225,41 @@ jobs: run: | bash ./test/scripts/install-tools.sh + - name: Download old spiderpool-agent image with tag ${{ needs.call_build_old_ci_image.outputs.imageTag }} + if: ${{ needs.get_ref.outputs.build_old_image_tag == 'true' }} + uses: actions/download-artifact@v3 + with: + name: old-image-tar-spiderpool-agent + path: test/.download + + - name: Download old spiderpool-controller image with tag ${{ needs.call_build_old_ci_image.outputs.imageTag }} + if: ${{ needs.get_ref.outputs.build_old_image_tag == 'true' }} + uses: actions/download-artifact@v3 + with: + name: old-image-tar-spiderpool-controller + path: test/.download + + - name: Load Images + if: ${{ needs.get_ref.outputs.build_old_image_tag == 'true' }} + run: | + TAR_FILES=` ls test/.download ` + echo $TAR_FILES + for ITEM in $TAR_FILES ; do + IMAGE_NAME=${ITEM%*.tar} + echo ${IMAGE_NAME} + cat test/.download/${ITEM} | docker import - ${IMAGE_NAME}:${{ needs.call_build_old_ci_image.outputs.imageTag }} + done + + - name: Prepare + id: prepare + run: | + echo "ref: ${{ inputs.ref }} " + echo "===== image " + echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-controller:${{ needs.get_ref.outputs.old_version }}" + TMP=` date +%m%d%H%M%S ` + E2E_CLUSTER_NAME="spiderpool${TMP}" + echo "E2E_CLUSTER_NAME=${E2E_CLUSTER_NAME}" >> $GITHUB_ENV + - name: Setup Kind Cluster on oldVersion ${{ needs.get_ref.outputs.old_version }} uses: nick-invision/retry@v2 with: @@ -242,13 +272,17 @@ jobs: echo "restart docker before trying again" systemctl restart docker command: | - E2E_SPIDERPOOL_TAG="" - if ${{ needs.get_ref.outputs.old_version == 'main' }} ; then - E2E_SPIDERPOOL_TAG=latest - else - E2E_SPIDERPOOL_TAG=${{ needs.get_ref.outputs.old_version }} + E2E_SPIDERPOOL_TAG=${{ needs.get_ref.outputs.old_version }} + SPIDERPOOL_AGENT_IMAGE_NAME=ghcr.io/spidernet-io/spiderpool/spiderpool-agent + SPIDERPOOL_CONTROLLER_IMAGE_NAME=ghcr.io/spidernet-io/spiderpool/spiderpool-controller + if ${{ needs.get_ref.outputs.build_old_image_tag == 'true' }} ; then + SPIDERPOOL_AGENT_IMAGE_NAME=spiderpool-agent-race + SPIDERPOOL_CONTROLLER_IMAGE_NAME=spiderpool-controller-race + E2E_SPIDERPOOL_TAG=${{ needs.call_build_old_ci_image.outputs.imageTag }} fi make e2e_init_underlay -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }} \ + -e SPIDERPOOL_AGENT_IMAGE_NAME=${SPIDERPOOL_AGENT_IMAGE_NAME} \ + -e SPIDERPOOL_CONTROLLER_IMAGE_NAME=${SPIDERPOOL_CONTROLLER_IMAGE_NAME} \ -e E2E_SPIDERPOOL_TAG=${E2E_SPIDERPOOL_TAG} \ -e PYROSCOPE_LOCAL_PORT="" \ -e INSTALL_KUBEVIRT=true \ @@ -265,7 +299,7 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - ref: ${{ needs.get_ref.outputs.ref }} + ref: ${{ needs.get_ref.outputs.new_version }} fetch-depth: 0 - name: Move kubeconfig from oldVersion ${{ needs.get_ref.outputs.old_version }} to the upgradeVersion ${{ needs.get_ref.outputs.new_version }} @@ -273,38 +307,38 @@ jobs: mkdir -p ${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/ cp -r /tmp/config ${{ env.KUBECONFIG_PATH }}/${{ env.E2E_CLUSTER_NAME }}/.kube/config - - name: Download spiderpool-agent image + - name: Download new spiderpool-agent image with tag ${{ needs.call_build_new_ci_image.outputs.imageTag }} uses: actions/download-artifact@v3 with: - name: image-tar-spiderpool-agent + name: new-image-tar-spiderpool-agent path: test/.download - - name: Download spiderpool-controller image + - name: Download new spiderpool-controller image with tag ${{ needs.call_build_new_ci_image.outputs.imageTag }} uses: actions/download-artifact@v3 with: - name: image-tar-spiderpool-controller + name: new-image-tar-spiderpool-controller path: test/.download - - name: Load Images + - name: Load Images with tag ${{ needs.call_build_new_ci_image.outputs.imageTag }} run: | TAR_FILES=` ls test/.download ` echo $TAR_FILES for ITEM in $TAR_FILES ; do IMAGE_NAME=${ITEM%*.tar} echo ${IMAGE_NAME} - cat test/.download/${ITEM} | docker import - ${IMAGE_NAME}:${{ needs.call_build_ci_image.outputs.imageTag }} + cat test/.download/${ITEM} | docker import - ${IMAGE_NAME}:${{ needs.call_build_new_ci_image.outputs.imageTag }} done - - name: Upgrade to version ${{ needs.get_ref.outputs.ref }} + - name: Upgrade to version ${{ needs.get_ref.outputs.new_version }} id: upgrade continue-on-error: true run: | make upgrade_e2e_spiderpool -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }} \ - -e E2E_SPIDERPOOL_TAG=${{ needs.call_build_ci_image.outputs.imageTag }} \ + -e E2E_SPIDERPOOL_TAG=${{ needs.call_build_new_ci_image.outputs.imageTag }} \ -e SPIDERPOOL_AGENT_IMAGE_NAME=spiderpool-agent-race \ -e SPIDERPOOL_CONTROLLER_IMAGE_NAME=spiderpool-controller-race - - name: Run e2e Test on ${{ needs.get_ref.outputs.ref }} + - name: Run e2e Test on ${{ needs.get_ref.outputs.new_version }} id: run_e2e continue-on-error: true if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }} @@ -358,7 +392,7 @@ jobs: creat_issue: runs-on: ubuntu-latest - needs: [run_upgrade_tests_on_release_version, trivy_scan_images, get_ref] + needs: [run_upgrade_tests_on_release_version, get_ref] if: ${{ always() && needs.run_upgrade_tests_on_release_version.result == 'failure' && needs.get_ref.outputs.skip_create_pr == 'false' }} steps: - name: echo diff --git a/.github/workflows/build-image-ci.yaml b/.github/workflows/build-image-ci.yaml index c46c3e6d15..d356183152 100644 --- a/.github/workflows/build-image-ci.yaml +++ b/.github/workflows/build-image-ci.yaml @@ -24,6 +24,10 @@ on: push: required: true type: string + imageTarName: + required: false + type: string + default: "image-tar" outputs: imageTag: description: "tag of image ci" @@ -256,7 +260,7 @@ jobs: - name: Upload artifact race image tar uses: actions/upload-artifact@v3.1.3 with: - name: image-tar-${{ matrix.name }} + name: ${{ inputs.imageTarName }}-${{ matrix.name }} path: /tmp/${{ matrix.name }}-race.tar retention-days: 1 diff --git a/Makefile b/Makefile index 1cc324880e..d22ab56bd7 100644 --- a/Makefile +++ b/Makefile @@ -428,6 +428,10 @@ clean: clean_e2e clean_e2e_spiderpool: -$(QUIET) make -C test uninstall_spiderpool +.PHONY: upgrade_e2e_spiderpool +upgrade_e2e_spiderpool: + -$(QUIET) make -C test upgrade_spiderpool + .PHONY: codegen codegen: @echo "Generate k8s SDK with code-generator." diff --git a/test/e2e/common/constant.go b/test/e2e/common/constant.go index 33ef075217..2cfba1fbfa 100644 --- a/test/e2e/common/constant.go +++ b/test/e2e/common/constant.go @@ -26,7 +26,7 @@ const ( IPReclaimTimeout = time.Minute * 5 ExecCommandTimeout = time.Minute * 5 EventOccurTimeout = time.Second * 30 - ServiceAccountReadyTimeout = time.Second * 20 + ServiceAccountReadyTimeout = time.Minute NodeReadyTimeout = time.Minute ResourceDeleteTimeout = time.Minute * 5 BatchCreateTimeout = time.Minute * 5