Auto Nightly E2E CI #715
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Auto Nightly E2E CI | |
| env: | |
| CLUSTER_NAME: spider | |
| E2E_TIME_OUT: 60m | |
| PERFORMANCESCRIPT: "./tools/scripts/e2eperformance.sh" | |
| permissions: write-all | |
| on: | |
| schedule: | |
| # each day | |
| - cron: "0 20 * * *" | |
| push: | |
| branches: | |
| - main | |
| - release-* | |
| jobs: | |
| get_ref: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| ref: ${{ steps.result.outputs.ref }} | |
| e2e_labels: ${{ steps.get_ref.outputs.e2e_labels }} | |
| unitest_enabled: ${{ steps.get_ref.outputs.unitest_enabled }} | |
| e2e_enabled: ${{ steps.get_ref.outputs.e2e_enabled }} | |
| ipfamily_ipv4only_e2e: ${{ steps.get_ref.outputs.ipfamily_ipv4only_e2e }} | |
| ipfamily_ipv6only_e2e: ${{ steps.get_ref.outputs.ipfamily_ipv6only_e2e }} | |
| ipfamily_dual_e2e: ${{ steps.get_ref.outputs.ipfamily_dual_e2e }} | |
| steps: | |
| - name: Check Code Changes | |
| uses: dorny/[email protected] | |
| if: ${{ github.event_name == 'pull_request_target' }} | |
| id: filter_pr | |
| with: | |
| base: ${{ github.event.pull_request.base.sha }} | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| filters: | | |
| run_e2e: | |
| - '**/*.sh' | |
| - '**/*.go' | |
| - 'go.mod' | |
| - 'go.sum' | |
| - 'charts/**' | |
| - 'Makefile*' | |
| - '**/Makefile*' | |
| all_e2e: | |
| - 'test/**' | |
| - 'vendor/github.com/spidernet-io/**/*.go' | |
| - name: Get Ref | |
| id: get_ref | |
| run: | | |
| if ${{ github.event_name == 'workflow_dispatch' }} ; then | |
| echo "call by self workflow_dispatch" | |
| echo ::set-output name=tag::${{ github.event.inputs.ref }} | |
| echo ::set-output name=e2e_labels::${{ github.event.inputs.e2e_labels }} | |
| echo ::set-output name=e2e_enabled::true | |
| echo ::set-output name=unitest_enabled::true | |
| if ${{ github.event.inputs.ipfamily == 'ipv4' }}; then | |
| echo ::set-output name=ipfamily_ipv4only_e2e::true | |
| echo ::set-output name=ipfamily_ipv6only_e2e::false | |
| echo ::set-output name=ipfamily_dual_e2e::false | |
| elif ${{ github.event.inputs.ipfamily == 'ipv6' }}; then | |
| echo ::set-output name=ipfamily_ipv4only_e2e::false | |
| echo ::set-output name=ipfamily_ipv6only_e2e::true | |
| echo ::set-output name=ipfamily_dual_e2e::false | |
| elif ${{ github.event.inputs.ipfamily == 'dual' }}; then | |
| echo ::set-output name=ipfamily_ipv4only_e2e::false | |
| echo ::set-output name=ipfamily_ipv6only_e2e::false | |
| echo ::set-output name=ipfamily_dual_e2e::true | |
| elif ${{ github.event.inputs.ipfamily == 'all' }}; then | |
| echo ::set-output name=ipfamily_ipv4only_e2e::true | |
| echo ::set-output name=ipfamily_ipv6only_e2e::true | |
| echo ::set-output name=ipfamily_dual_e2e::true | |
| else | |
| echo "error, unknown input ipfamily: ${{ github.event.inputs.ipfamily }} " | |
| exit 1 | |
| fi | |
| elif ${{ github.event_name == 'push' }} ; then | |
| echo "trigger by push" | |
| echo ::set-output name=tag::${{ github.sha }} | |
| echo ::set-output name=push::false | |
| echo ::set-output name=e2e_labels::smoke | |
| echo ::set-output name=e2e_enabled::true | |
| # do it in another workflow | |
| echo ::set-output name=unitest_enabled::false | |
| echo ::set-output name=ipfamily_ipv4only_e2e::false | |
| echo ::set-output name=ipfamily_ipv6only_e2e::false | |
| echo ::set-output name=ipfamily_dual_e2e::true | |
| elif ${{ github.event_name == 'pull_request_target' }} ; then | |
| echo "trigger by pull_request_target" | |
| echo ::set-output name=tag::${{ github.event.pull_request.head.sha }} | |
| echo ::set-output name=push::false | |
| if ${{ steps.filter_pr.outputs.all_e2e == 'true' }} ; then | |
| # run all e2e | |
| echo ::set-output name=e2e_labels:: | |
| echo ::set-output name=ipfamily_ipv4only_e2e::true | |
| echo ::set-output name=ipfamily_ipv6only_e2e::true | |
| echo ::set-output name=ipfamily_dual_e2e::true | |
| else | |
| echo ::set-output name=e2e_labels::smoke | |
| echo ::set-output name=ipfamily_ipv4only_e2e::false | |
| echo ::set-output name=ipfamily_ipv6only_e2e::false | |
| echo ::set-output name=ipfamily_dual_e2e::true | |
| fi | |
| echo ::set-output name=e2e_enabled::${{ steps.filter_pr.outputs.run_e2e }} | |
| # do it in another workflow | |
| echo ::set-output name=unitest_enabled::false | |
| else | |
| # schedule event | |
| # use main sha for ci image tag | |
| echo "trigger by schedule" | |
| echo ::set-output name=tag::main | |
| echo ::set-output name=push::false | |
| # nightly ci, run all e2e | |
| echo ::set-output name=e2e_labels:: | |
| echo ::set-output name=e2e_enabled::true | |
| echo ::set-output name=unitest_enabled::true | |
| echo ::set-output name=ipfamily_ipv4only_e2e::true | |
| echo ::set-output name=ipfamily_ipv6only_e2e::true | |
| echo ::set-output name=ipfamily_dual_e2e::true | |
| fi | |
| # some event, the tag is not sha, so checkout it and get sha | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ steps.get_ref.outputs.tag }} | |
| - name: Result Ref | |
| id: result | |
| run: | | |
| ref=$( git show -s --format='format:%H') | |
| echo ::set-output name=ref::${ref} | |
| call_unitest: | |
| needs: get_ref | |
| if: ${{ needs.get_ref.outputs.unitest_enabled == 'true' }} | |
| # forbid to specify version for local workflow, GITHUB_REF Same as the caller workflow | |
| uses: ./.github/workflows/lint-golang.yaml | |
| with: | |
| ref: ${{ needs.get_ref.outputs.ref }} | |
| secrets: inherit | |
| call_build_ci_image: | |
| needs: [get_ref] | |
| if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }} | |
| # get image:${{ needs.get_ref.outputs.ref }} and image-ci:${{ needs.get_ref.outputs.ref }} | |
| uses: ./.github/workflows/build-image-ci.yaml | |
| with: | |
| ref: ${{ needs.get_ref.outputs.ref }} | |
| push: false | |
| secrets: inherit | |
| lint_chart_against_release_image: | |
| needs: get_ref | |
| if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }} | |
| # forbid to specify version for local workflow, GITHUB_REF Same as the caller workflow | |
| uses: ./.github/workflows/call-lint-chart.yaml | |
| with: | |
| ref: ${{ needs.get_ref.outputs.ref }} | |
| secrets: inherit | |
| call_release_chart: | |
| needs: [get_ref] | |
| if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }} | |
| uses: ./.github/workflows/call-release-chart.yaml | |
| with: | |
| ref: ${{ needs.get_ref.outputs.ref }} | |
| submit: false | |
| secrets: inherit | |
| e2e_against_commit: | |
| runs-on: ubuntu-latest | |
| needs: [call_build_ci_image, get_ref, call_release_chart] | |
| if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }} | |
| steps: | |
| - name: Prepare | |
| id: prepare | |
| run: | | |
| echo "ref: ${{ needs.get_ref.outputs.ref }} " | |
| echo "e2e_labels: ${{ needs.get_ref.outputs.e2e_labels }}" | |
| echo "ipfamily_ipv4only_e2e: ${{ needs.get_ref.outputs.ipfamily_ipv4only_e2e }}" | |
| echo "ipfamily_ipv6only_e2e: ${{ needs.get_ref.outputs.ipfamily_ipv6only_e2e }}" | |
| echo "ipfamily_dual_e2e: ${{ needs.get_ref.outputs.ipfamily_dual_e2e }}" | |
| echo "===== image " | |
| echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-controller-ci:${{ needs.call_build_ci_image.outputs.imageTag }}" | |
| echo "ci race image tag: ghcr.io/${{ github.repository }}/spiderpool-controller-ci:${{ needs.call_build_ci_image.outputs.imageTag }}-race" | |
| echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-agent-ci:${{ needs.call_build_ci_image.outputs.imageTag }}" | |
| echo "ci race image tag: ghcr.io/${{ github.repository }}/spiderpool-agent-ci:${{ needs.call_build_ci_image.outputs.imageTag }}-race" | |
| echo "===== chart " | |
| echo "chart package artifact: ${{ needs.call_release_chart.outputs.artifact }}" | |
| TMP=` date +%m%d%H%M%S ` | |
| E2E_CLUSTER_NAME="spiderpool${TMP}" | |
| echo ::set-output name=clusterName::${E2E_CLUSTER_NAME} | |
| - name: Checkout Code | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| ref: ${{ needs.get_ref.outputs.ref }} | |
| - name: Setup Golang | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version: 1.20.0 | |
| id: go | |
| # https://github.com/helm/kind-action | |
| - name: Install Kind Bin | |
| uses: helm/[email protected] | |
| with: | |
| install_only: true | |
| - name: Install Tools | |
| run: | | |
| # install kind/p2ctl/helm/ginkgo | |
| bash ./test/scripts/install-tools.sh | |
| - name: Download spiderpool-agent image | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: image-tar-spiderpool-agent | |
| path: test/.download | |
| - name: Download spiderpool-controller image | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: image-tar-spiderpool-controller | |
| path: test/.download | |
| - name: Load Spiderpool Image | |
| run: | | |
| TAR_FILES=` ls test/.download ` | |
| echo $TAR_FILES | |
| for ITEM in $TAR_FILES ; do | |
| IMAGE_NAME=${ITEM%*.tar} | |
| cat test/.download/${ITEM} | docker import - ${IMAGE_NAME}:${{ needs.call_build_ci_image.outputs.imageTag }} | |
| done | |
| # ------------- by default , dual stack | |
| # test against commit version | |
| - name: Setup Kind Cluster For Dual-stack | |
| if: ${{ needs.get_ref.outputs.ipfamily_dual_e2e == 'true' }} | |
| uses: nick-invision/retry@v2 | |
| with: | |
| timeout_minutes: 10 | |
| max_attempts: 3 | |
| shell: bash | |
| command: | | |
| make e2e_init -e E2E_CLUSTER_NAME=${{ steps.prepare.outputs.clusterName }} \ | |
| -e TEST_IMAGE_TAG=${{ needs.call_build_ci_image.outputs.imageTag }} \ | |
| -e SPIDERPOOL_AGENT_IMAGE_NAME=spiderpool-agent-race \ | |
| -e SPIDERPOOL_CONTROLLER_IMAGE_NAME=spiderpool-controller-race \ | |
| -e E2E_IP_FAMILY=dual -e PYROSCOPE_LOCAL_PORT="" | |
| - name: Run e2e Test For Dual-stack | |
| id: dualstack_e2e | |
| if: ${{ needs.get_ref.outputs.ipfamily_dual_e2e == 'true' }} | |
| continue-on-error: true | |
| run: | | |
| echo ${{ github.event.inputs.labels }} | |
| RESULT=0 | |
| make e2e_test -e E2E_CLUSTER_NAME=${{ steps.prepare.outputs.clusterName }} \ | |
| -e E2E_GINKGO_LABELS=${{ needs.get_ref.outputs.e2e_labels }} \ | |
| -e E2E_TIMEOUT=${{ env.E2E_TIME_OUT }} \ | |
| -e E2E_IP_FAMILY=dual || RESULT=1 | |
| if ((RESULT==0)) ; then | |
| echo ::set-output name=pass::true | |
| else | |
| echo ::set-output name=pass::false | |
| fi | |
| if [ -f "test/e2edebugLog" ] ; then | |
| echo ::set-output name=updaloadlog::true | |
| else | |
| echo ::set-output name=updaloadlog::false | |
| fi | |
| if [ -f "./e2ereport.json" ] ; then | |
| echo "error, did not find e2e report" | |
| echo ::set-output name=upload::true | |
| else | |
| echo ::set-output name=upload::false | |
| fi | |
| - name: Upload e2e log For Dual-stack | |
| if: ${{ needs.get_ref.outputs.ipfamily_dual_e2e == 'true' }} | |
| uses: actions/[email protected] | |
| with: | |
| name: dualstack-debuglog.txt | |
| path: test/e2edebugLog.txt | |
| retention-days: 30 | |
| - name: Upload e2e report For Dual-stack | |
| if: ${{ steps.dualstack_e2e.outputs.upload == 'true' && needs.get_ref.outputs.ipfamily_dual_e2e == 'true' }} | |
| uses: actions/[email protected] | |
| with: | |
| name: dualstack-e2ereport.json | |
| path: e2ereport.json | |
| retention-days: 14 | |
| - name: Show e2e Result For Dual-stack | |
| if: ${{ needs.get_ref.outputs.ipfamily_dual_e2e == 'true' }} | |
| run: | | |
| if ${{ steps.dualstack_e2e.outputs.pass == 'true' }} ;then | |
| exit 0 | |
| else | |
| exit 1 | |
| fi | |
| - name: Get the E2E Performance Result For Dual-stack | |
| # run all e2e cases | |
| id: get_performance | |
| if: ${{ needs.get_ref.outputs.ipfamily_dual_e2e == 'true' }} | |
| shell: bash | |
| run: | | |
| # sudo apt-get install -y jq | |
| TOOL_PATH=${{ env.PERFORMANCESCRIPT }} | |
| TODO_STATUS=$( ${TOOL_PATH} "e2ereport.json" ) | |
| # echo "PERFORMANCE=${TODO_STATUS}" >> $GITHUB_ENV | |
| echo ::set-output name=result::${TODO_STATUS} | |
| - name: Update Badge For Dual-stack | |
| if: ${{ steps.get_performance.outputs.result != '' && needs.get_ref.outputs.ipfamily_dual_e2e == 'true' }} | |
| uses: schneegans/[email protected] | |
| with: | |
| auth: ${{ secrets.WELAN_PAT }} | |
| gistID: 38d00a872e830eedb46870c886549561 | |
| filename: spiderpoolperformance.json | |
| label: performance | |
| message: ${{ steps.get_performance.outputs.result }} | |
| color: lightgrey | |
| # ------------------ ipv4-only | |
| # test against commit version | |
| - name: Setup Kind Cluster For ipv4-only | |
| if: ${{ needs.get_ref.outputs.ipfamily_ipv4only_e2e == 'true' }} | |
| uses: nick-invision/retry@v2 | |
| with: | |
| timeout_minutes: 10 | |
| max_attempts: 3 | |
| shell: bash | |
| command: | | |
| make e2e_init -e E2E_CLUSTER_NAME=${{ steps.prepare.outputs.clusterName }} \ | |
| -e TEST_IMAGE_TAG=${{ needs.call_build_ci_image.outputs.imageTag }} \ | |
| -e SPIDERPOOL_AGENT_IMAGE_NAME=spiderpool-agent-race \ | |
| -e SPIDERPOOL_CONTROLLER_IMAGE_NAME=spiderpool-controller-race \ | |
| -e E2E_IP_FAMILY=ipv4 -e PYROSCOPE_LOCAL_PORT="" | |
| - name: Run e2e Test For ipv4-only | |
| id: ipv4only_e2e | |
| if: ${{ needs.get_ref.outputs.ipfamily_ipv4only_e2e == 'true' }} | |
| continue-on-error: true | |
| run: | | |
| echo ${{ github.event.inputs.labels }} | |
| RESULT=0 | |
| make e2e_test -e E2E_CLUSTER_NAME=${{ steps.prepare.outputs.clusterName }} \ | |
| -e E2E_GINKGO_LABELS=${{ needs.get_ref.outputs.e2e_labels }} \ | |
| -e E2E_TIMEOUT=${{ env.E2E_TIME_OUT }} \ | |
| -e E2E_IP_FAMILY=ipv4 -e PYROSCOPE_LOCAL_PORT="" || RESULT=1 | |
| if ((RESULT==0)) ; then | |
| echo ::set-output name=pass::true | |
| else | |
| echo ::set-output name=pass::false | |
| fi | |
| if [ -f "test/e2edebugLog" ] ; then | |
| echo ::set-output name=updaloadlog::true | |
| else | |
| echo ::set-output name=updaloadlog::false | |
| fi | |
| if [ -f "./e2ereport.json" ] ; then | |
| echo "error, did not find e2e report" | |
| echo ::set-output name=upload::true | |
| else | |
| echo ::set-output name=upload::false | |
| fi | |
| - name: Upload e2e log For ipv4-only | |
| if: ${{ needs.get_ref.outputs.ipfamily_ipv4only_e2e == 'true' }} | |
| uses: actions/[email protected] | |
| with: | |
| name: ipv4Only-debuglog.txt | |
| path: test/e2edebugLog.txt | |
| retention-days: 30 | |
| - name: Upload e2e report For ipv4-only | |
| if: ${{ steps.ipv4only_e2e.outputs.upload == 'true' && needs.get_ref.outputs.ipfamily_ipv4only_e2e == 'true' }} | |
| uses: actions/[email protected] | |
| with: | |
| name: ipv4Only-e2ereport.json | |
| path: e2ereport.json | |
| retention-days: 14 | |
| - name: Show e2e Result For ipv4-only | |
| if: ${{ needs.get_ref.outputs.ipfamily_ipv4only_e2e == 'true' }} | |
| run: | | |
| if ${{ steps.ipv4only_e2e.outputs.pass == 'true' }} ;then | |
| exit 0 | |
| else | |
| exit 1 | |
| fi | |
| # ------------------ ipv6-only | |
| # test against commit version | |
| - name: Setup Kind Cluster For ipv6-only | |
| if: ${{ needs.get_ref.outputs.ipfamily_ipv6only_e2e == 'true' }} | |
| uses: nick-invision/retry@v2 | |
| with: | |
| timeout_minutes: 10 | |
| max_attempts: 3 | |
| shell: bash | |
| command: | | |
| make e2e_init -e E2E_CLUSTER_NAME=${{ steps.prepare.outputs.clusterName }} \ | |
| -e TEST_IMAGE_TAG=${{ needs.call_build_ci_image.outputs.imageTag }} \ | |
| -e SPIDERPOOL_AGENT_IMAGE_NAME=spiderpool-agent-race \ | |
| -e SPIDERPOOL_CONTROLLER_IMAGE_NAME=spiderpool-controller-race \ | |
| -e E2E_IP_FAMILY=ipv6 -e PYROSCOPE_LOCAL_PORT="" | |
| - name: Run e2e Test For ipv6-only | |
| id: ipv6only_e2e | |
| if: ${{ needs.get_ref.outputs.ipfamily_ipv6only_e2e == 'true' }} | |
| continue-on-error: true | |
| run: | | |
| echo ${{ github.event.inputs.labels }} | |
| RESULT=0 | |
| make e2e_test -e E2E_CLUSTER_NAME=${{ steps.prepare.outputs.clusterName }} \ | |
| -e E2E_GINKGO_LABELS=${{ needs.get_ref.outputs.e2e_labels }} \ | |
| -e E2E_TIMEOUT=${{ env.E2E_TIME_OUT }} \ | |
| -e E2E_IP_FAMILY=ipv6 -e PYROSCOPE_LOCAL_PORT="" || RESULT=1 | |
| if ((RESULT==0)) ; then | |
| echo ::set-output name=pass::true | |
| else | |
| echo ::set-output name=pass::false | |
| fi | |
| if [ -f "test/e2edebugLog" ] ; then | |
| echo ::set-output name=updaloadlog::true | |
| else | |
| echo ::set-output name=updaloadlog::false | |
| fi | |
| if [ -f "./e2ereport.json" ] ; then | |
| echo "error, did not find e2e report" | |
| echo ::set-output name=upload::true | |
| else | |
| echo ::set-output name=upload::false | |
| fi | |
| - name: Upload e2e log For ipv6-only | |
| if: ${{ needs.get_ref.outputs.ipfamily_ipv6only_e2e == 'true' }} | |
| uses: actions/[email protected] | |
| with: | |
| name: ipv6Only-debuglog.txt | |
| path: test/e2edebugLog.txt | |
| retention-days: 30 | |
| - name: Upload e2e report For ipv6-only | |
| if: ${{ steps.ipv6only_e2e.outputs.upload == 'true' && needs.get_ref.outputs.ipfamily_ipv6only_e2e == 'true' }} | |
| uses: actions/[email protected] | |
| with: | |
| name: ipv6Only-e2ereport.json | |
| path: e2ereport.json | |
| retention-days: 14 | |
| - name: Show e2e Result For ipv6-only | |
| if: ${{ needs.get_ref.outputs.ipfamily_ipv6only_e2e == 'true' }} | |
| run: | | |
| if ${{ steps.ipv6only_e2e.outputs.pass == 'true' }} ;then | |
| exit 0 | |
| else | |
| exit 1 | |
| fi | |
| creat_issue: | |
| runs-on: ubuntu-latest | |
| needs: [e2e_against_commit, call_unitest, lint_chart_against_release_image] | |
| if: ${{ always() && needs.e2e_against_commit.result == 'failure' || needs.call_unitest.result == 'failure' || needs.lint_chart_against_release_image.result == 'failure' }} | |
| steps: | |
| - name: echo | |
| run: | | |
| echo ${{ github.repository }} | |
| echo ${{ github.repository_owner }} | |
| echo "TIMESTAMP=`date +%Y-%m-%d`" >> $GITHUB_ENV | |
| - name: create an issue | |
| uses: dacbd/[email protected] | |
| with: | |
| token: ${{ secrets.WELAN_PAT }} | |
| title: "Night CI ${{ ENV.TIMESTAMP }}: Failed" | |
| body: | | |
| action url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| labels: "kind/ci-bug" | |
| assignees: "Icarus9913,ty-dc" |