auto-nightly-ci.yaml

name: Auto Nightly E2E CI

env:
  CLUSTER_NAME: spider
  E2E_TIME_OUT: 60m
  PERFORMANCESCRIPT: "./tools/scripts/e2eperformance.sh"

permissions: write-all

on:
  schedule:
    # each day
    - cron: "0 20 * * *"
  push:
    branches:
      - main
      - release-*

jobs:
  get_ref:
    runs-on: ubuntu-latest
    outputs:
      ref: ${{ steps.result.outputs.ref }}
      e2e_labels: ${{ steps.get_ref.outputs.e2e_labels }}
      unitest_enabled: ${{ steps.get_ref.outputs.unitest_enabled }}
      e2e_enabled: ${{ steps.get_ref.outputs.e2e_enabled }}
      ipfamily_ipv4only_e2e: ${{ steps.get_ref.outputs.ipfamily_ipv4only_e2e }}
      ipfamily_ipv6only_e2e: ${{ steps.get_ref.outputs.ipfamily_ipv6only_e2e }}
      ipfamily_dual_e2e: ${{ steps.get_ref.outputs.ipfamily_dual_e2e }}
    steps:
      - name: Check Code Changes
        uses: dorny/paths-filter@v2.11.1
        if: ${{ github.event_name == 'pull_request_target' }}
        id: filter_pr
        with:
          base: ${{ github.event.pull_request.base.sha }}
          ref: ${{ github.event.pull_request.head.sha }}
          filters: |
            run_e2e:
              - '**/*.sh'
              - '**/*.go'
              - 'go.mod'
              - 'go.sum'
              - 'charts/**'
              - 'Makefile*'
              - '**/Makefile*'
            all_e2e:
              - 'test/**'
              - 'vendor/github.com/spidernet-io/**/*.go'

      - name: Get Ref
        id: get_ref
        run: |
          if ${{ github.event_name == 'workflow_dispatch' }} ; then
            echo "call by self workflow_dispatch"
            echo ::set-output name=tag::${{ github.event.inputs.ref }}
            echo ::set-output name=e2e_labels::${{ github.event.inputs.e2e_labels }}
            echo ::set-output name=e2e_enabled::true
            echo ::set-output name=unitest_enabled::true
            if ${{ github.event.inputs.ipfamily == 'ipv4' }}; then
              echo ::set-output name=ipfamily_ipv4only_e2e::true
              echo ::set-output name=ipfamily_ipv6only_e2e::false
              echo ::set-output name=ipfamily_dual_e2e::false
            elif ${{ github.event.inputs.ipfamily == 'ipv6' }}; then
              echo ::set-output name=ipfamily_ipv4only_e2e::false
              echo ::set-output name=ipfamily_ipv6only_e2e::true
              echo ::set-output name=ipfamily_dual_e2e::false
            elif ${{ github.event.inputs.ipfamily == 'dual' }}; then
              echo ::set-output name=ipfamily_ipv4only_e2e::false
              echo ::set-output name=ipfamily_ipv6only_e2e::false
              echo ::set-output name=ipfamily_dual_e2e::true
            elif ${{ github.event.inputs.ipfamily == 'all' }}; then
              echo ::set-output name=ipfamily_ipv4only_e2e::true
              echo ::set-output name=ipfamily_ipv6only_e2e::true
              echo ::set-output name=ipfamily_dual_e2e::true
            else
              echo "error, unknown input ipfamily: ${{ github.event.inputs.ipfamily }} "
              exit 1
            fi
          elif ${{ github.event_name == 'push' }} ; then
            echo "trigger by push"
            echo ::set-output name=tag::${{ github.sha }}
            echo ::set-output name=push::false
            echo ::set-output name=e2e_labels::smoke
            echo ::set-output name=e2e_enabled::true
            # do it in another workflow
            echo ::set-output name=unitest_enabled::false
            echo ::set-output name=ipfamily_ipv4only_e2e::false
            echo ::set-output name=ipfamily_ipv6only_e2e::false
            echo ::set-output name=ipfamily_dual_e2e::true
          elif ${{ github.event_name == 'pull_request_target' }} ; then
            echo "trigger by pull_request_target"
            echo ::set-output name=tag::${{ github.event.pull_request.head.sha }}
            echo ::set-output name=push::false
            if ${{ steps.filter_pr.outputs.all_e2e == 'true' }} ; then
                # run all e2e
                echo ::set-output name=e2e_labels::
                echo ::set-output name=ipfamily_ipv4only_e2e::true
                echo ::set-output name=ipfamily_ipv6only_e2e::true
                echo ::set-output name=ipfamily_dual_e2e::true
            else
                echo ::set-output name=e2e_labels::smoke
                echo ::set-output name=ipfamily_ipv4only_e2e::false
                echo ::set-output name=ipfamily_ipv6only_e2e::false
                echo ::set-output name=ipfamily_dual_e2e::true
            fi
            echo ::set-output name=e2e_enabled::${{ steps.filter_pr.outputs.run_e2e }}
            # do it in another workflow
            echo ::set-output name=unitest_enabled::false
          else
            # schedule event
            # use main sha for ci image tag
            echo "trigger by schedule"
            echo ::set-output name=tag::main
            echo ::set-output name=push::false
            # nightly ci, run all e2e
            echo ::set-output name=e2e_labels::
            echo ::set-output name=e2e_enabled::true
            echo ::set-output name=unitest_enabled::true
            echo ::set-output name=ipfamily_ipv4only_e2e::true
            echo ::set-output name=ipfamily_ipv6only_e2e::true
            echo ::set-output name=ipfamily_dual_e2e::true
          fi

      # some event, the tag is not sha, so checkout it and get sha
      - name: Checkout code
        uses: actions/checkout@v3
        with:
          persist-credentials: false
          ref: ${{ steps.get_ref.outputs.tag }}

      - name: Result Ref
        id: result
        run: |
          ref=$( git show -s --format='format:%H')
          echo ::set-output name=ref::${ref}

  call_unitest:
    needs: get_ref
    if: ${{ needs.get_ref.outputs.unitest_enabled == 'true' }}
    # forbid to specify version for local workflow, GITHUB_REF Same as the caller workflow
    uses: ./.github/workflows/lint-golang.yaml
    with:
      ref: ${{ needs.get_ref.outputs.ref }}
    secrets: inherit

  call_build_ci_image:
    needs: [get_ref]
    if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }}
    # get  image:${{ needs.get_ref.outputs.ref }} and image-ci:${{ needs.get_ref.outputs.ref }}
    uses: ./.github/workflows/build-image-ci.yaml
    with:
      ref: ${{ needs.get_ref.outputs.ref }}
      push: false
    secrets: inherit

  lint_chart_against_release_image:
    needs: get_ref
    if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }}
    # forbid to specify version for local workflow, GITHUB_REF Same as the caller workflow
    uses: ./.github/workflows/call-lint-chart.yaml
    with:
      ref: ${{ needs.get_ref.outputs.ref }}
    secrets: inherit

  call_release_chart:
    needs: [get_ref]
    if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }}
    uses: ./.github/workflows/call-release-chart.yaml
    with:
      ref: ${{ needs.get_ref.outputs.ref }}
      submit: false
    secrets: inherit

  e2e_against_commit:
    runs-on: ubuntu-latest
    needs: [call_build_ci_image, get_ref, call_release_chart]
    if: ${{ needs.get_ref.outputs.e2e_enabled == 'true' }}
    steps:
      - name: Prepare
        id: prepare
        run: |
          echo "ref: ${{ needs.get_ref.outputs.ref }} "
          echo "e2e_labels: ${{ needs.get_ref.outputs.e2e_labels }}"
          echo "ipfamily_ipv4only_e2e: ${{ needs.get_ref.outputs.ipfamily_ipv4only_e2e }}"
          echo "ipfamily_ipv6only_e2e: ${{ needs.get_ref.outputs.ipfamily_ipv6only_e2e }}"
          echo "ipfamily_dual_e2e: ${{ needs.get_ref.outputs.ipfamily_dual_e2e }}"
          echo "===== image "
          echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-controller-ci:${{ needs.call_build_ci_image.outputs.imageTag }}"
          echo "ci race image tag: ghcr.io/${{ github.repository }}/spiderpool-controller-ci:${{ needs.call_build_ci_image.outputs.imageTag }}-race"
          echo "ci image tag: ghcr.io/${{ github.repository }}/spiderpool-agent-ci:${{ needs.call_build_ci_image.outputs.imageTag }}"
          echo "ci race image tag: ghcr.io/${{ github.repository }}/spiderpool-agent-ci:${{ needs.call_build_ci_image.outputs.imageTag }}-race"
          echo "===== chart "
          echo "chart package artifact: ${{ needs.call_release_chart.outputs.artifact }}"
          TMP=` date +%m%d%H%M%S `
          E2E_CLUSTER_NAME="spiderpool${TMP}"
          echo ::set-output name=clusterName::${E2E_CLUSTER_NAME}

      - name: Checkout Code
        uses: actions/checkout@v3
        with:
          persist-credentials: false
          ref: ${{ needs.get_ref.outputs.ref }}

      - name: Setup Golang
        uses: actions/setup-go@v3
        with:
          go-version: 1.20.0
          id: go

      # https://github.com/helm/kind-action
      - name: Install Kind Bin
        uses: helm/kind-action@v1.7.0
        with:
          install_only: true

      - name: Install Tools
        run: |
          # install kind/p2ctl/helm/ginkgo
          bash ./test/scripts/install-tools.sh

      - name: Download spiderpool-agent image
        uses: actions/download-artifact@v3
        with:
          name: image-tar-spiderpool-agent
          path: test/.download

      - name: Download spiderpool-controller image
        uses: actions/download-artifact@v3
        with:
          name: image-tar-spiderpool-controller
          path: test/.download

      - name: Load Spiderpool Image
        run: |
          TAR_FILES=` ls test/.download `
          echo $TAR_FILES
          for ITEM in $TAR_FILES ; do
              IMAGE_NAME=${ITEM%*.tar}
              cat test/.download/${ITEM} |  docker import - ${IMAGE_NAME}:${{ needs.call_build_ci_image.outputs.imageTag }}
          done

      # ------------- by default , dual stack
      # test against commit version
      - name: Setup Kind Cluster For Dual-stack
        if: ${{ needs.get_ref.outputs.ipfamily_dual_e2e == 'true' }}
        uses: nick-invision/retry@v2
        with:
          timeout_minutes: 10
          max_attempts: 3
          shell: bash
          command: |
            make e2e_init_underlay_subnet -e E2E_CLUSTER_NAME=${{ steps.prepare.outputs.clusterName }} \
                -e E2E_SPIDERPOOL_TAG=${{ needs.call_build_ci_image.outputs.imageTag }} \
                -e SPIDERPOOL_AGENT_IMAGE_NAME=spiderpool-agent-race   \
                -e SPIDERPOOL_CONTROLLER_IMAGE_NAME=spiderpool-controller-race \
                -e E2E_IP_FAMILY=dual -e PYROSCOPE_LOCAL_PORT=""

      - name: Run e2e Test For Dual-stack
        id: dualstack_e2e
        if: ${{ needs.get_ref.outputs.ipfamily_dual_e2e == 'true' }}
        continue-on-error: true
        run: |
          echo ${{ github.event.inputs.labels }}
          RESULT=0
          make e2e_test_underlay_subnet -e E2E_CLUSTER_NAME=${{ steps.prepare.outputs.clusterName }}  \
              -e E2E_GINKGO_LABELS=${{ needs.get_ref.outputs.e2e_labels }} \
              -e E2E_TIMEOUT=${{ env.E2E_TIME_OUT }} \
              -e E2E_IP_FAMILY=dual || RESULT=1
          if ((RESULT==0)) ; then
              echo ::set-output name=pass::true
          else
              echo ::set-output name=pass::false
          fi
          if [ -f "test/e2edebugLog" ] ; then
              echo ::set-output name=updaloadlog::true
          else
              echo ::set-output name=updaloadlog::false
          fi
          if [ -f "./e2ereport.json" ] ; then
              echo "error, did not find e2e report"
              echo ::set-output name=upload::true
          else
              echo ::set-output name=upload::false
          fi

      - name: Upload e2e log For Dual-stack
        if: ${{ needs.get_ref.outputs.ipfamily_dual_e2e == 'true' }}
        uses: actions/upload-artifact@v3.1.2
        with:
          name: dualstack-debuglog.txt
          path: test/e2edebugLog.txt
          retention-days: 30

      - name: Upload e2e report For Dual-stack
        if: ${{ steps.dualstack_e2e.outputs.upload == 'true' && needs.get_ref.outputs.ipfamily_dual_e2e == 'true' }}
        uses: actions/upload-artifact@v3.1.2
        with:
          name: dualstack-e2ereport.json
          path: e2ereport.json
          retention-days: 14

      - name: Show e2e Result For Dual-stack
        if: ${{ needs.get_ref.outputs.ipfamily_dual_e2e == 'true' }}
        run: |
          if ${{ steps.dualstack_e2e.outputs.pass == 'true' }} ;then
              exit 0
          else
              exit 1
          fi

      - name: Get the E2E Performance Result For Dual-stack
        # run all e2e cases
        id: get_performance
        if: ${{ needs.get_ref.outputs.ipfamily_dual_e2e == 'true' }}
        shell: bash
        run: |
          # sudo apt-get install -y jq
          TOOL_PATH=${{ env.PERFORMANCESCRIPT }}
          TODO_STATUS=$( ${TOOL_PATH}  "e2ereport.json"  )
          # echo "PERFORMANCE=${TODO_STATUS}" >> $GITHUB_ENV
          echo ::set-output name=result::${TODO_STATUS}

      - name: Update Badge For Dual-stack
        if: ${{ steps.get_performance.outputs.result != '' && needs.get_ref.outputs.ipfamily_dual_e2e == 'true' }}
        uses: schneegans/dynamic-badges-action@v1.6.0
        with:
          auth: ${{ secrets.WELAN_PAT }}
          gistID: 38d00a872e830eedb46870c886549561
          filename: spiderpoolperformance.json
          label: performance
          message: ${{ steps.get_performance.outputs.result }}
          color: lightgrey

      # ------------------ ipv4-only
      # test against commit version
      - name: Setup Kind Cluster For ipv4-only
        if: ${{ needs.get_ref.outputs.ipfamily_ipv4only_e2e == 'true' }}
        uses: nick-invision/retry@v2
        with:
          timeout_minutes: 10
          max_attempts: 3
          shell: bash
          command: |
            make e2e_init_underlay_subnet -e E2E_CLUSTER_NAME=${{ steps.prepare.outputs.clusterName }} \
                -e E2E_SPIDERPOOL_TAG=${{ needs.call_build_ci_image.outputs.imageTag }} \
                -e SPIDERPOOL_AGENT_IMAGE_NAME=spiderpool-agent-race   \
                -e SPIDERPOOL_CONTROLLER_IMAGE_NAME=spiderpool-controller-race \
                -e E2E_IP_FAMILY=ipv4 -e PYROSCOPE_LOCAL_PORT=""

      - name: Run e2e Test For ipv4-only
        id: ipv4only_e2e
        if: ${{ needs.get_ref.outputs.ipfamily_ipv4only_e2e == 'true' }}
        continue-on-error: true
        run: |
          echo ${{ github.event.inputs.labels }}
          RESULT=0
          make e2e_test_underlay_subnet -e E2E_CLUSTER_NAME=${{ steps.prepare.outputs.clusterName }}  \
              -e E2E_GINKGO_LABELS=${{ needs.get_ref.outputs.e2e_labels }} \
              -e E2E_TIMEOUT=${{ env.E2E_TIME_OUT }}  \
              -e E2E_IP_FAMILY=ipv4 -e PYROSCOPE_LOCAL_PORT="" || RESULT=1
          if ((RESULT==0)) ; then
              echo ::set-output name=pass::true
          else
              echo ::set-output name=pass::false
          fi
          if [ -f "test/e2edebugLog" ] ; then
              echo ::set-output name=updaloadlog::true
          else
              echo ::set-output name=updaloadlog::false
          fi
          if [ -f "./e2ereport.json" ] ; then
              echo "error, did not find e2e report"
              echo ::set-output name=upload::true
          else
              echo ::set-output name=upload::false
          fi

      - name: Upload e2e log For ipv4-only
        if: ${{ needs.get_ref.outputs.ipfamily_ipv4only_e2e == 'true' }}
        uses: actions/upload-artifact@v3.1.2
        with:
          name: ipv4Only-debuglog.txt
          path: test/e2edebugLog.txt
          retention-days: 30

      - name: Upload e2e report For ipv4-only
        if: ${{ steps.ipv4only_e2e.outputs.upload == 'true' && needs.get_ref.outputs.ipfamily_ipv4only_e2e == 'true' }}
        uses: actions/upload-artifact@v3.1.2
        with:
          name: ipv4Only-e2ereport.json
          path: e2ereport.json
          retention-days: 14

      - name: Show e2e Result For ipv4-only
        if: ${{ needs.get_ref.outputs.ipfamily_ipv4only_e2e == 'true' }}
        run: |
          if ${{ steps.ipv4only_e2e.outputs.pass == 'true' }} ;then
              exit 0
          else
              exit 1
          fi

      # ------------------ ipv6-only
      # test against commit version
      - name: Setup Kind Cluster For ipv6-only
        if: ${{ needs.get_ref.outputs.ipfamily_ipv6only_e2e == 'true' }}
        uses: nick-invision/retry@v2
        with:
          timeout_minutes: 10
          max_attempts: 3
          shell: bash
          command: |
            make e2e_init_underlay_subnet -e E2E_CLUSTER_NAME=${{ steps.prepare.outputs.clusterName }} \
                -e E2E_SPIDERPOOL_TAG=${{ needs.call_build_ci_image.outputs.imageTag }} \
                -e SPIDERPOOL_AGENT_IMAGE_NAME=spiderpool-agent-race   \
                -e SPIDERPOOL_CONTROLLER_IMAGE_NAME=spiderpool-controller-race \
                -e E2E_IP_FAMILY=ipv6 -e PYROSCOPE_LOCAL_PORT=""

      - name: Run e2e Test For ipv6-only
        id: ipv6only_e2e
        if: ${{ needs.get_ref.outputs.ipfamily_ipv6only_e2e == 'true' }}
        continue-on-error: true
        run: |
          echo ${{ github.event.inputs.labels }}
          RESULT=0
          make e2e_test_underlay_subnet -e E2E_CLUSTER_NAME=${{ steps.prepare.outputs.clusterName }}  \
              -e E2E_GINKGO_LABELS=${{ needs.get_ref.outputs.e2e_labels }} \
              -e E2E_TIMEOUT=${{ env.E2E_TIME_OUT }}  \
              -e E2E_IP_FAMILY=ipv6 -e PYROSCOPE_LOCAL_PORT="" || RESULT=1
          if ((RESULT==0)) ; then
              echo ::set-output name=pass::true
          else
              echo ::set-output name=pass::false
          fi
          if [ -f "test/e2edebugLog" ] ; then
              echo ::set-output name=updaloadlog::true
          else
              echo ::set-output name=updaloadlog::false
          fi
          if [ -f "./e2ereport.json" ] ; then
              echo "error, did not find e2e report"
              echo ::set-output name=upload::true
          else
              echo ::set-output name=upload::false
          fi

      - name: Upload e2e log For ipv6-only
        if: ${{ needs.get_ref.outputs.ipfamily_ipv6only_e2e == 'true' }}
        uses: actions/upload-artifact@v3.1.2
        with:
          name: ipv6Only-debuglog.txt
          path: test/e2edebugLog.txt
          retention-days: 30

      - name: Upload e2e report For ipv6-only
        if: ${{ steps.ipv6only_e2e.outputs.upload == 'true' && needs.get_ref.outputs.ipfamily_ipv6only_e2e == 'true' }}
        uses: actions/upload-artifact@v3.1.2
        with:
          name: ipv6Only-e2ereport.json
          path: e2ereport.json
          retention-days: 14

      - name: Show e2e Result For ipv6-only
        if: ${{ needs.get_ref.outputs.ipfamily_ipv6only_e2e == 'true' }}
        run: |
          if ${{ steps.ipv6only_e2e.outputs.pass == 'true' }} ;then
              exit 0
          else
              exit 1
          fi

  creat_issue:
    runs-on: ubuntu-latest
    needs: [e2e_against_commit, call_unitest, lint_chart_against_release_image]
    if: ${{ always() && needs.e2e_against_commit.result == 'failure' || needs.call_unitest.result == 'failure' || needs.lint_chart_against_release_image.result == 'failure' }}
    steps:
      - name: echo
        run: |
          echo ${{ github.repository }}
          echo ${{ github.repository_owner }}
          echo "TIMESTAMP=`date +%Y-%m-%d`" >> $GITHUB_ENV

      - name: create an issue
        uses: dacbd/create-issue-action@v1.2.1
        with:
          token: ${{ secrets.WELAN_PAT }}
          title: "Night CI ${{ ENV.TIMESTAMP }}: Failed"
          body: |
            action url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          labels: "kind/ci-bug"
          assignees: "Icarus9913,ty-dc"