push-ci.yml

name: PUSH-CI

on:
  push:
    branches: [master, develop]
  #schedule:
  #  - cron: "0 18 * * *" # TimeZone: UTC 0

concurrency:
  group: rocketmq-${{ github.ref }}

env:
  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
  DOCKER_REPO: apache/rocketmq-ci

jobs:
  dist-tar:
    if: github.repository == 'apache/rocketmq'
    name: Build dist tar
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v3
        with:
          submodules: true
      - uses: actions/setup-java@v3
        with:
          distribution: "temurin"
          java-version: "8"
          cache: "maven"
      - name: Build distribution tar
        run: |
          mvn -Prelease-all -DskipTests -Dspotbugs.skip=true clean install -U
      - uses: actions/upload-artifact@v4
        name: Upload distribution tar
        with:
          name: rocketmq
          path: distribution/target/rocketmq*/rocketmq*

  docker:
    if: ${{ success() }}
    name: Docker images
    needs: [dist-tar]
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      matrix:
        base-image: ["ubuntu"]
        java-version: ["8"]
    steps:
      - uses: actions/checkout@v3
        with:
          repository: apache/rocketmq-docker.git
          ref: master
          path: rocketmq-docker
      - uses: actions/download-artifact@v4
        name: Download distribution tar
        with:
          name: rocketmq
          path: rocketmq
      - name: docker-login
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ secrets.DOCKERHUB_USER }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Build and save docker images
        id: build-images
        run: |
          cd rocketmq-docker/image-build-ci
          version=${{ github.event.pull_request.number || github.ref_name }}-$(uuidgen)
          mkdir versionlist
          touch versionlist/"${version}-`echo ${{ matrix.base-image }} | sed -e "s/:/-/g"`"
          sh ./build-image-local.sh ${version} ${{ matrix.base-image }} ${{ matrix.java-version }} ${DOCKER_REPO}
      - uses: actions/upload-artifact@v4
        name: Upload distribution tar
        with:
          name: versionlist
          path: rocketmq-docker/image-build-ci/versionlist/*

  list-version:
    if: >
      github.repository == 'apache/rocketmq' && 
      always()
    name: List version
    needs: [docker]
    runs-on: ubuntu-latest
    timeout-minutes: 30
    outputs:
      version-json: ${{ steps.show_versions.outputs.version-json }}
    steps:
      - uses: actions/download-artifact@v4
        name: Download versionlist
        with:
          name: versionlist
          path: versionlist
      - name: Show versions
        id: show_versions
        run: | 
          a=(`ls versionlist`)
          printf '%s\n' "${a[@]}" | jq -R . | jq -s .
          echo version-json=`printf '%s\n' "${a[@]}" | jq -R . | jq -s .` >> $GITHUB_OUTPUT

  deploy-e2e:
    if: ${{ success() }}
    name: Deploy RocketMQ For E2E
    needs: [list-version,docker]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      matrix:
        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
    steps:
      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288
        name: Deploy rocketmq
        with:
          action: "deploy"
          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
          test-version: "${{ matrix.version }}"
          chart-git: "https://ghproxy.com/https://github.com/apache/rocketmq-docker.git"
          chart-branch: "master"
          chart-path: "./rocketmq-k8s-helm"
          job-id: ${{ strategy.job-index }}
          helm-values: |
            nameserver:
              image:
                repository: ${{env.DOCKER_REPO}}
                tag: ${{ matrix.version }}
            broker:
              image:
                repository: ${{env.DOCKER_REPO}}
                tag: ${{ matrix.version }}
            proxy:
              image:
                repository: ${{env.DOCKER_REPO}}
                tag: ${{ matrix.version }}

  deploy-benchmark:
    if: ${{ success() }}
    name: Deploy RocketMQ For Benchmarking
    needs: [list-version,docker]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      matrix:
        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
    steps:
      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288
        name: Deploy rocketmq
        with:
          action: "deploy"
          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
          test-version: "${{ matrix.version }}"
          chart-git: "https://ghproxy.com/https://github.com/apache/rocketmq-docker.git"
          chart-branch: "master"
          chart-path: "./rocketmq-k8s-helm"
          job-id: "001-${{ strategy.job-index }}"
          helm-values: |
            nameserver:
              image:
                repository: ${{env.DOCKER_REPO}}
                tag: ${{ matrix.version }}
            broker:
              image:
                repository: ${{env.DOCKER_REPO}}
                tag: ${{ matrix.version }}
            proxy:
              image:
                repository: ${{env.DOCKER_REPO}}
                tag: ${{ matrix.version }}

  test-e2e-grpc-java:
    if: ${{ success() }}
    name: Test E2E grpc java
    needs: [list-version, deploy-e2e]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      matrix:
        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
    steps:
      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288
        name: e2e test
        with:
          action: "test"
          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
          test-version: "${{ matrix.version }}"
          test-code-git: "https://ghproxy.com/https://github.com/apache/rocketmq-e2e"
          test-code-branch: "master"
          test-code-path: java/e2e
          test-cmd: "mvn -B test"
          job-id: 0
      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v3
        if: always() # always run even if the previous step fails
        with:
          report_paths: '**/test_report/TEST-*.xml'
          annotate_only: true
          include_passed: true
          detailed_summary: true
      - uses: actions/upload-artifact@v4
        if: always()
        name: Upload test log
        with:
          name: test-e2e-grpc-java-log.txt
          path: testlog.txt
          
  test-e2e-golang:
    if: ${{ success() }}
    name: Test E2E golang
    needs: [list-version, deploy-e2e]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      matrix:
        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
    steps:
      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288
        name: e2e test
        with:
          action: "test"
          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
          test-version: "${{ matrix.version }}"
          test-code-git: "https://ghproxy.com/https://github.com/apache/rocketmq-e2e"
          test-code-branch: "master"
          test-code-path: golang
          test-cmd: |
            cd ../common &&  mvn -Prelease -DskipTests clean package -U
            cd ../rocketmq-admintools && source bin/env.sh
            LATEST_GO_VERSION=$(curl -s https://go.dev/VERSION?m=text | awk 'NR==1')
            wget "https://go.dev/dl/${LATEST_GO_VERSION}.linux-amd64.tar.gz" && \
            rm -rf /usr/local/go && tar -C /usr/local -xzf ${LATEST_GO_VERSION}.linux-amd64.tar.gz
            cd ../golang && go get -u github.com/apache/rocketmq-clients/golang && gotestsum --junitfile ./target/surefire-reports/TEST-report.xml ./mqgotest/... -timeout 2m  -v
          job-id: 0
      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v3
        if: always() # always run even if the previous step fails
        with:
          report_paths: '**/test_report/TEST-*.xml'
          annotate_only: true
          include_passed: true
          detailed_summary: true
      - uses: actions/upload-artifact@v4
        if: always()
        name: Upload test log
        with:
          name: test-e2e-golang-log.txt
          path: testlog.txt
          
  test-e2e-remoting-java:
    if: ${{ success() }}
    name: Test E2E remoting java
    needs: [ list-version, deploy-e2e ]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      matrix:
        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
    steps:
      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288
        name: e2e test
        with:
          action: "test"
          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
          test-version: "${{ matrix.version }}"
          test-code-git: "https://ghproxy.com/https://github.com/apache/rocketmq-e2e"
          test-code-branch: "master"
          test-code-path: java/e2e-v4
          test-cmd: "mvn -B test"
          job-id: 0
      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v3
        if: always() # always run even if the previous step fails
        with:
          report_paths: '**/test_report/TEST-*.xml'
          annotate_only: true
          include_passed: true
          detailed_summary: true
      - uses: actions/upload-artifact@v4
        if: always()
        name: Upload test log
        with:
          name: test-e2e-remoting-java-log.txt
          path: testlog.txt

  benchmark-test:
    if: ${{ success() }}
    runs-on: ubuntu-latest
    name: Performance benchmark test
    needs: [ list-version, deploy-benchmark ]
    timeout-minutes: 60
    steps:
      - uses: apache/rocketmq-test-tool/benchmark-runner@ce372e5f3906ca1891e4918b05be14608eae608e
        name: Performance benchmark
        with:
          action: "performance-benchmark"
          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
          job-id: "001-${{ strategy.job-index }}"
          # The time to run the test, 15 minutes
          test-time: "900"
          # Some thresholds set in advance
          min-send-tps-threshold: "12000"
          max-rt-ms-threshold: "500"
          avg-rt-ms-threshold: "10"
          max-2c-rt-ms-threshold: "150"
          avg-2c-rt-ms-threshold: "10"
      - name: Upload test report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-report
          path: benchmark/

  clean-e2e:
    if: always()
    name: Clean E2E
    needs: [ list-version, test-e2e-grpc-java, test-e2e-golang, test-e2e-remoting-java ]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      matrix:
        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
    steps:
      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288
        name: clean
        with:
          action: "clean"
          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
          test-version: "${{ matrix.version }}"
          job-id: ${{ strategy.job-index }}
          
  clean-benchmark:
    if: always()
    name: Clean Benchmarking
    needs: [ list-version, benchmark-test ]
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      matrix:
        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
    steps:
      - uses: apache/rocketmq-test-tool@7d84d276ad7755b1dc5cf9657a7a9bff6ae6d288
        name: clean
        with:
          action: "clean"
          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
          test-version: "${{ matrix.version }}"
          job-id: "001-${{ strategy.job-index }}"