perf.yml

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License.
# A copy of the License is located at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# or in the "license" file accompanying this file. This file is distributed
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.

name: 'Performance test'
on:
  repository_dispatch:
    types: [trigger-perf]
  workflow_dispatch:
    inputs:
      sha:
        description: 'the commit SHA to benchmark'
        required: true

env:
  GH_PAGES_BRANCH: gh-pages
  MAX_BENCHMARKS_TO_KEEP: 100
  COMMIT_USER: Github Actions
  COMMIT_EMAIL: actions@github.com
  TESTING_FRAMEWORK_REPO: aws-observability/aws-otel-test-framework
  GO_VERSION: ~1.22.3

permissions:
  id-token: write
  contents: write

jobs:
  create-test-ref:
    runs-on: ubuntu-latest
    outputs:
      testRef: ${{ steps.setRef.outputs.ref }}
    steps:
      - name: Set testRef output
        id: setRef
        run: |
          if [[ ${{ github.ref_name }} == release/v* ]]; then
            echo "ref=${{github.ref_name}}" >> $GITHUB_OUTPUT
          else
            echo "ref=terraform" >> $GITHUB_OUTPUT
          fi

  get-testing-version:
    runs-on: ubuntu-latest
    outputs:
      version: ${{ steps.get-testing-version.outputs.version }}
      testing_version: ${{ steps.get-testing-version.outputs.testing_version }}
      commit_id: ${{ steps.get-testing-version.outputs.commit_id }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.COLLECTOR_ASSUMABLE_ROLE_ARN }}
          aws-region: us-west-2

      - name: Download candidate based on dispatch payload
        if: github.event_name == 'repository_dispatch'
        run: aws s3 cp "s3://aws-otel-collector-release-candidate/${{ github.event.client_payload.sha }}.tar.gz" ./candidate.tar.gz

      - name: Download candidate based on workflow dispatch input
        if: github.event_name == 'workflow_dispatch'
        run: aws s3 cp "s3://aws-otel-collector-release-candidate/${{ github.event.inputs.sha }}.tar.gz" ./candidate.tar.gz

      - name: Uncompress the candidate package
        run: tar -zxf ./candidate.tar.gz

      - name: Get testing version
        id: get-testing-version
        run: |
          version=$(cat build/packages/VERSION)
          testing_version=$(cat build/packages/TESTING_VERSION)
          commit_id=$(cat build/packages/GITHUB_SHA)
          echo "version=$version" >> $GITHUB_OUTPUT
          echo "testing_version=$testing_version" >> $GITHUB_OUTPUT
          echo "commit_id=$commit_id" >> $GITHUB_OUTPUT

      - name: Check benchmarks for commit SHA
        id: benchmark-check
        run: COMMIT_SHA=${{ steps.get-testing-version.outputs.commit_id }} bash tools/workflow/is-commit-benchmarked.sh

      - name: Validate benchmark
        run: |
          if [[ ${{ steps.benchmark-check.outputs.has-commit }} == true ]]; then
            echo "::error::Benchmark already exists for ${{ steps.get-testing-version.outputs.commit_id }}. Skipping."
            exit 1
          fi

  get-perf-test-cases:
    runs-on: ubuntu-latest
    outputs:
      perf_matrix: ${{ steps.get-test-cases.outputs.perf_matrix }}
    steps:
      - uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'

      - name: Get all the perf test cases (all the soaking test cases are perf test cases)
        id: get-test-cases
        run: |
          perf_matrix=$(python e2etest/get-testcases.py perf_matrix)
          echo "perf_matrix=$perf_matrix" >> $GITHUB_OUTPUT

  build-aotutil:
    runs-on: ubuntu-latest
    needs: create-test-ref
    steps:
      - name: Check out testing framework
        uses: actions/checkout@v4
        with:
          repository: ${{ env.TESTING_FRAMEWORK_REPO }}
          path: testing-framework
          ref: ${{ needs.create-test-ref.outputs.testRef }}
      - name: Cache aotutil
        uses: actions/cache@v3
        id: cache
        with:
          key: "aotutil_${{ hashFiles('testing-framework/cmd/aotutil/*.go') }}_${{ hashFiles('testing-framework/cmd/aotutil/go.sum') }}"
          path: testing-framework/cmd/aotutil/aotutil
      - name: Set up Go 1.x
        uses: actions/setup-go@v5
        if: steps.cache.outputs.cache-hit != 'true'
        with:
          go-version: ${{ env.GO_VERSION }}
          cache-dependency-path: testing-framework/cmd/aotutil/go.sum
      - name: Build aotutil
        if: steps.cache.outputs.cache-hit != 'true'
        run: cd testing-framework/cmd/aotutil && make build


  run-perf-test:
    runs-on: ubuntu-latest
    needs: [get-perf-test-cases, get-testing-version, create-test-ref, build-aotutil]
    strategy:
      matrix: ${{ fromJson(needs.get-perf-test-cases.outputs.perf_matrix) }}
      max-parallel: 10
    steps:
      - uses: actions/checkout@v4

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.COLLECTOR_ASSUMABLE_ROLE_ARN }}
          aws-region: us-west-2
          # 3 hours
          role-duration-seconds: 10800

      - name: Set up JDK 11
        uses: actions/setup-java@v4
        with:
          distribution: 'zulu'
          java-version: '11'

      - name: Set up terraform
        uses: hashicorp/setup-terraform@v3
        with:
          terraform_version: "~1.5"

      - name: Check out testing framework
        uses: actions/checkout@v4
        with:
          repository: ${{ env.TESTING_FRAMEWORK_REPO}}
          path: testing-framework
          ref: ${{ needs.create-test-ref.outputs.testRef }}

      - name: Restore aotutil
        uses: actions/cache@v3
        with:
          key: "aotutil_${{ hashFiles('testing-framework/cmd/aotutil/*.go') }}_${{ hashFiles('testing-framework/cmd/aotutil/go.sum') }}"
          path: testing-framework/cmd/aotutil/aotutil
          fail-on-cache-miss: true

      - name: Run Performance test
        run: |
          if [[ -f testing-framework/terraform/testcases/${{ matrix.testcase }}/parameters.tfvars ]] ; then opts="-var-file=../testcases/${{ matrix.testcase }}/parameters.tfvars" ; else opts="" ; fi
          cd testing-framework/terraform/performance && terraform init && terraform apply -auto-approve -lock=false -var="data_rate=${{ matrix.data_rate }}" -var="commit_id=${{ needs.get-testing-version.outputs.commit_id }}" $opts -var="aoc_version=${{ needs.get-testing-version.outputs.testing_version }}" -var="testcase=../testcases/${{ matrix.testcase }}" -var="testing_ami=${{ matrix.testing_ami }}" -var="ssh_key_name=aoc-ssh-key-2022-09-13" -var="sshkey_s3_bucket=aoc-ssh-key" -var="sshkey_s3_private_key=aoc-ssh-key-2022-09-13.txt"

      - name: Upload the performance model as an artifact
        uses: actions/upload-artifact@v3
        with:
          name: "performance-model-${{ matrix.testcase }}-${{ matrix.data_rate }}-${{ matrix.testing_ami }}"
          path: testing-framework/terraform/performance/output/performance.json
          retention-days: 1

      - name: Destroy terraform resources
        if: ${{ always() }}
        run: |
          cd testing-framework/terraform/performance && terraform destroy -auto-approve

  upload-performance-model:
    runs-on: ubuntu-latest
    needs: [get-testing-version, run-perf-test]
    steps:
      - uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'

      - name: Install Jinja2
        run: pip install Jinja2

      - name: get all artifacts
        uses: actions/download-artifact@v3
        with:
          path: artifacts

      - name: Produce performance report
        run: python e2etest/get-performance-model-table.py -v ${{ needs.get-testing-version.outputs.version }}

      # Uses github-action-benchmark to update historic benchmark data
      - name: Store benchmark result
        uses: benchmark-action/github-action-benchmark@v1.18.0
        with:
          tool: "customSmallerIsBetter"
          output-file-path: performance-data.json
          benchmark-data-dir-path: benchmark/trend
          max-items-in-chart: ${{ env.MAX_BENCHMARKS_TO_KEEP }}
          gh-pages-branch: ${{ env.GH_PAGES_BRANCH }}
          github-token: ${{ secrets.GITHUB_TOKEN }}
          ref: ${{ needs.get-testing-version.outputs.commit_id }}
          auto-push: false

      - name: Commit to gh-pages branch
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          git switch ${{ env.GH_PAGES_BRANCH }}
          rsync -avv performance-report.md benchmark/report.md
          rsync -avv performance-data.json benchmark/data/
          git add benchmark/data/* benchmark/report.md
          git -c user.name="${{ env.COMMIT_USER }}" -c user.email="${{ env.COMMIT_EMAIL }}" commit --amend --reset-author -m "Update benchmarking for ${{ needs.get-testing-version.outputs.commit_id }}"
          git push origin ${{ env.GH_PAGES_BRANCH }}:${{ env.GH_PAGES_BRANCH }}