reports.yml

name: Reports

on:
  push:
    branches:
      - master
  pull_request:

jobs:
  build-nargo:
    runs-on: ubuntu-22.04

    steps:
      - name: Checkout Noir repo
        uses: actions/checkout@v4

      - name: Setup toolchain
        uses: dtolnay/rust-toolchain@1.74.1

      - uses: Swatinem/rust-cache@v2
        with:
          key: x86_64-unknown-linux-gnu
          cache-on-failure: true
          save-if: ${{ github.event_name != 'merge_group' }}

      - name: Build Nargo
        run: cargo build --package nargo_cli --release

      - name: Package artifacts
        run: |
          mkdir dist
          cp ./target/release/nargo ./dist/nargo

      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: nargo
          path: ./dist/*
          retention-days: 3

  compare_gates_reports:
    name: Circuit sizes
    needs: [build-nargo]
    runs-on: ubuntu-22.04
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - name: Install `bb`
        run: |
          ./scripts/install_bb.sh
          echo "$HOME/.bb/" >> $GITHUB_PATH

      - name: Download nargo binary
        uses: actions/download-artifact@v4
        with:
          name: nargo
          path: ./nargo

      - name: Set nargo on PATH
        run: |
          nargo_binary="${{ github.workspace }}/nargo/nargo"
          chmod +x $nargo_binary
          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
          export PATH="$PATH:$(dirname $nargo_binary)"
          nargo -V

      - name: Generate gates report
        working-directory: ./test_programs
        run: |
          ./rebuild.sh
          ./gates_report.sh
          mv gates_report.json ../gates_report.json
      
      - name: Compare gates reports
        id: gates_diff
        uses: noir-lang/noir-gates-diff@1931aaaa848a1a009363d6115293f7b7fc72bb87
        with:
          report: gates_report.json
          summaryQuantile: 0.9 # only display the 10% most significant circuit size diffs in the summary (defaults to 20%)

      - name: Add gates diff to sticky comment
        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          # delete the comment in case changes no longer impact circuit sizes
          delete: ${{ !steps.gates_diff.outputs.markdown }}
          message: ${{ steps.gates_diff.outputs.markdown }}

  compare_brillig_bytecode_size_reports:
    name: Brillig bytecode sizes
    needs: [build-nargo]
    runs-on: ubuntu-22.04
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - name: Download nargo binary
        uses: actions/download-artifact@v4
        with:
          name: nargo
          path: ./nargo

      - name: Set nargo on PATH
        run: |
          nargo_binary="${{ github.workspace }}/nargo/nargo"
          chmod +x $nargo_binary
          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
          export PATH="$PATH:$(dirname $nargo_binary)"
          nargo -V

      - name: Generate Brillig bytecode size report
        working-directory: ./test_programs
        run: |
          ./gates_report_brillig.sh
          mv gates_report_brillig.json ../gates_report_brillig.json

      - name: Compare Brillig bytecode size reports
        id: brillig_bytecode_diff
        uses: noir-lang/noir-gates-diff@d88f7523b013b9edd3f31c5cfddaef87a3fe1b48
        with:
          report: gates_report_brillig.json
          header: |
            # Changes to Brillig bytecode sizes
          brillig_report: true
          summaryQuantile: 0.9 # only display the 10% most significant bytecode size diffs in the summary (defaults to 20%)

      - name: Add bytecode size diff to sticky comment
        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: brillig
          # delete the comment in case changes no longer impact brillig bytecode sizes
          delete: ${{ !steps.brillig_bytecode_diff.outputs.markdown }}
          message: ${{ steps.brillig_bytecode_diff.outputs.markdown }}

  compare_brillig_execution_reports:
    name: Brillig execution trace sizes  
    needs: [build-nargo]
    runs-on: ubuntu-22.04
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - name: Download nargo binary
        uses: actions/download-artifact@v4
        with:
          name: nargo
          path: ./nargo

      - name: Set nargo on PATH
        run: |
          nargo_binary="${{ github.workspace }}/nargo/nargo"
          chmod +x $nargo_binary
          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
          export PATH="$PATH:$(dirname $nargo_binary)"
          nargo -V

      - name: Generate Brillig execution report
        working-directory: ./test_programs
        run: |
          ./gates_report_brillig_execution.sh
          mv gates_report_brillig_execution.json ../gates_report_brillig_execution.json

      - name: Compare Brillig execution reports
        id: brillig_execution_diff
        uses: noir-lang/noir-gates-diff@d88f7523b013b9edd3f31c5cfddaef87a3fe1b48
        with:
          report: gates_report_brillig_execution.json
          header: |
            # Changes to number of Brillig opcodes executed
          brillig_report: true
          summaryQuantile: 0.9 # only display the 10% most significant bytecode size diffs in the summary (defaults to 20%)

      - name: Add bytecode size diff to sticky comment
        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: brillig_execution
          # delete the comment in case changes no longer impact brillig bytecode sizes
          delete: ${{ !steps.brillig_execution_diff.outputs.markdown }}
          message: ${{ steps.brillig_execution_diff.outputs.markdown }}

  generate_memory_report:
    name: Peak memory usage  
    needs: [build-nargo]
    runs-on: ubuntu-22.04
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - name: Download nargo binary
        uses: actions/download-artifact@v4
        with:
          name: nargo
          path: ./nargo

      - name: Set nargo on PATH
        run: |
          nargo_binary="${{ github.workspace }}/nargo/nargo"
          chmod +x $nargo_binary
          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
          export PATH="$PATH:$(dirname $nargo_binary)"
          nargo -V

      - name: Generate Memory report
        working-directory: ./test_programs
        run: |
          ./memory_report.sh
          mv memory_report.json ../memory_report.json

      - name: Upload memory report
        uses: actions/upload-artifact@v4
        with:
          name: in_progress_memory_report
          path: memory_report.json
          retention-days: 3
          overwrite: true

  generate_compilation_report:
    name: Compilation time
    needs: [build-nargo]
    runs-on: ubuntu-22.04
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - name: Download nargo binary
        uses: actions/download-artifact@v4
        with:
          name: nargo
          path: ./nargo

      - name: Set nargo on PATH
        run: |
          nargo_binary="${{ github.workspace }}/nargo/nargo"
          chmod +x $nargo_binary
          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
          export PATH="$PATH:$(dirname $nargo_binary)"
          nargo -V

      - name: Generate Compilation report
        working-directory: ./test_programs
        run: |
          ./compilation_report.sh
          cat compilation_report.json
          mv compilation_report.json ../compilation_report.json

      - name: Upload compilation report
        uses: actions/upload-artifact@v4
        with:
          name: in_progress_compilation_report
          path: compilation_report.json
          retention-days: 3
          overwrite: true

  external_repo_compilation_report:
    needs: [build-nargo]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    strategy:
      fail-fast: false
      matrix:
        include:
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset }
          # TODO: Bring these back once they no longer time out
          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private }
          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public }

    name: External repo compilation report - ${{ matrix.project.repo }}/${{ matrix.project.path }}
    steps:
      - name: Download nargo binary
        uses: actions/download-artifact@v4
        with:
          name: nargo
          path: ./nargo

      - name: Set nargo on PATH
        run: |
          nargo_binary="${{ github.workspace }}/nargo/nargo"
          chmod +x $nargo_binary
          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
          export PATH="$PATH:$(dirname $nargo_binary)"
          nargo -V

      - uses: actions/checkout@v4
        with:
          path: scripts
          sparse-checkout: |
            test_programs/compilation_report.sh
          sparse-checkout-cone-mode: false

      - name: Checkout
        uses: actions/checkout@v4
        with:
          repository: ${{ matrix.project.repo }}
          path: test-repo
          ref: ${{ matrix.project.ref }}

      - name: Generate compilation report
        working-directory: ./test-repo/${{ matrix.project.path }}
        run: |
          mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh
          chmod +x ./compilation_report.sh
          ./compilation_report.sh 1

      - name: Move compilation report
        id: report
        shell: bash
        run: |
          PACKAGE_NAME=${{ matrix.project.path }}
          PACKAGE_NAME=$(basename $PACKAGE_NAME)
          mv ./test-repo/${{ matrix.project.path }}/compilation_report.json ./compilation_report_$PACKAGE_NAME.json
          echo "compilation_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT

      - name: Upload compilation report
        uses: actions/upload-artifact@v4
        with:
          name: compilation_report_${{ steps.report.outputs.compilation_report_name }}
          path: compilation_report_${{ steps.report.outputs.compilation_report_name }}.json
          retention-days: 3
          overwrite: true

  external_repo_compilation_report_workspace_packages:
    needs: [build-nargo]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    strategy:
      fail-fast: false
      matrix:
        include:
          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts }
          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root }
          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner }
          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail }
          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset }
          # TODO: Bring these back once they no longer time out
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits, package: rollup_base_public }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits, package: rollup_base_private }

    name: External repo compilation report - ${{ matrix.project.repo }}/${{ matrix.project.path }}
    steps:
      - name: Download nargo binary
        uses: actions/download-artifact@v4
        with:
          name: nargo
          path: ./nargo

      - name: Set nargo on PATH
        run: |
          nargo_binary="${{ github.workspace }}/nargo/nargo"
          chmod +x $nargo_binary
          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
          export PATH="$PATH:$(dirname $nargo_binary)"
          nargo -V

      - uses: actions/checkout@v4
        with:
          path: scripts
          sparse-checkout: |
            test_programs/compilation_report.sh
          sparse-checkout-cone-mode: false

      - name: Checkout
        uses: actions/checkout@v4
        with:
          repository: ${{ matrix.project.repo }}
          path: test-repo
          ref: ${{ matrix.project.ref }}

      - uses: actions/checkout@v4
      - name: Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: '20.x'

      - name: Generate compilation report
        working-directory: ./test-repo/${{ matrix.project.path }}
        run: |
          chmod +x ./scripts/generate_variants.js
          node ./scripts/generate_variants.js
          mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh
          chmod +x ./compilation_report.sh
          ./compilation_report.sh 1 ${{ matrix.project.package }}

      - name: Move compilation report
        id: report
        shell: bash
        run: |
          PACKAGE_NAME=${{ matrix.project.path }}
          PACKAGE_NAME="$(basename $PACKAGE_NAME)${{ matrix.project.package }}"
          echo $PACKAGE_NAME
          mv ./test-repo/${{ matrix.project.path }}/compilation_report.json ./compilation_report_$PACKAGE_NAME.json
          echo "compilation_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT

      - name: Upload compilation report
        uses: actions/upload-artifact@v4
        with:
          name: compilation_report_${{ steps.report.outputs.compilation_report_name }}
          path: compilation_report_${{ steps.report.outputs.compilation_report_name }}.json
          retention-days: 3
          overwrite: true


  upload_compilation_report:
    name: Upload compilation report 
    needs: [generate_compilation_report, external_repo_compilation_report]
    # We want this job to run even if one variation of the matrix in `external_repo_compilation_report` fails
    if: always() 
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - name: Download initial compilation report
        uses: actions/download-artifact@v4
        with:
          name: in_progress_compilation_report

      - name: Download matrix compilation reports
        uses: actions/download-artifact@v4
        with:
          pattern: compilation_report_*
          path: ./reports

      - name: Merge compilation reports using jq
        run: |
          mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh
          ./merge-bench-reports.sh compilation_report

      - name: Parse compilation report
        id: compilation_report
        uses: noir-lang/noir-bench-report@0d7464a8c39170523932d7846b6e6b458a294aea
        with:
          report: compilation_report.json
          header: |
            # Compilation Report
          memory_report: false

      - name: Add memory report to sticky comment
        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: compilation
          message: ${{ steps.compilation_report.outputs.markdown }}

  external_repo_memory_report:
    needs: [build-nargo]
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        include:
          # TODO: Bring this report back under a flag. The `noir-contracts` report takes just under 30 minutes.
          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset }
          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail }

    name: External repo memory report - ${{ matrix.project.repo }}/${{ matrix.project.path }}
    steps:
      - name: Download nargo binary
        uses: actions/download-artifact@v4
        with:
          name: nargo
          path: ./nargo

      - name: Set nargo on PATH
        run: |
          nargo_binary="${{ github.workspace }}/nargo/nargo"
          chmod +x $nargo_binary
          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
          export PATH="$PATH:$(dirname $nargo_binary)"
          nargo -V

      - uses: actions/checkout@v4
        with:
          path: scripts
          sparse-checkout: |
            test_programs/memory_report.sh
          sparse-checkout-cone-mode: false

      - name: Checkout
        uses: actions/checkout@v4
        with:
          repository: ${{ matrix.project.repo }}
          path: test-repo
          ref: ${{ matrix.project.ref }}

      - name: Generate compilation report
        working-directory: ./test-repo/${{ matrix.project.path }}
        run: |
          mv /home/runner/work/noir/noir/scripts/test_programs/memory_report.sh ./memory_report.sh
          chmod +x ./memory_report.sh
          ./memory_report.sh 1

      - name: Move compilation report
        id: report
        shell: bash
        run: |
          PACKAGE_NAME=${{ matrix.project.path }}
          PACKAGE_NAME=$(basename $PACKAGE_NAME)
          mv ./test-repo/${{ matrix.project.path }}/memory_report.json ./memory_report_$PACKAGE_NAME.json
          echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT

      - name: Upload memory report
        uses: actions/upload-artifact@v4
        with:
          name: memory_report_${{ steps.report.outputs.memory_report_name }}
          path: memory_report_${{ steps.report.outputs.memory_report_name }}.json
          retention-days: 3
          overwrite: true

  upload_memory_report:
    name: Upload memory report 
    needs: [generate_memory_report, external_repo_memory_report]
    # We want this job to run even if one variation of the matrix in `external_repo_memory_report` fails
    if: always() 
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write

    steps:
      - uses: actions/checkout@v4

      - name: Download initial memory report
        uses: actions/download-artifact@v4
        with:
          name: in_progress_memory_report

      - name: Download matrix memory reports
        uses: actions/download-artifact@v4
        with:
          pattern: memory_report_*
          path: ./reports

      - name: Merge memory reports using jq
        run: |
          mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh
          ./merge-bench-reports.sh memory_report

      - name: Parse memory report
        id: memory_report
        uses: noir-lang/noir-bench-report@0d7464a8c39170523932d7846b6e6b458a294aea
        with:
          report: memory_report.json
          header: |
            # Memory Report
          memory_report: true

      - name: Add memory report to sticky comment
        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: memory
          message: ${{ steps.memory_report.outputs.markdown }}