refactor to properly use matrix

.github/workflows/adhoc-matrix.yml

@@ -0,0 +1,130 @@
+name: Execute ZKVM-Perf (Matrix)
+
+on:
+  workflow_dispatch:
+    inputs:
+      provers:
+        description: 'Provers to use (comma-separated)'
+        required: false
+        type: string
+        default: 'sp1'
+      programs:
+        description: 'Programs to benchmark (comma-separated)'
+        required: false
+        type: string
+        default: 'loop,fibonacci,tendermint,reth1,reth2'
+      filename:
+        description: 'Filename for the benchmark'
+        required: false
+        type: string
+        default: 'benchmark'
+      trials:
+        description: 'Number of trials to run'
+        required: false
+        type: string
+        default: '1'
+      sp1_ref:
+        description: 'SP1 reference (commit hash or branch name)'
+        required: false
+        type: string
+        default: '2e8b0a8'
+      additional_params:
+        description: 'Additional parameters as JSON'
+        required: false
+        type: string
+        default: '{"hashfns":"poseidon","shard_sizes":"22"}'
+
+jobs:
+  run-benchmarks:
+    strategy:
+      matrix:
+        include:
+          - instance_type: g6.16xlarge
+            enable_gpu: true
+            ami_id: ami-079a6a210557ef0e4
+          - instance_type: r7i.16xlarge
+            enable_gpu: false
+            ami_id: ami-079a6a210557ef0e4
+
+    name: Run on ${{ matrix.instance_type }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: xJonathanLEI/ec2-github-runner@main
+        with:
+          mode: start
+          github-token: ${{ secrets.GH_PAT }}
+          ec2-image-id: ${{ matrix.ami_id }}
+          ec2-instance-type: ${{ matrix.instance_type }}
+          subnet-id: ${{ secrets.AWS_SUBNET_ID }}
+          security-group-id: ${{ secrets.AWS_SG_ID }}
+          storage-size: 1024
+
+      - name: Run benchmarks
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GH_PAT }}
+          script: |
+            const runnerName = '${{ steps.start-ec2-runner.outputs.label }}';
+            const maxAttempts = 5;
+            const initialDelay = 30000; // 30 seconds
+            
+            for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+              console.log(`Attempt ${attempt} to trigger benchmark workflow`);
+              
+              await new Promise(resolve => setTimeout(resolve, initialDelay * attempt));
+              
+              try {
+                await github.rest.actions.createWorkflowDispatch({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  workflow_id: 'run-on-runner.yml',
+                  ref: context.ref,
+                  inputs: {
+                    runner_name: runnerName,
+                    instance_type: '${{ matrix.instance_type }}',
+                    enable_gpu: '${{ matrix.enable_gpu }}',
+                    provers: '${{ inputs.provers }}',
+                    programs: '${{ inputs.programs }}',
+                    filename: '${{ inputs.filename }}_${{ matrix.instance_type }}',
+                    trials: '${{ inputs.trials }}',
+                    sp1_ref: '${{ inputs.sp1_ref }}',
+                    additional_params: '${{ inputs.additional_params }}'
+                  }
+                });
+                console.log('Benchmark workflow triggered successfully');
+                break;
+              } catch (error) {
+                console.log(`Failed to trigger workflow: ${error.message}`);
+                if (attempt === maxAttempts) {
+                  core.setFailed('Failed to trigger benchmark workflow after multiple attempts');
+                }
+              }
+            }
+
+      - name: Wait for benchmark completion
+        run: |
+          while [[ $(gh run list --workflow=run-on-runner.yml --json status --jq '.[] | select(.status=="in_progress") | .status' | wc -l) -gt 0 ]]; do
+            echo "Waiting for benchmark to complete..."
+            sleep 60
+          done
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT }}
+
+      - name: Stop EC2 runner
+        if: always()
+        uses: xJonathanLEI/ec2-github-runner@main
+        with:
+          mode: stop
+          github-token: ${{ secrets.GH_PAT }}
+          label: ${{ steps.start-ec2-runner.outputs.label }}
+          ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}

.github/workflows/run-on-runner.yml

@@ -0,0 +1,93 @@
+name: Run Benchmarks on EC2 Runner
+
+on:
+  workflow_dispatch:
+    inputs:
+      runner_name:
+        required: true
+        type: string
+      instance_type:
+        required: true
+        type: string
+      enable_gpu:
+        required: true
+        type: string
+      provers:
+        required: false
+        type: string
+        default: 'sp1'
+      programs:
+        required: false
+        type: string
+        default: 'loop,fibonacci,tendermint,reth1,reth2'
+      filename:
+        required: false
+        type: string
+        default: 'benchmark'
+      trials:
+        required: false
+        type: string
+        default: '1'
+      sp1_ref:
+        required: false
+        type: string
+        default: '2e8b0a8'
+      additional_params:
+        required: false
+        type: string
+        default: '{"hashfns":"poseidon","shard_sizes":"22"}'
+
+jobs:
+  run-benchmark:
+    runs-on: ${{ inputs.runner_name }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v1
+
+      - name: Parse additional parameters
+        id: parse-params
+        run: |
+          echo 'ADDITIONAL_PARAMS<<EOF' >> $GITHUB_ENV
+          echo '${{ inputs.additional_params }}' >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+
+      - name: Update SP1 and build
+        env:
+          SP1_REF: ${{ inputs.sp1_ref }}
+          RUN_BUILD: "true"
+        run: |
+          chmod +x update_sp1_and_build.sh
+          ./update_sp1_and_build.sh
+
+      - name: Build Docker image
+        run: |
+          docker build -t zkvm-perf --platform linux/amd64 -f Dockerfile.gpu .
+
+      - name: Run benchmark
+        run: |
+          docker run ${{ inputs.enable_gpu == 'true' && '--gpus all' || '' }} --platform linux/amd64 \
+            -v /var/run/docker.sock:/var/run/docker.sock \
+            -v ${{ github.workspace }}/benchmarks:/usr/src/app/benchmarks \
+            -e RUST_BACKTRACE=full \
+            --network host \
+            zkvm-perf \
+            python3 sweep.py \
+              --filename ${{ inputs.filename }} \
+              --trials ${{ inputs.trials }} \
+              --programs ${{ inputs.programs }} \
+              --provers ${{ inputs.provers }} \
+              --hashfns ${{ fromJson(env.ADDITIONAL_PARAMS).hashfns }} \
+              --shard-sizes ${{ fromJson(env.ADDITIONAL_PARAMS).shard_sizes }}
+
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v2
+        with:
+          name: benchmark-results-${{ inputs.instance_type }}
+          path: ${{ github.workspace }}/benchmarks/*.csv
+
+      - name: Print Results
+        run: |
+          cat ${{ github.workspace }}/benchmarks/*.csv