Fix auto eval workflow (#224)

*Issue #, if available:*

*Description of changes:* This PR fixes the auto evaluation workflow.
The second workflow step did not work because it did not know the right
PR number to post the comment on. The fix is to include the PR number in
the CSV file name and read it in the second workflow.

PS: This is a really poor user experience because there's no way to test
that this works right without merging!


By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

---------

Co-authored-by: Abdul Fatir Ansari <[email protected]>

.github/workflows/eval-model.yml

@@ -5,11 +5,18 @@ on:
   # Runs only with read privilages for the GITHUB_TOKEN
   pull_request:
     branches: ["main"] # Run on PRs to main branch
+    types:
+      - opened      # When a PR is created
+      - reopened    # When a closed PR is reopened
+      - synchronize # When new commits are pushed to the PR
+      - labeled     # When a label is added to the PR
 
 jobs:
   evaluate-and-post:
     if: contains(github.event.pull_request.labels.*.name, 'run-eval')  # Only run if 'run-eval' label is added
     runs-on: ubuntu-latest
+    env:
+      RESULTS_CSV: "eval-ci-metrics-${{ github.event.pull_request.number }}.csv"
 
     steps:
       - name: Checkout Repository
@@ -24,12 +31,12 @@ jobs:
         run: pip install ".[evaluation]" -f https://download.pytorch.org/whl/cpu/torch_stable.html
 
       - name: Run Eval Script
-        run: python scripts/evaluation/evaluate.py ci/evaluate/backtest_config.yaml eval-ci-metrics.csv --chronos-model-id=amazon/chronos-bolt-small --device=cpu --torch-dtype=float32
+        run: python scripts/evaluation/evaluate.py ci/evaluate/backtest_config.yaml $RESULTS_CSV --chronos-model-id=amazon/chronos-bolt-small --device=cpu --torch-dtype=float32
 
       - name: Upload CSV
         uses: actions/upload-artifact@v4
         with:
           name: eval-metrics
-          path: eval-ci-metrics.csv
+          path: ${{ env.RESULTS_CSV }}
           retention-days: 1
           overwrite: true

.github/workflows/eval-pr-comment.yml

@@ -29,12 +29,19 @@ jobs:
 
       - name: Display structure of downloaded files
         run: ls -R
+
+      - name: Extract PR number from file name
+        id: extract-pr
+        run: |
+          FILE_NAME=$(ls eval-metrics-artifact/eval-ci-metrics-*.csv)
+          PR_NUMBER=$(echo $FILE_NAME | sed -E 's/.*eval-ci-metrics-(.*).csv/\1/')
+          echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_ENV
 
       - name: Read CSV
         id: csv
         uses: juliangruber/read-file-action@v1
         with:
-          path: eval-metrics-artifact/eval-ci-metrics.csv
+          path: eval-metrics-artifact/eval-ci-metrics-${{ env.PR_NUMBER }}.csv
 
       - name: Create Markdown Table
         uses: petems/csv-to-md-table-action@master
@@ -47,7 +54,7 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           repository: ${{ github.repository }}
-          issue-number: ${{ github.event.pull_request.number }}
+          issue-number: ${{ env.PR_NUMBER }}
           body: |
             ### Evaluation Metrics
             ${{steps.csv-table-output.outputs.markdown-table}}