From b322a937a14a7e26ebef12bc5f928939a7b0fe91 Mon Sep 17 00:00:00 2001 From: Abdul Fatir Ansari Date: Mon, 2 Dec 2024 09:40:49 +0000 Subject: [PATCH 1/3] Initial commit --- .github/workflows/eval-model.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/eval-model.yml b/.github/workflows/eval-model.yml index 0446870..83ec666 100644 --- a/.github/workflows/eval-model.yml +++ b/.github/workflows/eval-model.yml @@ -10,6 +10,8 @@ jobs: evaluate-and-post: if: contains(github.event.pull_request.labels.*.name, 'run-eval') # Only run if 'run-eval' label is added runs-on: ubuntu-latest + env: + RESULTS_CSV: "eval-ci-metrics-${{ github.event.pull_request.number }}.csv" steps: - name: Checkout Repository @@ -24,12 +26,12 @@ jobs: run: pip install ".[evaluation]" -f https://download.pytorch.org/whl/cpu/torch_stable.html - name: Run Eval Script - run: python scripts/evaluation/evaluate.py ci/evaluate/backtest_config.yaml eval-ci-metrics.csv --chronos-model-id=amazon/chronos-bolt-small --device=cpu --torch-dtype=float32 + run: python scripts/evaluation/evaluate.py ci/evaluate/backtest_config.yaml $RESULTS_CSV --chronos-model-id=amazon/chronos-bolt-small --device=cpu --torch-dtype=float32 - name: Upload CSV uses: actions/upload-artifact@v4 with: name: eval-metrics - path: eval-ci-metrics.csv + path: ${{ env.RESULTS_CSV }} retention-days: 1 overwrite: true From afd7a6253af7a8c0b3b082eeb8ca8e6229a3748b Mon Sep 17 00:00:00 2001 From: Abdul Fatir Ansari Date: Mon, 2 Dec 2024 09:45:13 +0000 Subject: [PATCH 2/3] Add types --- .github/workflows/eval-model.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/eval-model.yml b/.github/workflows/eval-model.yml index 83ec666..95f7e27 100644 --- a/.github/workflows/eval-model.yml +++ b/.github/workflows/eval-model.yml @@ -5,6 +5,11 @@ on: # Runs only with read privilages for the GITHUB_TOKEN pull_request: branches: ["main"] # Run on PRs to main branch + types: + - opened # When a PR is created + - reopened # When a closed PR is reopened + - synchronize # When new commits are pushed to the PR + - labeled # When a label is added to the PR jobs: evaluate-and-post: From 0d7e4dfcab0a882b4de27a326003b335447dc17a Mon Sep 17 00:00:00 2001 From: Abdul Fatir Ansari Date: Mon, 2 Dec 2024 09:58:56 +0000 Subject: [PATCH 3/3] Fix comment workflow --- .github/workflows/eval-pr-comment.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/eval-pr-comment.yml b/.github/workflows/eval-pr-comment.yml index 1f0d18c..56b84ae 100644 --- a/.github/workflows/eval-pr-comment.yml +++ b/.github/workflows/eval-pr-comment.yml @@ -29,12 +29,19 @@ jobs: - name: Display structure of downloaded files run: ls -R + + - name: Extract PR number from file name + id: extract-pr + run: | + FILE_NAME=$(ls eval-metrics-artifact/eval-ci-metrics-*.csv) + PR_NUMBER=$(echo $FILE_NAME | sed -E 's/.*eval-ci-metrics-(.*).csv/\1/') + echo "PR_NUMBER=$PR_NUMBER" >> $GITHUB_ENV - name: Read CSV id: csv uses: juliangruber/read-file-action@v1 with: - path: eval-metrics-artifact/eval-ci-metrics.csv + path: eval-metrics-artifact/eval-ci-metrics-${{ env.PR_NUMBER }}.csv - name: Create Markdown Table uses: petems/csv-to-md-table-action@master @@ -47,7 +54,7 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} repository: ${{ github.repository }} - issue-number: ${{ github.event.pull_request.number }} + issue-number: ${{ env.PR_NUMBER }} body: | ### Evaluation Metrics ${{steps.csv-table-output.outputs.markdown-table}}