eval-model.yml

# Evaluates Chronos-Bolt (Tiny) Model on Some Datasets
name: Evaluate

on:
  pull_request:
    branches: ["**"] # Run on any branch

jobs:
  evaluate-and-post:
    if: contains(github.event.pull_request.labels.*.name, 'run-eval')  # Only run if 'run-eval' label is added
    runs-on: ubuntu-latest

    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install Dependencies
        run: pip install ".[evaluation]" -f https://download.pytorch.org/whl/cpu/torch_stable.html

      - name: Run Eval Script
        run: python scripts/evaluation/evaluate.py ci/evaluate/backtest_config.yaml eval-ci-metrics.csv --chronos-model-id=amazon/chronos-bolt-tiny --device=cpu --torch-dtype=float32
      
      - name: Read CSV
        id: csv
        uses: juliangruber/read-file-action@v1
        with:
          path: eval-ci-metrics.csv

      - name: Create Markdown Table
        uses: petems/csv-to-md-table-action@master
        id: csv-table-output
        with:
          csvinput: ${{ steps.csv.outputs.content }}

      - name: Post Table as a Comment
        uses: peter-evans/create-or-update-comment@v4
        with:
          token: ${{ secrets.PR_COMMENT_TOKEN }}
          repository: ${{ github.repository }}
          issue-number: ${{ github.event.pull_request.number }}
          body: |
            ### Evaluation Metrics
            ${{steps.csv-table-output.outputs.markdown-table}}