[QA] Automatic vizro ai score tests #7
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Score tests for VizroAI | |
| defaults: | |
| run: | |
| working-directory: vizro-ai | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| env: | |
| PYTHONUNBUFFERED: 1 | |
| FORCE_COLOR: 1 | |
| jobs: | |
| test-score-vizro-ai-fork: | |
| if: ${{ github.event.pull_request.head.repo.fork }} | |
| name: test-score-vizro-ai on Py${{ matrix.config.python-version }} ${{ matrix.config.label }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: | |
| - python-version: "3.9" | |
| hatch-env: all.py3.9 | |
| - python-version: "3.10" | |
| hatch-env: all.py3.10 | |
| - python-version: "3.11" | |
| hatch-env: all.py3.11 | |
| - python-version: "3.12" | |
| hatch-env: all.py3.12 | |
| - python-version: "3.9" | |
| hatch-env: lower-bounds | |
| label: lower bounds | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Passed fork step | |
| run: echo "Success!" | |
| test-score-vizro-ai: | |
| if: ${{ ! github.event.pull_request.head.repo.fork }} | |
| name: test-score-vizro-ai on Py${{ matrix.config.python-version }} ${{ matrix.config.label }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: | |
| - python-version: "3.9" | |
| hatch-env: all.py3.9 | |
| - python-version: "3.10" | |
| hatch-env: all.py3.10 | |
| - python-version: "3.11" | |
| hatch-env: all.py3.11 | |
| - python-version: "3.12" | |
| hatch-env: all.py3.12 | |
| - python-version: "3.9" | |
| hatch-env: lower-bounds | |
| label: lower bounds | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.config.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.config.python-version }} | |
| - name: Install Hatch | |
| run: pip install hatch | |
| - name: Show dependency tree | |
| run: hatch run ${{ matrix.config.hatch-env }}:pip tree | |
| - name: Run vizro-ai score tests with PyPI vizro | |
| run: hatch run ${{ matrix.config.hatch-env }}:test-score | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }} | |
| VIZRO_TYPE: pypi | |
| BRANCH: ${{ github.head_ref }} | |
| PYTHON_VERSION: ${{ matrix.config.python-version }} | |
| - name: Run vizro-ai score tests with local vizro | |
| run: | | |
| hatch run ${{ matrix.config.hatch-env }}:pip install ../vizro-core | |
| hatch run ${{ matrix.config.hatch-env }}:test-score | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }} | |
| VIZRO_TYPE: local | |
| BRANCH: ${{ github.head_ref }} | |
| PYTHON_VERSION: ${{ matrix.config.python-version }} | |
| - name: Send custom JSON data to Slack | |
| id: slack | |
| uses: slackapi/[email protected] | |
| if: failure() | |
| with: | |
| payload: | | |
| { | |
| "text": "Vizro-ai ${{ matrix.config.hatch-env }} score tests build result: ${{ job.status }}\nBranch: ${{ github.head_ref }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| - name: Report artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: Report-${{ matrix.config.python-version }}-${{ matrix.config.label }} | |
| path: | | |
| /home/runner/work/vizro/vizro/vizro-ai/tests/score/reports/report*.csv | |
| test-score-vizro-ai-report: | |
| needs: test-score-vizro-ai | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download All Artifacts | |
| uses: actions/download-artifact@v4 | |
| - name: Set current date as env variable | |
| id: date | |
| run: | | |
| echo "::set-output name=date::$(date +'%Y-%m-%dT%H-%M-%S')" | |
| echo "TIME_NOW=$(date +'%Y-%m-%dT%H-%M-%S')" >> $GITHUB_ENV | |
| - name: Create one csv report | |
| run: | | |
| cd /home/runner/work/vizro/vizro/ | |
| head -n 1 Report-3.11-/report_model_gpt-4o-mini_pypi.csv > report-aggregated-${{ steps.date.outputs.date }}.csv && tail -n+2 -q */*.csv >> report-aggregated-${{ steps.date.outputs.date }}.csv | |
| gawk -F, -i inplace 'FNR>1 {$1="${{ steps.date.outputs.date }}"} {print}' OFS=, report-aggregated-${{ steps.date.outputs.date }}.csv | |
| - name: Report artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: Report-aggregated-${{ steps.date.outputs.date }} | |
| path: | | |
| /home/runner/work/vizro/vizro/report-aggregated-${{ steps.date.outputs.date }}.csv | |
| - name: Upload reports to artifactory | |
| run: | | |
| cd /home/runner/work/vizro/vizro/ | |
| curl -fL https://getcli.jfrog.io | sh | |
| ./jfrog config add vizro --artifactory-url ${{ secrets.ARTIFACTORY_URL}} --user ${{ secrets.ARTIFACTORY_USER}} --apikey ${{ secrets.ARTIFACTORY_PASS}} --interactive=false | |
| ./jfrog rt u --flat=false "report-aggregated-${{ steps.date.outputs.date }}.csv" vizx-generic-local/vizro-ai-reports/ |