eval.yml

name: Run evaluation
on:
  push:
    branches: [cav-eval]
  workflow_dispatch:

jobs:
  eval:
    strategy:
      fail-fast: false
      matrix:
        verifier:
          - name: 'carbon'
          - name: 'dafny'
            args: 'lt dafny/Source/IntegrationTests/TestFiles/LitTests/LitTest/dafny2'
            tag:  '-1'
          - name: 'dafny'
            args: 'gt dafny/Source/IntegrationTests/TestFiles/LitTests/LitTest/dafny2'
            tag:  '-2'
          - name: 'fstar'
          - name: 'silicon'
          - name: 'smt-comp'
            args: 'lt smt-comp/non-incremental'
            tag:  '-1'
          - name: 'smt-comp'
            args: 'gt smt-comp/non-incremental'
            tag:  '-2'
          - name: 'verus'
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: 'recursive'
      - name: Install Z3
        uses: pavpanchekha/setup-z3@1.2.2
        with:
          version: '4.8.7'
      - name: Cache cargo
        uses: Swatinem/rust-cache@v2
        with:
          shared-key: "shared"
          save-if: "false"
      - run: ./eval/build.sh
      - run: ./eval/eval.sh ${{ matrix.verifier }} ${{ matrix.verifier.args }}

      - name: Archive data for upload
        if: always()
        run: tar -czf ${{ matrix.verifier }}${{ matrix.verifier.tag }}.tar.bz2 eval/data/${{ matrix.verifier }}
      - uses: actions/upload-artifact@v4.4.3
        if: always()
        with:
          name: "${{ matrix.verifier }}${{ matrix.verifier.tag }}"
          path: ${{ matrix.verifier }}${{ matrix.verifier.tag }}.tar.bz2
          retention-days: 30