test_benchmarks.yml

name: Test Benchmark

on:
  workflow_call:
    inputs:
      benchopt_branch:
        description: Branch of benchopt to install to test the benchmark.
        default: benchopt@main
        required: false
        type: string
      benchopt_version:
        description: |
          If set, use a specific version of benchopt for the tests,
          thus ignoring the benchopt_branch input.
        default: git
        required: false
        type: string
      python_version:
        description: Python version to use for the tests.
        default: "3.10"
        required: false
        type: string
      extra_args:
        description: |
          Extra arguments to be passed in benchopt test.
        default: ""
        required: false
        type: string
      benchmark_dir:
        description: |
          Directory where the benchmark is located. This can be used to run
          the tests for benchmarks that are sub-folders of a repo.
        default: "."
        required: false
        type: string

# Cancel in-progress workflows when pushing
# a new commit on the same branch

jobs:
  test-benchmark:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: ['ubuntu-latest', 'macos-latest']
        exclude:
          # Only run OSX test on version==git, not on the release ones.
          - os: ${{ inputs.benchopt_version == 'git' || 'macos-latest' }}
    env:
      CONDA_ENV: 'test_env'
      BENCHOPT_BRANCH: ${{ inputs.benchopt_branch }}
      BENCHOPT_VERSION: ${{ inputs.benchopt_version }}
      PYTEST_EXTRA_ARGS: ${{ inputs.extra_args }}
      BENCHOPT_DEBUG: 1
      BENCHOPT_CONDA_CMD: mamba

    concurrency:
      group: ${{ github.workflow }}-${{ github.job }}-${{ matrix.os }}-${{ inputs.benchopt_version }}-${{ inputs.extra_args }}-${{ github.ref }}
      cancel-in-progress: true

    defaults:
      run:
        # Need to use this shell to get conda working properly.
        # See https://github.com/marketplace/actions/setup-miniconda#important
        shell: bash -l {0}

    steps:
    - uses: actions/checkout@v4
    - name: Setup Conda
      uses: conda-incubator/setup-miniconda@v3
      with:
        auto-update-conda: true
        miniforge-version: latest
        mamba-version: "*"
        use-mamba: true
        channels: conda-forge
        python-version: ${{ inputs.python_version }}
        activate-environment: ${{ env.CONDA_ENV }}

    - run: conda info

    - name: Install benchopt and its dependencies
      run: |
        conda info
        mamba install -yq pip

        # Get the correct branch of benchopt
        if [[ "$BENCHOPT_VERSION" == "git" ]]
        then
          user=${BENCHOPT_BRANCH%@*}
          branch=${BENCHOPT_BRANCH##*@}
          pip install -U git+https://github.com/$user/benchopt@$branch
        elif [[ "$BENCHOPT_VERSION" == "latest" ]]
        then
          pip install -U benchopt
        else
          pip install -U benchopt==$BENCHOPT_VERSION
        fi
    - name: Check if benchopt is compatible with this benchmark
      id: check_min_version
      run: |
          min_version=$(grep -Po 'min_benchopt_version = "\K[^"]*' ${{ inputs.benchmark_dir }}/objective.py || echo "0.0")
          if [[ "$BENCHOPT_VERSION" == "git" ]]
          then
            # Always test dev version
            benchopt_version="99.0"
          else
            benchopt_version=$(benchopt --version)
          fi
          echo "$benchopt_version and $min_version"
          if [[ "$benchopt_version" < "$min_version" ]]
          then
            echo "not compatible"
            echo "compatible=false" >> $GITHUB_OUTPUT
          else
            echo "compatible"
            echo "compatible=true" >> $GITHUB_OUTPUT
          fi

    - name: Test
      if: ${{ steps.check_min_version.outputs.compatible == 'true' }}
      run: |
        benchopt test ${{ inputs.benchmark_dir }} --env-name bench_test_env -vl $PYTEST_EXTRA_ARGS
        # Avoid having too large cache between two runs
        benchopt clean ${{ inputs.benchmark_dir }}
        benchopt test ${{ inputs.benchmark_dir }} --env-name bench_test_env -vl --skip-install $PYTEST_EXTRA_ARGS