feat: automatic conda environment handling #560
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: : 2021-2024 The PyPSA-Eur Authors | |
| # | |
| # SPDX-License-Identifier: CC0-1.0 | |
| name: Test workflows | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| branches: | |
| - master | |
| schedule: | |
| - cron: "0 5 * * 1-6" | |
| - cron: "0 5 * * 0" | |
| # Cancel any in-progress runs when a new run is triggered | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| BASE_ENV: envs/environment.yaml | |
| jobs: | |
| run-tests: | |
| name: OS | |
| runs-on: ${{ matrix.os }}-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Run windows only on scheduled runs on Sundays, otherwise ignore | |
| os: ${{ github.event.schedule == '0 5 * * 0' && fromJson('["ubuntu", "macos", "windows"]') || fromJson('["ubuntu", "macos"]') }} | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup env file path (ubuntu) | |
| if: matrix.os == 'ubuntu' | |
| run: | | |
| echo "env_file=envs/linux-pinned.yaml" >> $GITHUB_ENV | |
| - name: Setup env file path (macos and windows) | |
| if: matrix.os != 'ubuntu' | |
| run: | | |
| echo "env_file=envs/${{ matrix.os }}-pinned.yaml" >> $GITHUB_ENV | |
| - name: Use base env file if it was changed | |
| run: | | |
| git fetch origin master | |
| if git diff --name-only origin/master | grep '${{ env.BASE_ENV }}'; then | |
| echo "Base env ${{ env.BASE_ENV }} changed. Using it instead of pinned envs." | |
| echo "env_file=${{ env.BASE_ENV }}" >> $GITHUB_ENV | |
| else | |
| echo "Base env ${{ env.BASE_ENV }} not changed. Using pinned envs." | |
| fi | |
| - name: Setup cache keys | |
| run: | | |
| echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| data | |
| cutouts | |
| key: data-cutouts-${{ env.WEEK }} | |
| - uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| activate-environment: pypsa-eur | |
| - name: Cache Conda env | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CONDA }}/envs | |
| key: conda-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles(format('{0}', env.env_file)) }} | |
| id: cache-env | |
| - name: Update environment | |
| if: steps.cache-env.outputs.cache-hit != 'true' | |
| run: | | |
| conda env update -n pypsa-eur -f ${{ env.env_file }} | |
| echo "Run conda list" && conda list | |
| - name: Run snakemake test workflows | |
| run: | | |
| make test envs=pinned # TODO: Remove custom env handling | |
| - name: Run unit tests | |
| run: | | |
| make unit-test | |
| - name: Upload artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: results-${{ matrix.os }} | |
| path: | | |
| logs | |
| .snakemake/log | |
| results | |
| retention-days: 3 | |
| run-tests-on-dev-deps: | |
| name: Inhouse | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| inhouse: | |
| - pypsa | |
| - atlite | |
| - powerplantmatching | |
| - linopy | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Use base env file if it was changed | |
| run: | | |
| git fetch origin master | |
| if git diff --name-only origin/master | grep '${{ env.BASE_ENV }}'; then | |
| echo "Base env ${{ env.BASE_ENV }} changed. Using it instead of pinned envs." | |
| echo "env_file=${{ env.BASE_ENV }}" >> $GITHUB_ENV | |
| else | |
| echo "Base env ${{ env.BASE_ENV }} not changed. Using pinned envs." | |
| echo "env_file=envs/linux-pinned.yaml" >> $GITHUB_ENV | |
| fi | |
| # Only run checks if package is not pinned | |
| - name: Check if inhouse package is pinned | |
| run: | | |
| grep_line=$(grep -- '- ${{ matrix.inhouse }}' ${{ env.BASE_ENV }}) | |
| if [[ $grep_line == *"<"* || $grep_line == *"=="* ]]; then | |
| echo "pinned=true" >> $GITHUB_ENV | |
| else | |
| echo "pinned=false" >> $GITHUB_ENV | |
| fi | |
| - name: Setup cache keys | |
| if: env.pinned == 'false' | |
| run: | | |
| echo "WEEK=$(date +'%Y%U')" >> $GITHUB_ENV # data and cutouts | |
| - uses: actions/cache@v4 | |
| if: env.pinned == 'false' | |
| with: | |
| path: | | |
| data | |
| cutouts | |
| key: data-cutouts-${{ env.WEEK }} | |
| - uses: conda-incubator/setup-miniconda@v3 | |
| if: env.pinned == 'false' | |
| with: | |
| activate-environment: pypsa-eur | |
| - name: Cache Conda env | |
| if: env.pinned == 'false' | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.CONDA }}/envs | |
| key: conda-${{ runner.os }}-${{ runner.arch }}-${{ matrix.inhouse }}-${{ hashFiles(format('{0}', env.env_file)) }} | |
| id: cache-env | |
| - name: Update environment | |
| if: env.pinned == 'false' && steps.cache-env.outputs.cache-hit != 'true' | |
| run: | | |
| conda env update -n pypsa-eur -f ${{ env.env_file }} | |
| echo "Run conda list" && conda list | |
| - name: Install inhouse packages from master | |
| if: env.pinned == 'false' | |
| run: | | |
| python -m pip install git+https://github.com/PyPSA/${{ matrix.inhouse }}.git@master | |
| - name: Run snakemake test workflows | |
| if: env.pinned == 'false' | |
| run: | | |
| make test | |
| - name: Upload artifacts | |
| if: env.pinned == 'false' && always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: results-inhouse-${{ matrix.inhouse }} | |
| path: | | |
| logs | |
| .snakemake/log | |
| results | |
| retention-days: 3 |