Using xdist to run pytest in parallel #4849
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test-install | |
| on: | |
| pull_request: | |
| paths: | |
| - environment.yml | |
| - '**/requirements*.txt' | |
| - pyproject.toml | |
| - util/dependency_management.py | |
| - .github/workflows/test-install.yml | |
| branches-ignore: [gh-pages] | |
| schedule: | |
| - cron: 30 02 * * * # nightly build | |
| # https://docs.github.com/en/actions/using-jobs/using-concurrency | |
| concurrency: | |
| # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| validate-dependency-specification: | |
| # Note: The specification is also validated by the pre-commit hook. | |
| if: github.repository == 'aiidateam/aiida-core' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.9 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: Install utils/ dependencies | |
| run: pip install -r utils/requirements.txt | |
| - name: Validate | |
| run: | | |
| python ./utils/dependency_management.py check-requirements | |
| python ./utils/dependency_management.py validate-all | |
| resolve-pip-dependencies: | |
| # Check whether the environments defined in the requirements/* files are | |
| # resolvable. | |
| needs: [validate-dependency-specification] | |
| if: github.repository == 'aiidateam/aiida-core' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ['3.9', '3.10', '3.11', '3.12'] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Resolve dependencies from requirements file. | |
| run: | | |
| pip install --dry-run --ignore-installed -r requirements/requirements-py-${{ matrix.python-version }}.txt | |
| create-conda-environment: | |
| # Verify that we can create a valid conda environment from the environment.yml file. | |
| needs: [validate-dependency-specification] | |
| if: github.repository == 'aiidateam/aiida-core' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Conda | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| channels: conda-forge | |
| - run: conda --version | |
| - name: Test conda environment | |
| run: | | |
| conda env create --dry-run -f environment.yml -n test-environment | |
| install-with-pip: | |
| if: github.repository == 'aiidateam/aiida-core' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| extras: ['', '[atomic_tools,docs,notebook,rest,tests,tui]'] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.9 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: Pip install | |
| id: pip_install | |
| run: | | |
| python -m pip --version | |
| python -m pip install -e .${{ matrix.extras }} | |
| python -m pip freeze | |
| - name: Test importing aiida | |
| if: steps.pip_install.outcome == 'success' | |
| run: python -c "import aiida" | |
| install-with-conda: | |
| # Verify that we can install AiiDA with conda. | |
| if: github.repository == 'aiidateam/aiida-core' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 25 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ['3.9', '3.10', '3.11', '3.12'] | |
| # Not being able to install with conda on a specific Python version is | |
| # not sufficient to fail the run, but something we want to be aware of. | |
| optional: [true] | |
| include: | |
| # Installing with conda without specyfing the Python version should | |
| # not fail since this is advocated as part of the user documentation. | |
| - python-version: '' | |
| optional: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Conda | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| channels: conda-forge | |
| # Use mamba because conda is running out of memory | |
| # see https://github.com/conda-incubator/setup-miniconda/issues/274 | |
| - run: | | |
| conda install -n base conda-libmamba-solver | |
| conda config --set solver libmamba | |
| # Temporary workaround: https://github.com/mamba-org/mamba/issues/488 | |
| - run: rm /usr/share/miniconda/pkgs/cache/*.json | |
| - name: Test installation | |
| id: test_installation | |
| continue-on-error: ${{ matrix.optional }} | |
| run: > | |
| conda create --dry-run -n test-install aiida-core | |
| ${{ matrix.python-version && format('python={0}', matrix.python-version) }} | |
| - name: Warn about failure | |
| if: steps.test_installation.outcome == 'Failure' | |
| run: > | |
| echo "::warning ::Failed conda installation for | |
| Python ${{ matrix.python-version }}." | |
| tests: | |
| needs: [install-with-pip] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ['3.9', '3.10', '3.11', '3.12'] | |
| services: | |
| postgres: | |
| image: postgres:10 | |
| env: | |
| POSTGRES_DB: test_aiida | |
| POSTGRES_PASSWORD: '' | |
| POSTGRES_HOST_AUTH_METHOD: trust | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| rabbitmq: | |
| image: rabbitmq:3.8.14-management | |
| ports: | |
| - 5672:5672 | |
| - 15672:15672 | |
| slurm: | |
| image: xenonmiddleware/slurm:17 | |
| ports: | |
| - 5001:22 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install system dependencies | |
| run: sudo apt update && sudo apt install postgresql graphviz | |
| - name: Install aiida-core | |
| uses: ./.github/actions/install-aiida-core | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| extras: '[atomic_tools,docs,notebook,rest,tests,tui]' | |
| from-requirements: 'false' | |
| - name: Setup AiiDA environment | |
| run: .github/workflows/setup.sh | |
| - name: Run test suite | |
| env: | |
| AIIDA_TEST_PROFILE: test_aiida | |
| AIIDA_WARN_v3: 1 | |
| run: pytest --db-backend psql tests -m 'not nightly' tests/ | |
| - name: Freeze test environment | |
| run: pip freeze | sed '1d' | tee requirements-py-${{ matrix.python-version }}.txt | |
| # Add python-version specific requirements/ file to the requirements.txt artifact. | |
| # This artifact can be used in the next step to automatically create a pull request | |
| # updating the requirements (in case they are inconsistent with the pyproject.toml file). | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: requirements-py-${{ matrix.python-version }}.txt | |
| path: requirements-py-${{ matrix.python-version }}.txt | |
| # Check whether the requirements/ files are consistent with the dependency specification in the pyproject.toml file. | |
| # If the check fails, warn the user via a comment and try to automatically create a pull request to update the files | |
| # (does not work on pull requests from forks). | |
| check-requirements: | |
| needs: tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.9 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.9 | |
| - name: Install utils/ dependencies | |
| run: pip install -r utils/requirements.txt | |
| - name: Check consistency of requirements/ files | |
| id: check_reqs | |
| continue-on-error: true | |
| run: python ./utils/dependency_management.py check-requirements DEFAULT --no-github-annotate | |
| # | |
| # The following steps are only executed if the consistency check failed. | |
| # | |
| - name: Create commit comment | |
| if: steps.check_reqs.outcome == 'Failure' # only run if requirements/ are inconsistent | |
| uses: peter-evans/commit-comment@v3 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| path: pyproject.toml | |
| body: | | |
| The requirements/ files are inconsistent! | |
| # Check out the base branch so that we can prepare the pull request. | |
| - name: Checkout base branch | |
| if: steps.check_reqs.outcome == 'Failure' # only run if requirements/ are inconsistent | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| clean: true | |
| - name: Download requirements.txt files | |
| if: steps.check_reqs.outcome == 'Failure' # only run if requirements/ are inconsistent | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: requirements-py-* | |
| merge-multiple: true | |
| path: requirements | |
| - name: Commit requirements files | |
| if: steps.check_reqs.outcome == 'Failure' # only run if requirements/ are inconsistent | |
| run: | | |
| git add requirements/* | |
| - name: Create pull request for updated requirements files | |
| if: steps.check_reqs.outcome == 'Failure' # only run if requirements/ are inconsistent | |
| id: create_update_requirements_pr | |
| continue-on-error: true | |
| uses: peter-evans/create-pull-request@v7 | |
| with: | |
| branch: update-requirements | |
| commit-message: Automated update of requirements/ files. | |
| title: Update requirements/ files. | |
| body: | | |
| Update requirements files to ensure that they are consistent | |
| with the dependencies specified in the 'pyproject.toml' file. | |
| Please note, that this pull request was likely created to | |
| resolve the inconsistency for a specific dependency, however | |
| other versions that have changed since the last update will | |
| be included as part of this commit as well. | |
| Click [here](https://github.com/aiidateam/aiida-core/wiki/AiiDA-Dependency-Management) for more information. | |
| - name: Create PR comment on success | |
| if: steps.create_update_requirements_pr.outcome == 'Success' | |
| uses: peter-evans/create-or-update-comment@v4 | |
| with: | |
| issue-number: ${{ github.event.number }} | |
| body: | | |
| I automatically created a pull request (#${{ steps.create_update_requirements_pr.outputs.pr_number }}) that adapts the | |
| requirements/ files according to the dependencies specified in the 'pyproject.toml' file. | |
| - name: Create PR comment on failure | |
| if: steps.create_update_requirements_pr.outcome == 'Failure' | |
| uses: peter-evans/create-or-update-comment@v4 | |
| with: | |
| issue-number: ${{ github.event.number }} | |
| body: | | |
| Please update the requirements/ files to ensure that they | |
| are consistent with the dependencies specified in the 'pyproject.toml' file. |