Better caching #284
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Python CI | |
| on: | |
| push: | |
| branches: [] | |
| paths-ignore: | |
| - 'README.md' | |
| tags: | |
| - 'v*' | |
| pull_request: | |
| branches: [] | |
| paths-ignore: | |
| - 'README.md' | |
| # Specify concurrency such that only one workflow can run at a time | |
| # * Different workflow files are not affected | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| # Registry for storing Container images | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| # Ensure the GitHub token can remove packages | |
| permissions: | |
| packages: write | |
| jobs: | |
| # Define a "test" job that runs on all branches and PRs | |
| # * Incorporates unittests and integration tests | |
| test: | |
| runs-on: ubuntu-latest | |
| container: quay.io/condaforge/miniforge3:latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| # Should mirror the build-venv stage in the Containerfile | |
| - name: Build venv | |
| run: | | |
| apt update && apt install -y build-essential | |
| conda create -p /venv python=3.10 | |
| /venv/bin/pip install --upgrade pip | |
| conda install -p /venv -y eccodes | |
| - name: Install dependencies | |
| run: /venv/bin/pip install -e .[dev] | |
| - name: Cache dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pipdev-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pipdev- | |
| # Run unittests on all branches and PRs | |
| # * Produce JUnit XML report | |
| - name: Run unit tests | |
| run: /venv/bin/python -m xmlrunner discover -s src/nwp_consumer -p "test_*.py" --output-file ut-report.xml | |
| # Run integration tests on main and PRs to it | |
| # * Requires secrets to be set in the repository settings | |
| # * Produce JUnit XML report | |
| - name: Run integration tests | |
| env: | |
| RAW_DIR: "/tmp/raw" | |
| ZARR_DIR: "/tmp/zarr" | |
| CEDA_FTP_PASS: ${{ secrets.CEDA_FTP_PASS }} | |
| CEDA_FTP_USER: ${{ secrets.CEDA_FTP_USER }} | |
| METOFFICE_CLIENT_ID: ${{ secrets.METOFFICE_CLIENT_ID }} | |
| METOFFICE_CLIENT_SECRET: ${{ secrets.METOFFICE_CLIENT_SECRET }} | |
| METOFFICE_ORDER_ID: ${{ secrets.METOFFICE_ORDER_ID }} | |
| if: | | |
| (github.ref == 'refs/heads/main' && github.event_name != 'tag') || | |
| (github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'main') | |
| run: /venv/bin/python -m xmlrunner discover -s src/test_integration -p "test_*.py" --output-file it-report.xml | |
| # Create test summary to be visualised on the job summary screen on GitHub | |
| # * Runs even if previous steps fail | |
| - name: Create test summary | |
| uses: test-summary/action@v2 | |
| with: | |
| paths: "*t-report.xml" | |
| show: "fail, skip" | |
| if: always() | |
| # Define a "build-container" job that runs on branch commits only | |
| # * Builds and pushes an OCI Container image to the registry defined in the environment variables | |
| # * Only runs if test job passes | |
| build-container: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| needs: test | |
| if: github.event_name != 'pull_request' | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Set up Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| # Tag the built image according to the event type | |
| # * If the event is a valid version tag, use the tag name | |
| # * If the event is a branch commit, use the commit sha | |
| - name: Extract metadata (tags, labels) for Container | |
| id: meta | |
| uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| tags: | | |
| type=ref,event=branch | |
| type=semver,pattern={{version}} | |
| # Build and push the Container image to the registry | |
| # * Creates a multiplatform-aware image | |
| # * Semantic versioning is handled via the meta action | |
| # * The image layers are cached between action runs | |
| - name: Build and push Container image | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| file: Containerfile | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| platforms: linux/amd64,linux/arm64 | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| # Define a "build-wheel" job that runs on main branch commits and tags only | |
| # * Only runs if test job passes | |
| build-wheel: | |
| runs-on: ubuntu-latest | |
| container: quay.io/condaforge/miniforge3:latest | |
| needs: test | |
| if: | | |
| (github.ref == 'refs/heads/main' || contains(github.ref, 'refs/tags/v')) && | |
| github.event_name != 'pull_request' | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| # Building the wheel dynamically assigns the version according to git | |
| # * The setuptools_git_versioning package reads the git tags and assigns the version | |
| # * The version is then used in the wheel filename and made available in the package | |
| # * setuptools_git_versioning is configured in pyproject.toml | |
| - name: Build wheel | |
| run: | | |
| apt update && apt install -y build-essential | |
| conda install -y -c eccodes | |
| python -m pip install --upgrade pip wheel | |
| pip install . | |
| pip wheel . --no-deps --wheel-dir dist | |
| - name: Cache dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Upload wheel | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheel | |
| path: dist/*.whl |