Update `run-pipeline-iteratively.sh to create local dir structure with subset of data #379
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| # Enable Buildkit and let compose use it to speed up image building | |
| env: | |
| DOCKER_BUILDKIT: 1 | |
| COMPOSE_DOCKER_CLI_BUILD: 1 | |
| CONDA_ENV_NAME: clim-recal | |
| # CONDA_ENV_PATH: conda-lock.yml | |
| CONDA_ENV_PATH: .conda-linux-64.lock | |
| MIN_PYTHON_VERSION: 3.12 | |
| PYTHON_MODULE_FOLDER: python/ | |
| COVERAGE_SVG_FOLDER: docs/assets/ | |
| COVERAGE_SVG_FILE_NAME: coverage.svg | |
| # replace below with references to previous config lines | |
| COVERAGE_SVG_PATH: docs/assets/coverage.svg | |
| GH_PAGE_PATH: _gh-page | |
| on: | |
| pull_request: | |
| branches: ['main', 'cache-cpm-for-hads', ] | |
| paths-ignore: ['docs/**'] | |
| push: | |
| branches: ['main', 'cache-cpm-for-hads', ] | |
| paths-ignore: ['docs/**'] | |
| concurrency: | |
| group: ${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: write | |
| jobs: | |
| linter: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Code Repository | |
| uses: actions/checkout@main | |
| - name: Set up Python | |
| uses: actions/setup-python@main | |
| with: | |
| python-version: ${{ env.MIN_PYTHON_VERSION }} | |
| - name: Run pre-commit | |
| uses: pre-commit/action@main | |
| - name: Update pre-commit | |
| uses: pre-commit-ci/lite-action@main | |
| if: always() | |
| pytest: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| steps: | |
| - name: Checkout Code Repository | |
| uses: actions/checkout@main | |
| - name: Build Conda Environment | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| activate-environment: ${{ env.CONDA_ENV_NAME }} | |
| environment-file: ${{ env.CONDA_ENV_PATH }} | |
| auto-activate-base: false | |
| miniforge-version: latest | |
| use-mamba: true | |
| - name: Run Python Tests | |
| # The first line below may be temporary, | |
| # see this solution: | |
| # https://github.com/pyenv/pyenv/issues/2625 | |
| run: | | |
| mamba install libsqlite --force-reinstall | |
| mamba run -n ${{ env.CONDA_ENV_NAME }} --cwd python pytest -m "not darwin and not mount" | |
| shell: bash -el {0} | |
| - name: Copy test coverage results | |
| run: | | |
| cp ${{ env.PYTHON_MODULE_FOLDER }}${{ env.COVERAGE_SVG_PATH }} ${{ env.COVERAGE_SVG_FOLDER }} | |
| - name: Archive coverage svg | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage-badge | |
| path: ${{ env.COVERAGE_SVG_PATH }} | |
| docs: | |
| needs: [linter, pytest] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Download coverage svg | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: coverage-badge | |
| path: ${{ env.GH_PAGE_PATH }}${{ env.COVERAGE_SVG_PATH }} | |
| - name: Build Conda Environment | |
| uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| activate-environment: ${{ env.CONDA_ENV_NAME }} | |
| environment-file: ${{ env.CONDA_ENV_PATH }} | |
| auto-activate-base: false | |
| miniforge-version: latest | |
| - name: Run `quartodoc` | |
| shell: bash -el {0} | |
| run: | | |
| mamba run -n ${{ env.CONDA_ENV_NAME }} quartodoc build | |
| - name: Set up Quarto | |
| uses: quarto-dev/quarto-actions/setup@v2 | |
| - name: Install Python and Dependencies | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.12' | |
| cache: 'pip' | |
| - run: pip install jupyter | |
| - name: Render and Publish | |
| uses: quarto-dev/quarto-actions/publish@v2 | |
| with: | |
| target: gh-pages | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # The below raises a code 137 error on `GitHub` Actions | |
| # docker: | |
| # needs: [linter, pytest] | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@main | |
| # | |
| # - name: Build and Test with Root permission | |
| # run: | | |
| # docker compose build jupyter | |
| # docker compose up jupyter --detach | |
| # docker compose exec jupyter bash -c "conda run -n ${{ env.CONDA_ENV_NAME }} --cwd python pytest" | |
| # | |
| # - name: Clean-up | |
| # run: docker compose down |