Skip to content
pytest

Clean up PUDL CLI tools; use Click framework #372

Sign in to view logs

Clean up PUDL CLI tools; use Click framework

Clean up PUDL CLI tools; use Click framework #372

Workflow file for this run

.github/workflows/pytest.yml at e60620d
---
name: pytest
on:
pull_request:
types:
- created
- opened
- synchronize
- ready_for_review
env:
PUDL_OUTPUT: /home/runner/pudl-work/output/
PUDL_INPUT: /home/runner/pudl-work/input/
DAGSTER_HOME: /home/runner/pudl-work/dagster_home/
jobs:
ci-docs:
runs-on: ubuntu-latest
strategy:
fail-fast: false
defaults:
run:
shell: bash -l {0}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Install conda-lock environment with micromamba
uses: mamba-org/setup-micromamba@v1
with:
environment-file: environments/conda-lock.yml
environment-name: pudl-dev
cache-environment: true
- name: Log environment details
run: |
conda info
conda list
conda config --show-sources
conda config --show
printenv | sort
- name: Make input, output and dagster dirs
run: mkdir -p ${{ env.PUDL_OUTPUT }} ${{ env.PUDL_INPUT}} ${{ env.DAGSTER_HOME }}
- name: Lint and build PUDL documentation with Sphinx
run: |
pip install --no-deps --editable .
make docs-build
- name: Upload coverage
uses: actions/upload-artifact@v3
with:
name: coverage-docs
path: coverage.xml
ci-unit:
runs-on: ubuntu-latest
strategy:
fail-fast: false
defaults:
run:
shell: bash -l {0}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Install conda-lock environment with micromamba
uses: mamba-org/setup-micromamba@v1
with:
environment-file: environments/conda-lock.yml
environment-name: pudl-dev
cache-environment: true
- name: Log environment details
run: |
conda info
conda list
conda config --show-sources
conda config --show
printenv | sort
- name: Make input, output and dagster dirs
run: mkdir -p ${{ env.PUDL_OUTPUT }} ${{ env.PUDL_INPUT}} ${{ env.DAGSTER_HOME }}
- name: Log SQLite3 version
run: |
which sqlite3
sqlite3 --version
- name: Run PUDL unit tests and collect test coverage
run: |
pip install --no-deps --editable .
make pytest-unit
- name: Upload coverage
uses: actions/upload-artifact@v3
with:
name: coverage-unit
path: coverage.xml
ci-integration:
runs-on:
group: large-runner-group
labels: ubuntu-22.04-4core
if: github.event.pull_request.draft == false
permissions:
contents: read
id-token: write
strategy:
fail-fast: false
defaults:
run:
shell: bash -l {0}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Install Conda environment using mamba
uses: mamba-org/setup-micromamba@v1
with:
environment-file: environments/conda-lock.yml
environment-name: pudl-dev
cache-environment: true
- name: Log environment details
run: |
conda info
conda list
conda config --show-sources
conda config --show
printenv | sort
- name: Log SQLite3 version
run: |
which sqlite3
sqlite3 --version
- name: Compile Zenodo datastore DOIs for cache invalidation
run:
grep -e '.*10\.\(5281\|5072\)/zenodo\..*' src/pudl/workspace/datastore.py
| sed -e 's/",*$//g' | sed -e 's/^.*"//g' | sort > datastore-dois.txt
- name: Restore Zenodo datastore from cache if possible
uses: actions/cache@v3
id: cache-zenodo-datastore
with:
path: ${{ env.PUDL_INPUT }}
key: zenodo-datastore-${{ hashFiles('datastore-dois.txt') }}
- name: Make input, output and dagster dirs
run: mkdir -p ${{ env.PUDL_OUTPUT }} ${{ env.PUDL_INPUT}} ${{ env.DAGSTER_HOME }}
- name: List workspace contents
run: find /home/runner/pudl-work
- name: Set default GCP credentials
id: gcloud-auth
continue-on-error: true
uses: "google-github-actions/auth@v1"
with:
workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider"
service_account: "tox-pytest-github-action@catalyst-cooperative-pudl.iam.gserviceaccount.com"
- name: Run integration tests, trying to use GCS cache if possible
run: |
pip install --no-deps --editable .
make pytest-integration
- name: Upload coverage
uses: actions/upload-artifact@v3
with:
name: coverage-integration
path: coverage.xml
- name: Log post-test Zenodo datastore contents
run: find ${{ env.PUDL_INPUT }}
ci-coverage:
runs-on: ubuntu-latest
needs:
- ci-docs
- ci-unit
- ci-integration
steps:
- uses: actions/checkout@v4
- name: Download coverage
id: download-coverage
uses: actions/download-artifact@v3
with:
path: coverage
- name: List downloaded files
run: |
ls -R
- name: Upload test coverage report to CodeCov
uses: codecov/codecov-action@v3
with:
directory: coverage
ci-notify:
runs-on: ubuntu-latest
if: ${{ always() }}
needs:
- ci-docs
- ci-unit
- ci-integration
steps:
- name: Inform the Codemonkeys
uses: 8398a7/action-slack@v3
continue-on-error: true
with:
status: custom
fields: workflow,job,commit,repo,ref,author,took
custom_payload: |
{
username: 'action-slack',
icon_emoji: ':octocat:',
attachments: [{
color: '${{ needs.ci-test.result }}' === 'success' ? 'good' : '${{ needs.ci-test.result }}' === 'failure' ? 'danger' : 'warning',
text: `${process.env.AS_REPO}@${process.env.AS_REF}\n ${process.env.AS_WORKFLOW} (${process.env.AS_COMMIT})\n by ${process.env.AS_AUTHOR}\n Status: ${{ needs.ci-test.result }}`,
}]
}
env:
GITHUB_TOKEN: ${{ github.token }} # required
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required
MATRIX_CONTEXT: ${{ toJson(matrix) }} # required