Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/dev' into parallel-pytest
Browse files Browse the repository at this point in the history
  • Loading branch information
rousik committed Nov 27, 2023
2 parents b08ead3 + 649db02 commit a23f7eb
Show file tree
Hide file tree
Showing 118 changed files with 27,462 additions and 1,845 deletions.
1 change: 1 addition & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
*.ipynb linguist-detectable=false
*.html linguist-detectable=false
eia861-transform.ipynb merge=ours
environments/conda-*lock.yml merge=ours
*.csv text
*.py text
*.json text
Expand Down
8 changes: 0 additions & 8 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -1,14 +1,6 @@
---
version: 2
updates:
- package-ecosystem: pip
directory: /
target-branch: dev
labels:
- dependencies
schedule:
interval: weekly

- package-ecosystem: github-actions
directory: /
target-branch: dev
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/bot-auto-merge.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ name: bot-auto-merge
on:
workflow_run:
types: [completed]
workflows: ["tox-pytest"]
workflows: ["pytest"]

jobs:
bot-auto-merge:
Expand Down
12 changes: 7 additions & 5 deletions .github/workflows/build-deploy-pudl.yml
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,12 @@ jobs:
gcloud compute instances update-container "$GCE_INSTANCE" \
--zone "$GCE_INSTANCE_ZONE" \
--container-image "docker.io/catalystcoop/pudl-etl:${{ env.GITHUB_REF }}" \
--container-command "conda" \
--container-command "micromamba" \
--container-arg="run" \
--container-arg="--no-capture-output" \
--container-arg="-p" \
--container-arg="/home/catalyst/env" \
--container-arg="--prefix" \
--container-arg="/home/mambauser/env" \
--container-arg="--attach" \
--container-arg='' \
--container-arg="bash" \
--container-arg="./docker/gcp_pudl_etl.sh" \
--container-env-file="./docker/.env" \
Expand All @@ -116,7 +117,8 @@ jobs:
--container-env DAGSTER_PG_PASSWORD="$DAGSTER_PG_PASSWORD" \
--container-env DAGSTER_PG_HOST="104.154.182.24" \
--container-env DAGSTER_PG_DB="dagster-storage" \
--container-env PUDL_SETTINGS_YML="/home/catalyst/src/pudl/package_data/settings/etl_full.yml" \
--container-env FLY_ACCESS_TOKEN=${{ secrets.FLY_ACCESS_TOKEN }} \
--container-env PUDL_SETTINGS_YML="/home/mambauser/src/pudl/package_data/settings/etl_full.yml" \
# Start the VM
- name: Start the deploy-pudl-vm
Expand Down
61 changes: 23 additions & 38 deletions .github/workflows/tox-pytest.yml → .github/workflows/pytest.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
---
name: Code validation
# TODO(rousik): We could name this "Code tests", as that would
# better represent what it does. Maybe even migrate yml file.

on:
pull_request:
Expand All @@ -16,22 +14,17 @@ concurrency:

env:
PUDL_OUTPUT: /home/runner/pudl-work/output/
PUDL_INPUT: /home/runner/pudl-work/data/
PUDL_INPUT: /home/runner/pudl-work/input/
DAGSTER_HOME: /home/runner/pudl-work/dagster_home/
ETL_CONFIG: src/pudl/package_data/settings/etl_fast.yml
ETL_COMMANDLINE_OPTIONS: --gcs-cache-path=gs://zenodo-cache.catalyst.coop
COVERAGE_OPTIONS: --concurrency=multiprocessing --parallel-mode

jobs:
ci-static:
name: Static tests
ci-docs:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
tox-env:
- linters
- docs
defaults:
run:
shell: bash -l {0}
Expand All @@ -41,16 +34,12 @@ jobs:
with:
fetch-depth: 2

- name: Install Conda environment using mamba
- name: Install conda-lock environment with micromamba
uses: mamba-org/setup-micromamba@v1
with:
environment-file: test/test-environment.yml
environment-file: environments/conda-lock.yml
environment-name: pudl-dev
cache-environment: true
condarc: |
channels:
- conda-forge
- defaults
channel_priority: strict

- name: Log environment details
run: |
Expand All @@ -60,13 +49,13 @@ jobs:
conda config --show
printenv | sort
- name: Build ${{ matrix.tox-env}} with Tox
- name: Lint and build PUDL documentation with Sphinx
run: |
tox -e ${{ matrix.tox-env }}
pip install --no-deps --editable .
make docs-build
- name: Upload coverage
uses: actions/upload-artifact@v3
if: ${{ matrix.tox-env == 'docs' }}
with:
name: coverage-docs
path: coverage.xml
Expand All @@ -84,16 +73,12 @@ jobs:
with:
fetch-depth: 2

- name: Install Conda environment using mamba
- name: Install conda-lock environment with micromamba
uses: mamba-org/setup-micromamba@v1
with:
environment-file: test/test-environment.yml
environment-file: environments/conda-lock.yml
environment-name: pudl-dev
cache-environment: true
condarc: |
channels:
- conda-forge
- defaults
channel_priority: strict

- name: Log environment details
run: |
Expand All @@ -108,9 +93,10 @@ jobs:
which sqlite3
sqlite3 --version
- name: Run unit tests with Tox
- name: Run PUDL unit tests and collect test coverage
run: |
tox -e unit -- --durations 0
pip install --no-deps --editable .
make pytest-unit
- name: Upload coverage
uses: actions/upload-artifact@v3
Expand All @@ -134,18 +120,17 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Install Conda environment using mamba

- name: Install conda-lock environment with micromamba
uses: mamba-org/setup-micromamba@v1
with:
environment-file: test/test-environment.yml
environment-file: environments/conda-lock.yml
environment-name: pudl-dev
cache-environment: true
condarc: |
channels:
- conda-forge
- defaults
channel_priority: strict

- name: Install PUDL and its dependencies
run: pip install .[test,datasette]
run: pip install --no-deps --editable .[test,datasette]

- name: Log environment details
run: |
conda info
Expand Down Expand Up @@ -213,13 +198,13 @@ jobs:
name: Upload coverage to CodeCov
runs-on: ubuntu-latest
needs:
- ci-docs
- ci-unit
- ci-integration
- ci-static
steps:
- uses: actions/checkout@v4
- name: Download coverage
id: download-unit
id: download-coverage
uses: actions/download-artifact@v3
with:
path: coverage
Expand Down
85 changes: 85 additions & 0 deletions .github/workflows/update-conda-lockfile.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
---
name: update-conda-lockfile

on:
workflow_dispatch:
schedule:
- cron: "0 9 * * 1" # Mondays at 9AM UTC
push:
paths:
- "Makefile"
- "pyproject.toml"
- "environments/*"
- ".github/workflows/update-conda-lockfile.yml"

# What branch does this action run on?
# - workflow_dispatch: Whatever branch it was run against.
# - schedule: Always the same branch (will be dev or main)
# - push: Base branch of the PR.

jobs:
update-conda-lockfile:
runs-on: ubuntu-latest
if: ${{ (github.event_name == 'push' && github.actor != 'pudlbot') || (github.event_name == 'schedule' && github.repository == 'catalyst-cooperative/pudl') || (github.event_name == 'workflow_dispatch') }}
defaults:
run:
shell: bash -l {0}
steps:
- name: Set GITHUB_REF for use with workflow_dispatch
if: ${{ (github.event_name == 'workflow_dispatch') }}
run: |
echo "GITHUB_REF="${{ github.ref_name }} >> $GITHUB_ENV
- name: Set GITHUB_REF for use with schedule
if: ${{ (github.event_name == 'schedule') }}
run: |
echo "GITHUB_REF=dev" >> $GITHUB_ENV
- name: Set GITHUB_REF for use with push
if: ${{ (github.event_name == 'push') }}
run: |
echo "GITHUB_REF="${{ github.ref_name }} >> $GITHUB_ENV
- name: Log final value of GITHUB_REF
run: |
echo "Final GITHUB_REF:" ${{ env.GITHUB_REF }}
- uses: actions/checkout@v4
with:
token: ${{ secrets.PUDL_BOT_PAT }}
ref: ${{ env.GITHUB_REF }}
- name: Install Micromamba
uses: mamba-org/setup-micromamba@v1
with:
environment-name: conda-lock
create-args: >-
python=3.11
conda-lock>=2.5.1
prettier
- name: Run conda-lock to recreate lockfile from scratch
run: |
make conda-clean
make conda-lock.yml
- name: Commit updated conda lockfiles to branch
# If running on push due to dependency changes, commit directly to the base
# branch of the existing PR. Don't trigger the workflow again if we're already
# running it as pudlbot (to avoid infinite recursion).
if: ${{ (github.event_name == 'push' && github.actor != 'pudlbot') }}
uses: stefanzweifel/git-auto-commit-action@v5
with:
file_pattern: "environments/*"
commit_message: "Update conda-lock.yml and rendered conda environment files."
- name: Make a PR to merge updated conda lockfiles
# If we are relocking dependencies on a schedule or workflow_dispatch, we need
# to make our own PR to check whether the updated environment actually solves
# and the tests pass.
if: ${{ (github.event_name == 'schedule' && github.repository == 'catalyst-cooperative/pudl') || (github.event_name == 'workflow_dispatch') }}
uses: peter-evans/create-pull-request@v5
with:
token: ${{ secrets.PUDL_BOT_PAT }}
commit-message: "Update conda-lock.yml and rendered conda environment files."
title: Update Lockfile
body: >
This pull request relocks the dependencies with conda-lock.
It is triggered by [update-conda-lockfile](https://github.com/catalyst-cooperative/pudl/blob/main/.github/workflows/update-conda-lockfile.yml).
labels: dependencies, conda-lock
reviewers: zaneselvans
branch: update-conda-lockfile
base: ${{ env.GITHUB_REF }}
delete-branch: true
10 changes: 3 additions & 7 deletions .github/workflows/zenodo-cache-sync.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ env:
PUBLIC_ZENODO_CACHE_BUCKET: gs://zenodo-cache.catalyst.coop
GITHUB_REF: ${{ github.ref_name }} # This is changed to dev if running on a schedule
PUDL_OUTPUT: ~/pudl-work/output
PUDL_INPUT: ~/pudl-work/data/
PUDL_INPUT: ~/pudl-work/input/

jobs:
zenodo-cache-sync:
Expand Down Expand Up @@ -47,13 +47,9 @@ jobs:
- name: Install Conda environment using mamba
uses: mamba-org/setup-micromamba@v1
with:
environment-file: test/test-environment.yml
environment-file: environments/conda-lock.yml
environment-name: pudl-dev
cache-environment: true
condarc: |
channels:
- conda-forge
- defaults
channel_priority: strict

- name: Log environment details
run: |
Expand Down
10 changes: 8 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ codecov.sh
.env_pudl/
*wheel-metadata
dask-worker-space*
devtools/user-requirements.txt
devtools/user-environment.yml
environments/user-requirements.txt
environments/user-environment.yml
.vscode/*
commit.txt
devtools/profiles/
Expand All @@ -38,3 +38,9 @@ notebooks/*.tgz
terraform/.terraform/*
.env
.hypothesis/

# generated by datasette/publish.py fresh for every deploy - we shouldn't track changes.
devtools/datasette/fly/Dockerfile
devtools/datasette/fly/inspect-data.json
devtools/datasette/fly/metadata.yml
devtools/datasette/fly/all_dbs.tar.zst
13 changes: 3 additions & 10 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,21 +26,14 @@ repos:
# Formatters: hooks that re-write Python & documentation files
####################################################################################
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.1.4
rev: v0.1.6
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix]

# Format the code
- repo: https://github.com/psf/black-pre-commit-mirror
rev: 23.10.1
hooks:
- id: black
language_version: python3.11
exclude: migrations/.*
- id: ruff-format

- repo: https://github.com/pre-commit/mirrors-prettier
rev: v3.0.3
rev: v3.1.0
hooks:
- id: prettier
types_or: [yaml]
Expand Down
6 changes: 2 additions & 4 deletions .readthedocs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ version: 2
build:
os: ubuntu-22.04
tools:
python: mambaforge-4.10
python: mambaforge-22.9

# Define the python environment using conda / mamba
conda:
environment: docs/docs-environment.yml
environment: environments/conda-linux-64.lock.yml

# Build documentation in the docs/ directory with Sphinx
sphinx:
Expand All @@ -27,5 +27,3 @@ python:
install:
- method: pip
path: .
extra_requirements:
- doc
14 changes: 0 additions & 14 deletions MANIFEST.in

This file was deleted.

Loading

0 comments on commit a23f7eb

Please sign in to comment.