Skip to content

Commit

Permalink
Add consistency with US data package (#29)
Browse files Browse the repository at this point in the history
* Add improvements

* Minor code quality improvements

* Update URLs

* Use US actions
  • Loading branch information
nikhilwoodruff authored Oct 15, 2024
1 parent b7f7998 commit 634120f
Show file tree
Hide file tree
Showing 17 changed files with 352 additions and 184 deletions.
75 changes: 75 additions & 0 deletions .github/workflows/pull_request.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# This workflow is used to check various parts of the
# run prior to merging:

# - The code's formatting
# - Versioning - note that this checks, but does not update
# - That the code builds successfully
# - That any automated tests pass
name: Pull request

on:
pull_request:
branches: [main]

jobs:
lint:
runs-on: ubuntu-latest
name: Lint
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.12
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install black
- name: Check formatting
run: black . -l 79 --check
check-version:
name: Check version
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0 # Fetch all history for all tags and branches
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.12
- name: Build changelog
run: pip install "yaml-changelog>=0.1.7" && make changelog
- name: Preview changelog update
run: ".github/get-changelog-diff.sh"
- name: Check version number has been properly updated
run: ".github/is-version-number-acceptable.sh"
test:
name: Build and test
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0 # Fetch all history for all tags and branches
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.12
- name: Install package
run: make install
- name: Download data inputs
run: make download
env:
POLICYENGINE_UK_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_UK_DATA_GITHUB_TOKEN }}
- name: Build datasets
run: make data
env:
LITE_MODE: true
- name: Run tests
run: pytest
- name: Test documentation builds
run: make documentation
133 changes: 48 additions & 85 deletions .github/workflows/ci_cd.yaml → .github/workflows/push.yaml
Original file line number Diff line number Diff line change
@@ -1,60 +1,16 @@
name: CI/CD
# After successful versioning, this script runs various
# parts of the push process
name: Push

on:
pull_request:
branches: [main]
push:
branches: [main]
workflow_run:
workflows: ["Update versioning"]
types: [completed]

jobs:
publish-to-pypi:
name: Publish to PyPI
runs-on: ubuntu-latest
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0 # Fetch all history for all tags and branches
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.12
- name: Install package
run: pip install -e .
- name: Build package
run: python -m build
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
user: __token__
password: ${{ secrets.PYPI }}
skip-existing: true
publish-docs:
name: Publish documentation
runs-on: ubuntu-latest
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0 # Fetch all history for all tags and branches
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.12
- name: Install package
run: pip install -e ".[dev]"
- name: Build Jupyter Book
run: make documentation
- name: Deploy documentation
uses: JamesIves/github-pages-deploy-action@releases/v4
with:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BRANCH: gh-pages
FOLDER: docs/_build/html
lint:
runs-on: ubuntu-latest
if: ${{ github.event.workflow_run.conclusion == 'success' }}
name: Lint
steps:
- uses: actions/checkout@v4
Expand All @@ -68,10 +24,10 @@ jobs:
pip install black
- name: Check formatting
run: black . -l 79 --check

test:
name: Build and test
runs-on: ubuntu-latest
if: ${{ github.event.workflow_run.conclusion == 'success' }}
steps:
- name: Checkout code
uses: actions/checkout@v4
Expand All @@ -82,59 +38,66 @@ jobs:
with:
python-version: 3.12
- name: Install package
run: pip install -e ".[dev]"
run: make install
- name: Download data inputs
run: make download
env:
POLICYENGINE_UK_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_UK_DATA_GITHUB_TOKEN }}
- name: Build datasets
run: make data
env:
LITE_MODE: true
- name: Run tests
run: pytest
- name: Test documentation builds
run: make documentation

- name: Build Jupyter Book
run: make documentation
- name: Deploy documentation
uses: JamesIves/github-pages-deploy-action@releases/v4
with:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BRANCH: gh-pages
FOLDER: docs/_build/html
publish-to-pypi:
name: Publish to PyPI
runs-on: ubuntu-latest
if: ${{ github.event.workflow_run.conclusion == 'success' }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0 # Fetch all history for all tags and branches
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.12
- name: Install package
run: make install
- name: Build package
run: python -m build
- name: Publish a git tag
run: ".github/publish-git-tag.sh || true"
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
user: __token__
password: ${{ secrets.PYPI }}
skip-existing: true
docker:
name: Publish docker image
name: Docker
runs-on: ubuntu-latest
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
if: ${{ github.event.workflow_run.conclusion == 'success' }}
steps:
- name: Checkout repo
uses: actions/checkout@v4
- name: Log in to the Container registry
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{github.actor}}
password: ${{secrets.GITHUB_TOKEN}}
- name: Build container
run: docker build . -f docker/policyengine_uk_data.Dockerfile -t ghcr.io/policyengine/policyengine-uk-data:latest
- name: Push container
run: docker push ghcr.io/policyengine/policyengine-uk-data:latest

upload:
name: Upload data
runs-on: ubuntu-latest
needs: [lint, test]
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0 # Fetch all history for all tags and branches
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.12
- name: Install package
run: pip install -e ".[dev]"
- name: Download data inputs
run: make download
env:
POLICYENGINE_UK_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_UK_DATA_GITHUB_TOKEN }}
- name: Build datasets
run: make data
- name: Upload data
run: make upload
env:
POLICYENGINE_UK_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_UK_DATA_GITHUB_TOKEN }}
run: docker push ghcr.io/policyengine/policyengine-uk-data:latest
42 changes: 42 additions & 0 deletions .github/workflows/update_versioning.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# This builds and pushes the changelog, then
# automatically runs push_2 using the new version number
# to allow for proper pushing to PyPI.

# This script must run first and complete to allow for
# proper versioning.

name: Update versioning

on:
push:
branches: [main]

jobs:
check-version:
name: Check version
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0 # Fetch all history for all tags and branches
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.ref }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.12
- name: Build changelog
run: pip install "yaml-changelog>=0.1.7" && make changelog
- name: Preview changelog update
run: ".github/get-changelog-diff.sh"
- name: Check version number has been properly updated
run: ".github/is-version-number-acceptable.sh"
- name: Update changelog
uses: EndBug/add-and-commit@v9
with:
add: "."
committer_name: Github Actions[bot]
author_name: Github Actions[bot]
message: Update PolicyEngine US data
github_token: ${{ secrets.POLICYENGINE_US_DATA_GITHUB_TOKEN }}
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ test:
pytest

install:
pip install policyengine-uk==2.1.1
pip install -e ".[dev]"

download:
Expand Down
5 changes: 5 additions & 0 deletions changelog_entry.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
- bump: minor
changes:
added:
- 5% dropout in calibration.
- Code quality improvements.
109 changes: 52 additions & 57 deletions docs/validation.ipynb

Large diffs are not rendered by default.

Loading

0 comments on commit 634120f

Please sign in to comment.