From 634120fe0d8b8e82ce3c0cc0c288d2886f2b0b5d Mon Sep 17 00:00:00 2001 From: Nikhil Woodruff <35577657+nikhilwoodruff@users.noreply.github.com> Date: Tue, 15 Oct 2024 10:41:04 +0100 Subject: [PATCH] Add consistency with US data package (#29) * Add improvements * Minor code quality improvements * Update URLs * Use US actions --- .github/workflows/pull_request.yaml | 75 ++++++++++ .github/workflows/{ci_cd.yaml => push.yaml} | 133 +++++++----------- .github/workflows/update_versioning.yaml | 42 ++++++ Makefile | 1 + changelog_entry.yaml | 5 + docs/validation.ipynb | 109 +++++++------- .../datasets/frs/enhanced_frs.py | 38 +++-- .../datasets/frs/extended_frs.py | 16 +-- policyengine_uk_data/datasets/frs/frs.py | 2 +- policyengine_uk_data/utils/__init__.py | 1 + .../utils/imputations/capital_gains.py | 9 +- .../utils/imputations/consumption.py | 6 +- .../utils/imputations/income.py | 8 +- policyengine_uk_data/utils/imputations/vat.py | 6 +- .../utils/imputations/wealth.py | 6 +- policyengine_uk_data/utils/qrf.py | 74 ++++++++++ pyproject.toml | 5 +- 17 files changed, 352 insertions(+), 184 deletions(-) create mode 100644 .github/workflows/pull_request.yaml rename .github/workflows/{ci_cd.yaml => push.yaml} (60%) create mode 100644 .github/workflows/update_versioning.yaml create mode 100644 policyengine_uk_data/utils/qrf.py diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml new file mode 100644 index 0000000..5a8e53a --- /dev/null +++ b/.github/workflows/pull_request.yaml @@ -0,0 +1,75 @@ +# This workflow is used to check various parts of the +# run prior to merging: + +# - The code's formatting +# - Versioning - note that this checks, but does not update +# - That the code builds successfully +# - That any automated tests pass +name: Pull request + +on: + pull_request: + branches: [main] + +jobs: + lint: + runs-on: ubuntu-latest + name: Lint + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install black + - name: Check formatting + run: black . -l 79 --check + check-version: + name: Check version + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for all tags and branches + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + - name: Build changelog + run: pip install "yaml-changelog>=0.1.7" && make changelog + - name: Preview changelog update + run: ".github/get-changelog-diff.sh" + - name: Check version number has been properly updated + run: ".github/is-version-number-acceptable.sh" + test: + name: Build and test + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for all tags and branches + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + - name: Install package + run: make install + - name: Download data inputs + run: make download + env: + POLICYENGINE_UK_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_UK_DATA_GITHUB_TOKEN }} + - name: Build datasets + run: make data + env: + LITE_MODE: true + - name: Run tests + run: pytest + - name: Test documentation builds + run: make documentation diff --git a/.github/workflows/ci_cd.yaml b/.github/workflows/push.yaml similarity index 60% rename from .github/workflows/ci_cd.yaml rename to .github/workflows/push.yaml index 271525c..f22487f 100644 --- a/.github/workflows/ci_cd.yaml +++ b/.github/workflows/push.yaml @@ -1,60 +1,16 @@ -name: CI/CD +# After successful versioning, this script runs various +# parts of the push process +name: Push on: - pull_request: - branches: [main] - push: - branches: [main] + workflow_run: + workflows: ["Update versioning"] + types: [completed] jobs: - publish-to-pypi: - name: Publish to PyPI - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history for all tags and branches - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Install package - run: pip install -e . - - name: Build package - run: python -m build - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.PYPI }} - skip-existing: true - publish-docs: - name: Publish documentation - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history for all tags and branches - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Install package - run: pip install -e ".[dev]" - - name: Build Jupyter Book - run: make documentation - - name: Deploy documentation - uses: JamesIves/github-pages-deploy-action@releases/v4 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: gh-pages - FOLDER: docs/_build/html lint: runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} name: Lint steps: - uses: actions/checkout@v4 @@ -68,10 +24,10 @@ jobs: pip install black - name: Check formatting run: black . -l 79 --check - test: name: Build and test runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -82,27 +38,61 @@ jobs: with: python-version: 3.12 - name: Install package - run: pip install -e ".[dev]" + run: make install - name: Download data inputs run: make download env: POLICYENGINE_UK_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_UK_DATA_GITHUB_TOKEN }} - name: Build datasets run: make data + env: + LITE_MODE: true - name: Run tests run: pytest - name: Test documentation builds run: make documentation - + - name: Build Jupyter Book + run: make documentation + - name: Deploy documentation + uses: JamesIves/github-pages-deploy-action@releases/v4 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: gh-pages + FOLDER: docs/_build/html + publish-to-pypi: + name: Publish to PyPI + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'success' }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for all tags and branches + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + - name: Install package + run: make install + - name: Build package + run: python -m build + - name: Publish a git tag + run: ".github/publish-git-tag.sh || true" + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI }} + skip-existing: true docker: - name: Publish docker image + name: Docker runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - name: Checkout repo uses: actions/checkout@v4 - name: Log in to the Container registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{github.actor}} @@ -110,31 +100,4 @@ jobs: - name: Build container run: docker build . -f docker/policyengine_uk_data.Dockerfile -t ghcr.io/policyengine/policyengine-uk-data:latest - name: Push container - run: docker push ghcr.io/policyengine/policyengine-uk-data:latest - - upload: - name: Upload data - runs-on: ubuntu-latest - needs: [lint, test] - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history for all tags and branches - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - name: Install package - run: pip install -e ".[dev]" - - name: Download data inputs - run: make download - env: - POLICYENGINE_UK_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_UK_DATA_GITHUB_TOKEN }} - - name: Build datasets - run: make data - - name: Upload data - run: make upload - env: - POLICYENGINE_UK_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_UK_DATA_GITHUB_TOKEN }} \ No newline at end of file + run: docker push ghcr.io/policyengine/policyengine-uk-data:latest \ No newline at end of file diff --git a/.github/workflows/update_versioning.yaml b/.github/workflows/update_versioning.yaml new file mode 100644 index 0000000..d5fd509 --- /dev/null +++ b/.github/workflows/update_versioning.yaml @@ -0,0 +1,42 @@ +# This builds and pushes the changelog, then +# automatically runs push_2 using the new version number +# to allow for proper pushing to PyPI. + +# This script must run first and complete to allow for +# proper versioning. + +name: Update versioning + +on: + push: + branches: [main] + +jobs: + check-version: + name: Check version + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for all tags and branches + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.ref }} + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + - name: Build changelog + run: pip install "yaml-changelog>=0.1.7" && make changelog + - name: Preview changelog update + run: ".github/get-changelog-diff.sh" + - name: Check version number has been properly updated + run: ".github/is-version-number-acceptable.sh" + - name: Update changelog + uses: EndBug/add-and-commit@v9 + with: + add: "." + committer_name: Github Actions[bot] + author_name: Github Actions[bot] + message: Update PolicyEngine US data + github_token: ${{ secrets.POLICYENGINE_US_DATA_GITHUB_TOKEN }} \ No newline at end of file diff --git a/Makefile b/Makefile index 975f206..f56d1e7 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ test: pytest install: + pip install policyengine-uk==2.1.1 pip install -e ".[dev]" download: diff --git a/changelog_entry.yaml b/changelog_entry.yaml index e69de29..af5448c 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -0,0 +1,5 @@ +- bump: minor + changes: + added: + - 5% dropout in calibration. + - Code quality improvements. diff --git a/docs/validation.ipynb b/docs/validation.ipynb index 545b83f..0c4bdd4 100644 --- a/docs/validation.ipynb +++ b/docs/validation.ipynb @@ -53,7 +53,7 @@ "text/html": [ "\n", - "
\n", + "
\n", "
\n", "\n", @@ -147,13 +147,13 @@ "\n", "
\n", "
\n", - "This is the init_notebook_mode cell from ITables v2.1.5
\n", + "This is the init_notebook_mode cell from ITables v2.2.1
\n", "(you should not see this message - is your notebook trusted?)\n", "
\n", "
\n", "\n" ], "text/plain": [ @@ -165,11 +165,6 @@ } ], "source": [ - "from policyengine_uk_data.utils.download_docs_prerequisites import (\n", - " download_data,\n", - ")\n", - "\n", - "download_data()\n", "from policyengine_uk_data import EnhancedFRS_2022_23, FRS_2022_23, SPI_2020_21\n", "from policyengine_uk_data.utils.loss import get_loss_results\n", "import pandas as pd\n", @@ -215,13 +210,13 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "\n", + "
\n", "\n", " \n", " \n", @@ -330,21 +325,21 @@ "\n", "\n", "
\n", - "Loading ITables v2.1.5 from the init_notebook_mode cell...\n", + "Loading ITables v2.2.1 from the init_notebook_mode cell...\n", "(need help?)\n", "
\n", "\n", "\n", "
\n", "