Skip to content

Commit

Permalink
Merge pull request #47 from hostcc/fix/python-deps-build-cache
Browse files Browse the repository at this point in the history 
fix: Properly collect Python dependencies during image build. Next attempt at build cache
  • Loading branch information
@hostcc
hostcc authored Oct 7, 2024
2 parents ee70b20 + ee0174b commit 4d6d9ce
Show file tree
Hide file tree
Showing 4 changed files with 225 additions and 94 deletions.
260 changes: 181 additions & 79 deletions .github/workflows/main.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,32 @@ on:
- master

jobs:
version:
name: Generate package version
runs-on: ubuntu-latest
outputs:
value: ${{ steps.package-version.outputs.value }}
steps:
- name: Checkout the code
uses: actions/checkout@v4
with:
# Disable shallow clone for `setuptools_scm`, as it needs access to the
# history
fetch-depth: 0

- name: Set Python up
uses: actions/setup-python@v5

- name: Install dependencies
run: >-
python -m pip install --upgrade setuptools setuptools_scm
- name: Determine package version
id: package-version
run: |
package_version=`python -m setuptools_scm --format plain`
echo "value=$package_version" >> $GITHUB_OUTPUT
tests:
name: Tests
strategy:
Expand All @@ -33,31 +59,31 @@ jobs:
python: '3.12'
toxenv: py
runs-on: ${{ matrix.os }}
outputs:
version: ${{ steps.package-version.outputs.VALUE }}
needs: [version]
steps:
- name: Checkout the code
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
# Disable shallow clone for Sonar scanner, as it needs access to the
# history
fetch-depth: 0

- name: Set Python up
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}

- name: Install testing tools
run: >-
python -m pip install --upgrade setuptools pip tox virtualenv coverage
python -m pip install --upgrade \
setuptools setuptools_scm pip tox virtualenv coverage
- name: Run the tests
run: tox -e ${{ matrix.toxenv }}

- name: Generage Coverage combined XML report
run: coverage xml
- name: Determine package version
id: package-version
run: |
package_version=`cat version.txt`
echo "VALUE=$package_version" >> $GITHUB_OUTPUT

- name: SonarCloud scanning
uses: sonarsource/sonarcloud-github-action@master
env:
Expand All @@ -68,73 +94,26 @@ jobs:
args: >-
-Dsonar.projectKey=${{ github.repository_owner }}_${{ github.event.repository.name }}
-Dsonar.organization=${{ github.repository_owner }}
-Dsonar.projectVersion=${{ steps.package-version.outputs.VALUE }}
-Dsonar.projectVersion=${{ needs.version.outputs.value }}
# yamllint enable rule:line-length

pypi-publish:
name: Publish to PyPi
runs-on: ubuntu-latest
# PyPi disallows to publish packages with direct dependencies (GitHub
# sourced dependency in this case), so disable publishing for now
if: false
needs: [tests]
steps:
- name: Checkout the code
uses: actions/checkout@v3
with:
fetch-depth: 0 # `setuptools_scm` needs tags
- name: Set Python up
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Install the PEP517 package builder
run: python -m pip install --upgrade build
- name: Build the package
run: python -m build
- name: Publish the package to Test PyPi
# Skip publishing to test PyPI if we're performing release, there might
# be already the version of the package from the merge to master branch
if: github.event_name != 'release'
uses: pypa/gh-action-pypi-publish@release/v1
with:
password: ${{ secrets.TEST_PYPI_TOKEN }}
repository_url: https://test.pypi.org/legacy/
- name: Publish the release to PyPi
# Publish to production PyPi only happens when a release published out
# of the main branch
if: >-
github.event_name == 'release'
&& github.event.action == 'published'
&& (github.event.release.target_commitish == 'main'
|| github.event.release.target_commitish == 'master')
uses: pypa/gh-action-pypi-publish@release/v1
with:
password: ${{ secrets.PYPI_TOKEN }}

docker-publish:
name: Build and publish Docker images
docker-metadata:
name: Generate metadata for container images
runs-on: ubuntu-latest
needs: [tests]
permissions:
contents: read
packages: write
needs: [version]
outputs:
version: ${{ steps.meta.outputs.version }}
labels: ${{ steps.meta.outputs.labels }}
annotations: ${{ steps.meta.outputs.annotations }}
json: ${{ steps.meta.outputs.json }}
steps:
- name: Checkout the code
uses: actions/checkout@v3

- name: Set up QEMU for more platforms supported by Buildx
uses: docker/setup-qemu-action@v3

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Prepare Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ghcr.io/${{ github.repository }}
tags: |
type=pep440,pattern={{raw}},value=${{ needs.tests.outputs.version }}
type=pep440,pattern={{raw}},value=${{ needs.version.outputs.value }}
type=raw,value=latest,enable=${{
github.event_name == 'release'
&& github.event.action == 'published'
Expand All @@ -144,22 +123,145 @@ jobs:
type=ref,event=pr
type=edge
docker-publish:
# The job uses platform as variations, since `buildx` can't properly cache
# those if done single shot (multiple platform specified to single command
# invocation)
name: Build and publish Docker images
strategy:
fail-fast: false
matrix:
include:
- platform_id: linux/arm/v7
platform_name: linux-arm-v7
- platform_id: linux/arm/v6
platform_name: linux-arm-v6
- platform_id: linux/arm64
platform_name: linux-arm64
- platform_id: linux/amd64
platform_name: linux-amd64
runs-on: ubuntu-latest
needs: [version, tests, docker-metadata]
permissions:
contents: read
packages: write
steps:
- name: Checkout the code
uses: actions/checkout@v4

- name: Set up QEMU for more platforms supported by Buildx
uses: docker/setup-qemu-action@v3

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Build and push images
uses: docker/build-push-action@v6
id: build
with:
# No explicit context used, since that makes cache misses most of the
# time.
# See https://github.com/docker/build-push-action/issues/286 for more
# details
platforms: ${{ matrix.platform_id }}
labels: ${{ needs.docker-metadata.outputs.labels }}
annotations: ${{ needs.docker-metadata.outputs.annotations }}
# Implicit context points to working copy, not Git respository, so
# `setuptools_scm` needs to receive the version explicitly
build-args: |
VERSION=${{ needs.version.outputs.value }}
# Push by digest only, manifest will be added later
outputs: >-
type=image,name=ghcr.io/${{ github.repository }},push-by-digest=true,name-canonical=true,push=true
# Cache the buildx cache between builds using GitHub registry. `gha`
# cache has been mentioned to introduce cache misses for
# multi-platform builds, see https://github.com/docker/buildx/discussions/1382
# for potential hints
cache-from: |
type=registry,ref=ghcr.io/${{ github.repository }}/buildcache:${{ matrix.platform_name }}
cache-to: |
type=registry,ref=ghcr.io/${{ github.repository }}/buildcache:${{ matrix.platform_name }},mode=max
- name: Store image information
uses: GoCodeAlone/github-action-matrix-outputs-write@v1
id: out
with:
matrix-step-name: ${{ github.job }}
matrix-key: ${{ matrix.platform_name }}
outputs: |-
image_digest:
value: ${{ steps.build.outputs.digest }}
docker-manifest:
# The job uses image for for variations, hence each corresponding manifest
# is created separately - multiple tags in single command invocation might
# result in GHCR errors (not fully confirmed)
name: Create and push Docker manifest
runs-on: ubuntu-latest
needs: [docker-metadata, docker-publish]
strategy:
fail-fast: false
matrix:
tag: ${{ fromJson(needs.docker-metadata.outputs.json).tags }}
steps:
- name: Read image information from publish job
uses: GoCodeAlone/github-action-matrix-outputs-read@v1
id: read
with:
matrix-step-name: docker-publish

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Login to GitHub Container Registry
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Build and push images
uses: docker/build-push-action@v6
# The token above should have read/write access
# (`Settings` -> `Actions` -> `General` -> `Workflow permissions` -> `Read and write permissions`)
- name: Create and push Docker manifest
run: >-
docker buildx imagetools create
--tag ${{ matrix.tag }}
${{ join(fromJson(steps.read.outputs.result).image_digest.*.value, ' ') }}
docker-test:
name: Test Docker images
runs-on: ubuntu-latest
needs: [docker-metadata, docker-manifest]
strategy:
fail-fast: false
matrix:
include:
- platform_id: linux/arm/v7
platform_name: linux-arm-v7
- platform_id: linux/arm/v6
platform_name: linux-arm-v6
- platform_id: linux/arm64
platform_name: linux-arm64
- platform_id: linux/amd64
platform_name: linux-amd64
steps:
- name: Set up QEMU for more platforms supported by Buildx
uses: docker/setup-qemu-action@v3
with:
context: .
platforms: linux/arm/v7,linux/arm/v6,linux/arm64
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
annotations: ${{ steps.meta.outputs.annotations }}
# Cache the buildx cache between builds using GitHub Actions cache
cache-from: type=gha
cache-to: type=gha,mode=max
platforms: ${{ matrix.platform_id }}

- name: Test the image
# Running the image with `--help` should be sufficient to ensure all
# dependencies are present
run: >-
docker run --rm
--platform ${{ matrix.platform_id }}
ghcr.io/${{ github.repository }}:${{ needs.docker-metadata.outputs.version }}
--help
43 changes: 35 additions & 8 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,48 @@
FROM python:3.12.5-alpine AS build
COPY . /usr/src/
WORKDIR /usr/src/
FROM python:3.12.5-alpine AS deps

# Rust and Cargo are required to build `pyndatic-core` on ARM platforms
RUN apk add -U cargo git rust \
&& pip install build \
&& apk cache clean

# Limit use of the build context to the requirements file only, to avoid cache
# invalidation when other files get changed
COPY requirements.txt .
# Install dependencies in a separate layer to cache them
RUN --mount=type=cache,target=/root/.cache/pip pip install -r requirements.txt
RUN pip install --root /tmp/target/ -r requirements.txt

FROM python:3.12.5-alpine AS build

RUN pip install build

# Build the package
RUN python -m build \
&& pip install --root target/ dist/*-`cat version.txt`*.whl
ARG VERSION
RUN test -z "${VERSION}" && echo "No 'VERSION' argument provided, exiting" \
&& exit 1 || true

# Writeable mount is needed for src/*.egg-info the `setup` module wants to
# create. `pip install --no-deps` is to skip installing dependencies to the
# package thus requiring extra prerequisites and extending the build time -
# those already fulfilled by `deps` stage
RUN \
--mount=type=bind,target=source/,rw \
SETUPTOOLS_SCM_PRETEND_VERSION_FOR_ENERGOMERA_HASS_MQTT=${VERSION} \
python -m build --outdir /tmp/dist/ source/ \
&& pip install --no-deps --root /tmp/target/ /tmp/dist/*-${VERSION}*.whl

FROM python:3.12.5-alpine
# Ensure all the OS updates are applied to the resulting image
RUN apk -U upgrade \
&& apk cache clean

COPY --from=deps \
/tmp/target/usr/local/lib/ \
/usr/local/lib/
COPY --from=build \
/usr/src/target/root/.local/lib/ /usr/local/lib/
/tmp/target/usr/local/lib/ \
/usr/local/lib/
COPY --from=build \
/usr/src/target/root/.local/bin/ \
/tmp/target/usr/local/bin/ \
/usr/local/bin/

ENTRYPOINT ["energomera-hass-mqtt"]
15 changes: 9 additions & 6 deletions README.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,15 @@ Usage

.. code::
usage: energomera-hass-mqtt [-h] [-c CONFIG_FILE]
optional arguments:
-h, --help show this help message and exit
-c CONFIG_FILE, --config-file CONFIG_FILE
Path to configuration file (default: '/etc/energomera/config.yaml')
usage: energomera-hass-mqtt [-h] [-c CONFIG_FILE] [-a] [-d] [-o]
options:
-h, --help show this help message and exit
-c CONFIG_FILE, --config-file CONFIG_FILE
Path to configuration file (default: '/etc/energomera/config.yaml')
-a, --dry-run Dry run, do not actually send any data
-d, --debug Enable debug logging
-o, --one-shot Run only once, then exit
Configuration file format
=========================
Expand Down
Loading

0 comments on commit 4d6d9ce

Please sign in to comment.