Refactored image publishing and tests

.github/workflows/pipeline.yml

@@ -40,7 +40,7 @@ jobs:
       matrix:
         python_version: ["3.9.18", "3.10.13", "3.11.5"]
         os_variant: ["bookworm", "slim-bookworm"]
-        poetry_version: ["1.4.2", "1.5.1", "1.6.1"]    
+        poetry_version: ["1.4.2", "1.5.1", "1.6.1"]
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v3
@@ -65,7 +65,7 @@ jobs:
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: |
           poetry install --no-interaction --no-root
-      - name: Run all tests with pytest
+      - name: Run tests for image builds with pytest
         env:
           PYTHON_VERSION: ${{ matrix.python_version }}
           OS_VARIANT: ${{ matrix.os_variant }}
@@ -82,6 +82,11 @@ jobs:
   run-publish-image-tests:
     needs: code-quality
     runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python_version: ["3.11.5"]
+        os_variant: ["slim-bookworm"]
+        poetry_version: ["1.6.1"]
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v3
@@ -94,7 +99,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
-          version: 1.4.1
+          version: 1.6.1
           virtualenvs-in-project: true
       - name: Load cached venv
         id: cached-poetry-dependencies
@@ -106,7 +111,11 @@ jobs:
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: |
           poetry install --no-interaction --no-root
-      - name: Run all tests with pytest
+      - name: Run tests for image publishing with pytest
+        env:
+          PYTHON_VERSION: ${{ matrix.python_version }}
+          OS_VARIANT: ${{ matrix.os_variant }}
+          POETRY_VERSION: ${{ matrix.poetry_version }}
         run: |
           source .venv/bin/activate
           pytest tests/publish_image --cov

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ repos:
       - id: detect-private-key
       - id: check-added-large-files
   - repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: 23.9.1
     hooks:
       - id: black
   - repo: local

build/publish.py

@@ -1,6 +1,6 @@
 import click
 
-from build.utils import get_docker_bake_file, get_context
+from build.utils import get_context
 from pathlib import Path
 from python_on_whales import DockerClient, Builder
 
@@ -17,9 +17,29 @@
     help="Docker Hub password",
 )
 @click.option(
-    "--version-tag", envvar="GIT_TAG_NAME", required=True, help="Version Tag"
+    "--version-tag", envvar="GIT_TAG_NAME", required=True, help="Version tag"
+)
+@click.option(
+    "--python-version",
+    envvar="PYTHON_VERSION",
+    required=True,
+    help="Python version",
+)
+@click.option(
+    "--os-variant",
+    envvar="OS_VARIANT",
+    required=True,
+    help="Operating system variant",
+)
+@click.option(
+    "--poetry-version",
+    envvar="POETRY_VERSION",
+    required=True,
+    help="Poetry version",
+)
+@click.option(
+    "--registry", envvar="REGISTRY", default="docker.io", help="Docker registry"
 )
-@click.option("--registry", envvar="REGISTRY", help="Docker registry")
 @click.option(
     "--use-local-cache-storage-backend",
     envvar="USE_LOCAL_CACHE_STORAGE_BACKEND",
@@ -30,25 +50,23 @@ def main(
     docker_hub_username: str,
     docker_hub_password: str,
     version_tag: str,
+    poetry_version: str,
+    python_version: str,
+    os_variant: str,
     registry: str,
     use_local_cache_storage_backend: bool,
 ) -> None:
     context: Path = get_context()
-    bake_file: Path = get_docker_bake_file()
-    variables: dict = {
-        "CONTEXT": str(context),
-        "IMAGE_VERSION": version_tag,
-    }
 
-    bake_file_overrides: dict = {}
-    if use_local_cache_storage_backend:
-        bake_file_overrides = {
-            "*.cache-to": "type=local,mode=max,dest=/tmp",
-            "*.cache-from": "type=local,src=/tmp",
-        }
+    tag: str = f"{registry}/pfeiffermax/python-poetry:{version_tag}-poetry{poetry_version}-python{python_version}-{os_variant}"
 
-    if registry:
-        variables["REGISTRY"] = registry
+    platforms: list[str] = ["linux/amd64", "linux/arm64/v8"]
+    cache_to: str = "type=gha,mode=max"
+    cache_from: str = "type=gha"
+
+    if use_local_cache_storage_backend:
+        cache_to = "type=local,mode=max,dest=/tmp"
+        cache_from = "type=local,src=/tmp"
 
     docker_client: DockerClient = DockerClient()
     builder: Builder = docker_client.buildx.create(
@@ -60,15 +78,21 @@ def main(
         username=docker_hub_username,
         password=docker_hub_password,
     )
-    build_config: dict = docker_client.buildx.bake(
-        targets=["python-poetry"],
+
+    docker_client.buildx.build(
+        context_path=context,
+        target="production-image",
+        build_args={
+            "POETRY_VERSION": poetry_version,
+            "OFFICIAL_PYTHON_IMAGE": f"python:{python_version}-{os_variant}",
+        },
+        tags=tag,
+        platforms=platforms,
         builder=builder,
-        files=[bake_file],
-        variables=variables,
-        set=bake_file_overrides,
+        cache_to=cache_to,
+        cache_from=cache_from,
         push=True,
     )
-    print(build_config)
 
     # Cleanup
     docker_client.buildx.stop(builder)

tests/build_image/conftest.py

@@ -9,7 +9,6 @@
     VERSION,
 )
 from tests.registry_container import DockerRegistryContainer
-from tests.utils import extract_image_references_from_build_config
 from os import getenv
 
 
@@ -57,12 +56,12 @@ def image_reference(
     docker_client.buildx.build(
         context_path=CONTEXT,
         target="production-image",
-        tags=tag,
-        platforms=platforms,
         build_args={
             "POETRY_VERSION": poetry_version,
             "OFFICIAL_PYTHON_IMAGE": f"python:{python_version}-{os_variant}",
         },
+        tags=tag,
+        platforms=platforms,
         builder=pow_buildx_builder,
         cache_to=cache_to,
         cache_from=cache_from,

tests/publish_image/conftest.py

@@ -1,9 +1,27 @@
 import pytest
 from click.testing import CliRunner
-from python_on_whales import DockerClient, Image
+from os import getenv
 
 
 @pytest.fixture(scope="package")
 def cli_runner() -> CliRunner:
     runner = CliRunner()
     return runner
+
+
+@pytest.fixture(scope="package")
+def python_version() -> str:
+    python_version: str = getenv("PYTHON_VERSION")
+    return python_version
+
+
+@pytest.fixture(scope="package")
+def os_variant() -> str:
+    os_variant: str = getenv("OS_VARIANT")
+    return os_variant
+
+
+@pytest.fixture(scope="package")
+def poetry_version() -> str:
+    poetry_version: str = getenv("POETRY_VERSION")
+    return poetry_version

tests/publish_image/test_cli.py

@@ -6,7 +6,12 @@
 from tests.registry_container import DockerRegistryContainer
 
 
-def test_registry_with_credentials(cli_runner: CliRunner):
+def test_registry_with_credentials(
+    cli_runner: CliRunner,
+    python_version: str,
+    os_variant: str,
+    poetry_version: str,
+):
     with DockerRegistryContainer(
         username=REGISTRY_USERNAME, password=REGISTRY_PASSWORD
     ).with_bind_ports(5000, 5000) as docker_registry:
@@ -19,14 +24,25 @@ def test_registry_with_credentials(cli_runner: CliRunner):
                 REGISTRY_PASSWORD,
                 "--version-tag",
                 VERSION,
+                "--python-version",
+                python_version,
+                "--os-variant",
+                os_variant,
+                "--poetry-version",
+                poetry_version,
                 "--registry",
                 docker_registry.get_registry(),
             ],
         )
         assert result.exit_code == 0
 
 
-def test_registry_with_wrong_credentials(cli_runner: CliRunner):
+def test_registry_with_wrong_credentials(
+    cli_runner: CliRunner,
+    python_version: str,
+    os_variant: str,
+    poetry_version: str,
+):
     with DockerRegistryContainer(
         username=REGISTRY_USERNAME, password=REGISTRY_PASSWORD
     ).with_bind_ports(5000, 5000) as docker_registry:
@@ -39,6 +55,12 @@ def test_registry_with_wrong_credentials(cli_runner: CliRunner):
                 "boom",
                 "--version-tag",
                 VERSION,
+                "--python-version",
+                python_version,
+                "--os-variant",
+                os_variant,
+                "--poetry-version",
+                poetry_version,
                 "--registry",
                 docker_registry.get_registry(),
             ],

tests/publish_image/test_env.py

@@ -6,7 +6,12 @@
 from tests.registry_container import DockerRegistryContainer
 
 
-def test_registry_with_credentials(cli_runner: CliRunner):
+def test_registry_with_credentials(
+    cli_runner: CliRunner,
+    python_version: str,
+    os_variant: str,
+    poetry_version: str,
+):
     with DockerRegistryContainer(
         username=REGISTRY_USERNAME, password=REGISTRY_PASSWORD
     ).with_bind_ports(5000, 5000) as docker_registry:
@@ -16,13 +21,21 @@ def test_registry_with_credentials(cli_runner: CliRunner):
                 "DOCKER_HUB_USERNAME": REGISTRY_USERNAME,
                 "DOCKER_HUB_PASSWORD": REGISTRY_PASSWORD,
                 "GIT_TAG_NAME": VERSION,
+                "PYTHON_VERSION": python_version,
+                "OS_VARIANT": os_variant,
+                "POETRY_VERSION": poetry_version,
                 "REGISTRY": docker_registry.get_registry(),
             },
         )
         assert result.exit_code == 0
 
 
-def test_registry_with_wrong_credentials(cli_runner: CliRunner):
+def test_registry_with_wrong_credentials(
+    cli_runner: CliRunner,
+    python_version: str,
+    os_variant: str,
+    poetry_version: str,
+):
     with DockerRegistryContainer(
         username=REGISTRY_USERNAME, password=REGISTRY_PASSWORD
     ).with_bind_ports(5000, 5000) as docker_registry:
@@ -32,6 +45,9 @@ def test_registry_with_wrong_credentials(cli_runner: CliRunner):
                 "DOCKER_HUB_USERNAME": "boom",
                 "DOCKER_HUB_PASSWORD": "bang",
                 "GIT_TAG_NAME": VERSION,
+                "PYTHON_VERSION": python_version,
+                "OS_VARIANT": os_variant,
+                "POETRY_VERSION": poetry_version,
                 "REGISTRY": docker_registry.get_registry(),
             },
         )