Revise local building and testing system. (#289)

.gitignore

@@ -276,3 +276,4 @@ submit_test
 # Custom.
 Pipfile.lock
 .doit*
+docker-bake.override.json

README.md

@@ -33,8 +33,12 @@ For manual testing, you can start the images with `doit up`, however please refe
 
 ### Test on different architectures
 
-Images are built for multiple architectures (amd64 and aarch64) during continuous integration and pushed to the GitHub Container Registry (ghcr.io).
-To run automated tests or manual test against those images, simply specify the registry with the aformentioned `up` and `tests` commands (e.g. `doit up --registry=ghcr.io/).
+The build system will attempt to detect the local architecture and automatically build images for it (tested with amd64 and arm64).
+All commands `build`, `tests`, and `up` will use the locally detected platform and use a version tag based on the state of the local git repository.
+However, you can also specify a custom platform or version with the `--platform` and `--version` parameters, example: `doit up --platform=linux/amd64 --version=my-version`.
+
+Further, images are built for linux/amd64 and linux/arm64 during continuous integration for all pull-requests into the default branch and pushed to the GitHub Container Registry (ghcr.io).
+You can run automated or manual tests against those images by specifying the registry and version for both the `up` and `tests` commands, example: `doit up --registry=ghcr.io/ --version=pr-123`.
 Note: You may have to [log into the registry first](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#authenticating-to-the-container-registry).
 
 ## Run AiiDAlab in production

bumpver.toml

@@ -13,9 +13,3 @@ push = false
 "build.json" = [
     '"default": "{version}"'
 ]
-"docker-compose.base-with-services.yml" = [
-    "aiidalab/base-with-services:{version}"
-]
-"docker-compose.lab.yml" = [
-    "aiidalab/lab:{version}"
-]

docker-compose.base-with-services.yml

@@ -4,7 +4,7 @@ version: '3.4'
 services:
 
     aiidalab:
-        image: ${REGISTRY:-}${BASE_WITH_SERVICES_IMAGE:-aiidalab/base-with-services:2022.1001}
+        image: ${REGISTRY:-}${BASE_WITH_SERVICES_IMAGE:-aiidalab/base-with-services}${VERSION:-}
         environment:
             TZ: Europe/Zurich
             DOCKER_STACKS_JUPYTER_CMD: notebook

docker-compose.lab.yml

@@ -25,7 +25,7 @@ services:
             - aiida-rmq-data:/var/lib/rabbitmq/
 
     aiidalab:
-        image: ${REGISTRY:-}${LAB_IMAGE:-aiidalab/lab:2022.1001}
+        image: ${REGISTRY:-}${LAB_IMAGE:-aiidalab/lab}${VERSION:-}
         environment:
             RMQHOST: messaging
             TZ: Europe/Zurich

dodo.py

@@ -1,23 +1,65 @@
+import json
+import platform
 from pathlib import Path
 
+import docker
+from dunamai import Version
+
+_DOCKER_CLIENT = docker.from_env()
+_DOCKER_ARCHITECTURE = _DOCKER_CLIENT.info()["Architecture"]
+
 DOIT_CONFIG = {"default_tasks": ["build"]}
 
+VERSION = Version.from_git().serialize(dirty=True).replace("+", "_")
+PLATFORM = {"aarch64": "linux/arm64"}.get(_DOCKER_ARCHITECTURE, _DOCKER_ARCHITECTURE)
+
+
 _REGISTRY_PARAM = {
     "name": "registry",
     "short": "r",
     "long": "registry",
     "type": str,
-    "default": "docker.io/",
+    "default": "",
     "help": "Specify the docker image registry.",
 }
 
+_VERSION_PARAM = {
+    "name": "version",
+    "long": "version",
+    "type": "str",
+    "default": VERSION,
+    "help": (
+        "Specify the version of the stack for building / testing. Defaults to a "
+        "version determined from the state of the local git repository."
+    ),
+}
+
 
 def task_build():
     """Build all docker images."""
 
+    def generate_version_override(version):
+        Path("docker-bake.override.json").write_text(json.dumps(dict(VERSION=version)))
+
     return {
-        "actions": ["docker buildx bake -f docker-bake.hcl -f build.json"],
-        "params": [_REGISTRY_PARAM],
+        "actions": [
+            generate_version_override,
+            "docker buildx bake -f docker-bake.hcl -f build.json "
+            "-f docker-bake.override.json "
+            "--set '*.platform=%(platform)s' "
+            "--load",
+        ],
+        "params": [
+            _REGISTRY_PARAM,
+            _VERSION_PARAM,
+            {
+                "name": "platform",
+                "long": "platform",
+                "type": str,
+                "default": PLATFORM or "linux/amd64",
+                "help": "Specify the platform to build for. Examples: linux/amd64 linux/arm64",
+            },
+        ],
         "verbosity": 2,
     }
 
@@ -26,8 +68,8 @@ def task_tests():
     """Run tests with pytest."""
 
     return {
-        "actions": ["REGISTRY=%(registry)s pytest -v"],
-        "params": [_REGISTRY_PARAM],
+        "actions": ["REGISTRY=%(registry)s VERSION=:%(version)s pytest -v"],
+        "params": [_REGISTRY_PARAM, _VERSION_PARAM],
         "verbosity": 2,
     }
 
@@ -36,7 +78,8 @@ def task_up():
     """Start AiiDAlab server for testing."""
     return {
         "actions": [
-            "AIIDALAB_PORT=%(port)i REGISTRY=%(registry)s docker-compose up --detach"
+            "AIIDALAB_PORT=%(port)i REGISTRY=%(registry)s VERSION=:%(version)s "
+            "docker-compose up --detach"
         ],
         "params": [
             {
@@ -48,6 +91,7 @@ def task_up():
                 "help": "Specify the AiiDAlab host port.",
             },
             _REGISTRY_PARAM,
+            _VERSION_PARAM,
         ],
         "verbosity": 2,
     }

environment.yml

@@ -6,6 +6,7 @@ dependencies:
     - bumpver=2022.1118
     - docker-compose=1.29.2
     - doit=0.36.0
+    - dunamai=1.13.0
     - pip=22.2.2
     - pytest=7.1.2
     - pip: