batou_ext.oci: Add support for podman

CHANGES.d/20241129_131748_mb_FC_37959_podman.md

@@ -0,0 +1,13 @@
+- `batou_ext.oci.Container`: allow to use `podman` as backend instead of `docker`.
+  This also enables the following features:
+
+  * Rootless containers: by setting the `user` option to a different user. By default,
+    the service user of the deployment is used.
+
+  * Only mark services as `active` if the container is up. Which metric is used
+    for that can be configured using the `sd_notify` option:
+
+    * `container`: the `sd_notify(3)` call must be sent from inside the container.
+    * `healthy`: if the container's health check(s) are green.
+    * `conmon`: if the container is up (but not necessarily the service in it).
+    * `ignore`: just mark the unit as `active`.

src/batou_ext/oci.py

@@ -1,3 +1,4 @@
+import json
 import os
 import shlex
 from textwrap import dedent
@@ -12,11 +13,38 @@
 import batou_ext.nix
 
 
+class PodmanRuntime(Component):
+    """
+    Marker to indicate that containers are running with podman instead of
+    docker.
+
+    The backend can only be set globally in NixOS, so this is done with a single,
+    global component here.
+    """
+
+    def configure(self):
+        self.provide("oci:podman", self)
+        self += File(
+            "/etc/local/nixos/oci-backend.nix",
+            content=dedent(
+                """\
+        {
+          virtualisation.podman.enable = true;
+        }
+        """
+            ),
+        )
+
+
 class Container(Component):
     """A OCI Container component.
 
-    With this component you can dynamically schedule docker containers to be
-    run on the target host.
+    With this component you can dynamically schedule docker or podman containers
+    to be run on the target host.
+
+    Note: the `podman` backend is still considered experimental and thus subject
+    to change. Running both docker and podman containers on the same VM is currently
+    not supported.
 
     Note: Docker image specifiers do not follow a properly resolvable pattern.
     Therefore, container registries have to be specified seperately if you need
@@ -70,17 +98,60 @@ class Container(Component):
 
     self += Rebuild()
     self += container.activate()
+    ```
+
+    Containers can use `podman` as backend by adding the `PodmanRuntime`
+    component:
+    ```
+    self += batou_ext.oci.PodmanRuntime()
+    self += batou_ext.oci.Container(
+        image="mysql"
+    )
+    ```
 
+    If a container image has a healthcheck configured and the systemd unit
+    is supposed to be activate _after_ the container went healthy, this
+    can be done using the `podman` backend by specifying the `sd_notify`
+    parameter:
 
+    ```
+    self += batou_ext.oci.PodmanRuntime()
+    self += batou_ext.oci.Container(
+        image="with-healthchecks",
+        sd_notify="healthy"
+    )
+    ```
 
+    By default, the unit gets into active state with `podman` when the container
+    is started, but not necessarily the service. This is still an improvement
+    over how the docker unit is handled which is "active" as soon as the unit
+    was started which means the service could still be busy downloading the image.
+
+    If a podman container doesn't have a healthcheck defined, it's possible to add
+    one via this component. The command is passed `/bin/sh -c`:
+
+    ```
+    self += batou_ext.oci.PodmanRuntime()
+    self += batou_ext.oci.Container(
+        image="without-healthcheck",
+        sd_notify="healthy",
+        health_cmd="curl --fail localhost || exit 1"
+    )
+    ```
 
+    Please note that the healthcheck is executed _inside_ the container, so
+    the container above would require a `curl` installed.
     """
 
     # general options
     image = Attribute(str)
     version: str = "latest"
     container_name = Attribute(str)
 
+    sd_notify = Attribute(str, None)
+    health_cmd = Attribute(str, None)
+    user = Attribute(str, None)
+
     # Set up monitoring
     monitor: bool = True
 
@@ -95,7 +166,7 @@ class Container(Component):
     ports: dict = {}
     env: dict = {}
     depends_on: list = None
-    extra_options: list = None
+    extra_options: list = []
 
     # secrets
     registry_address = Attribute(Optional[str], None)
@@ -108,6 +179,12 @@ class Container(Component):
     }
 
     def configure(self):
+        self.backend = (
+            "podman"
+            if self.require("oci:podman", strict=False, host=self.host)
+            else "docker"
+        )
+
         if (
             self.registry_user or self.registry_password
         ) and not self.registry_address:
@@ -145,6 +222,28 @@ def configure(self):
         if self.docker_cmd:
             self._docker_cmd_list = shlex.split(self.docker_cmd)
 
+        if self.backend != "podman":
+            for prop in ["sd_notify", "user", "health_cmd"]:
+                assert (
+                    getattr(self, prop) is None
+                ), f"Container '{self.container_name}' runs with Docker, so the '{prop}' option is not supported!"
+        else:
+            if self.sd_notify is not None:
+                assert self.sd_notify in [
+                    "conmon",
+                    "healthy",
+                    "ignore",
+                ], f"Container '{self.container_name}' set invalid value for 'sd_notify'. Allowed values: container, conmon, healthy, ignore!"
+
+            if self.health_cmd is not None:
+                # `json.dumps` quotes the quoted string in a way that it can
+                # be placed as valid string into a Nix expression.
+                # The bash-quoting is done in the OCI module in NixOS.
+                self.health_cmd = json.dumps(self.health_cmd)
+
+            if self.user is None:
+                self.user = self.host.service_user
+
         if not self.depends_on:
             self.depends_on = []
 
@@ -220,7 +319,11 @@ def verify(self):
         # query the local digest to compare against upstream
         local_digest = self._get_local_digest()
 
-        image_ident = f"{container.image}:{container.version}@{local_digest}"
+        image_ident = (
+            f"{container.image}:{container.version}@{local_digest}"
+            if self.container.backend == "docker"
+            else f"{container.image}@{local_digest}"
+        )
 
         # test whether the ident has been checked already
         if image_ident in self._remote_manifest_cache:
@@ -243,7 +346,7 @@ def verify(self):
     def update(self):
         if self._need_explicit_restart:
             self.cmd(
-                f"sudo systemctl restart docker-{self.container.container_name}"
+                f"sudo systemctl restart {self.container.backend}-{self.container.container_name}"
             )
 
     def _get_running_container_image_id(self):
@@ -254,7 +357,7 @@ def _get_running_container_image_id(self):
         image_id, stderr = self.cmd(
             dedent(
                 """\
-            docker container inspect {{component.container.container_name}} \
+            {{ component.container.backend }} container inspect {{component.container.container_name}} \
                 | jq -r '.[0].Image'
                 """
             )
@@ -269,7 +372,7 @@ def _get_local_image_id(self):
         local_image_id, stderr = self.cmd(
             dedent(
                 """\
-            docker image inspect {{component.container.image}}:{{component.container.version}} \
+            {{ component.container.backend }} image inspect {{component.container.image}}:{{component.container.version}} \
                 | jq -r '.[0].Id' \
                 || echo image not available locally
                 """
@@ -281,7 +384,7 @@ def _get_local_digest(self):
         local_digest, stderr = self.cmd(
             dedent(
                 """\
-            docker image inspect {{component.container.image}}:{{component.container.version}} \
+            {{ component.container.backend }} image inspect {{component.container.image}}:{{component.container.version}} \
                 | jq -r 'first | .RepoDigests | first | split("@") | last' \
                 || echo image not available locally
                 """
@@ -294,7 +397,7 @@ def _docker_login(self):
             self.expand(
                 dedent(
                     """\
-        docker login \\
+        {{ component.container.backend }} login \\
             {%- if component.container.registry_user and component.container.registry_password %}
             -u {{component.container.registry_user}} \\
             -p {{component.container.registry_password}} \\
@@ -309,7 +412,9 @@ def _validate_remote_image(self, image_ident):
         try:
             # check if the digest aligns with the remote image
             # if it does not, this command will throw an error
-            stdout, stderr = self.cmd(f"docker manifest inspect {image_ident}")
+            stdout, stderr = self.cmd(
+                f"{self.container.backend} manifest inspect {image_ident}"
+            )
         except CmdExecutionError as e:
             error = e.stderr
             if error.startswith("unsupported manifest format"):  # gitlab

src/batou_ext/resources/oci-template.nix

@@ -2,13 +2,13 @@
 {
   # {% if component.monitor %}
   flyingcircus = {
-    services.sensu-client.checks."docker-{{component.container_name}}" = {
+    services.sensu-client.checks."{{ component.backend }}-{{component.container_name}}" = {
       notification = "Status of container {{component.container_name}}";
       command = ''
-        if $(systemctl is-active --quiet docker-{{component.container_name}}); then
-          echo "docker container {{component.container_name}} is ok"
+        if $(systemctl is-active --quiet {{ component.backend }}-{{component.container_name}}); then
+          echo "{{ component.backend }} container {{component.container_name}} is ok"
         else
-          echo "docker container {{component.container_name}} is inactive"
+          echo "{{ component.backend }} container {{component.container_name}} is inactive"
           exit 2
         fi
       '';
@@ -17,7 +17,6 @@
   # {% endif %}
 
   virtualisation.oci-containers = {
-    backend = "docker";
     containers."{{component.container_name}}" = {
       # {% if component.entrypoint %}
       entrypoint = "{{component.entrypoint}}";
@@ -40,9 +39,20 @@
 
       extraOptions = [
         "--pull=always"
+        # {% if component.backend == "podman" %}
+        "--cgroups=enabled"
+        "--cidfile=/run/{{component.container_name}}/ctr-id"
+        # {% endif %}
         # {% for option in (component.extra_options or []) %}
         "{{option}}"
         # {% endfor %}
+        # {% if component.sd_notify %}
+        "--sdnotify={{ component.sd_notify }}"
+        # {% endif %}
+        # {% if component.health_cmd %}
+        "--health-cmd"
+        {{ component.health_cmd }}
+        # {% endif %}
       ];
 
       volumes = [
@@ -68,4 +78,23 @@
       ];
     };
   };
+
+  # {% if component.backend == "podman" %}
+  systemd.services."podman-{{ component.container_name }}".serviceConfig = {
+    User = "{{ component.user }}";
+    RuntimeDirectory = "{{component.container_name}}";
+    # {% if component.sd_notify == "healthy" %}
+    Delegate = true;
+    NotifyAccess = "all";
+    # {% endif %}
+  };
+
+  # {% if component.sd_notify == "healthy" %}
+  systemd.services."podman-{{ component.container_name }}".wants = [ "linger-users.service" ];
+  users.users."{{ component.user }}".linger = true;
+  # {% endif %}
+  # {% if component.sd_notify == "conmon" %}
+  users.users."{{ component.user }}".linger = false;
+  # {% endif %}
+  # {% endif %}
 }