From b8a8db8e96c6f6f8c7d1c1bcf945572412ff13b1 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <mb@flyingcircus.io>
Date: Mon, 7 Oct 2024 12:31:31 +0200
Subject: [PATCH 1/2] batou_ext.oci: Add support for `podman`
FC-37959
We mostly want this for healthchecks that must pass before the unit is
actually active.
This is kept intentionally simple, only healthchecks via sd_notify are
supported. Mixing containers in systemd units with `healthy` and `conmon`
strategy leads to setups with conflicting requirements (linger vs no
linger) and it's far too easy to end up with an unsupported
configuration (and containers not behaving properly).
---
.../20241129_131748_mb_FC_37959_podman.md | 9 ++
src/batou_ext/oci.py | 111 ++++++++++++++++--
src/batou_ext/resources/oci-template.nix | 32 ++++-
3 files changed, 137 insertions(+), 15 deletions(-)
create mode 100644 CHANGES.d/20241129_131748_mb_FC_37959_podman.md
diff --git a/CHANGES.d/20241129_131748_mb_FC_37959_podman.md b/CHANGES.d/20241129_131748_mb_FC_37959_podman.md
new file mode 100644
index 0000000..08ae902
--- /dev/null
+++ b/CHANGES.d/20241129_131748_mb_FC_37959_podman.md
@@ -0,0 +1,9 @@
+- `batou_ext.oci.Container`: allow to use `podman` as backend instead of `docker`.
+ This also enables the following features:
+
+ * Rootless containers: by setting the `user` option to a different user. By default,
+ the service user of the deployment is used.
+
+ * Only mark services as `active` if the container is up. This requires that the
+ container has a healthcheck. Alternatively, a healthcheck can be configured
+ with the health_cmd attribute.
diff --git a/src/batou_ext/oci.py b/src/batou_ext/oci.py
index 6cef240..da06bdc 100644
--- a/src/batou_ext/oci.py
+++ b/src/batou_ext/oci.py
@@ -1,3 +1,4 @@
+import json
import os
import shlex
from textwrap import dedent
@@ -12,11 +13,38 @@
import batou_ext.nix
+class PodmanRuntime(Component):
+ """
+ Marker to indicate that containers are running with podman instead of
+ docker.
+
+ The backend can only be set globally in NixOS, so this is done with a single,
+ global component here.
+ """
+
+ def configure(self):
+ self.provide("oci:podman", self)
+ self += File(
+ "/etc/local/nixos/oci-backend.nix",
+ content=dedent(
+ """\
+ {
+ virtualisation.podman.enable = true;
+ }
+ """
+ ),
+ )
+
+
class Container(Component):
"""A OCI Container component.
- With this component you can dynamically schedule docker containers to be
- run on the target host.
+ With this component you can dynamically schedule docker or podman containers
+ to be run on the target host.
+
+ Note: the `podman` backend is still considered experimental and thus subject
+ to change. Running both docker and podman containers on the same VM is currently
+ not supported.
Note: Docker image specifiers do not follow a properly resolvable pattern.
Therefore, container registries have to be specified seperately if you need
@@ -70,10 +98,43 @@ class Container(Component):
self += Rebuild()
self += container.activate()
+ ```
+ Containers can use `podman` as backend by adding the `PodmanRuntime`
+ component:
+ ```
+ self += batou_ext.oci.PodmanRuntime()
+ self += batou_ext.oci.Container(
+ image="mysql"
+ )
+ ```
+
+ This assumes that the container has a healthcheck configured.
+ The unit `podman-mysql` will remain in state `activating` until
+ the container is in `healthy` state. Then, it's transitioned into
+ `active` state.
+
+ The `batou_ext.nix.Rebuild` component will wait until newly started
+ and restarted units are `active`, i.e. it will wait until the container is
+ up with `podman`.
+
+ If a podman container doesn't have a healthcheck defined, it's possible to add
+ one via this component. The command is passed `/bin/sh -c`:
+ ```
+ self += batou_ext.oci.PodmanRuntime()
+ self += batou_ext.oci.Container(
+ image="without-healthcheck",
+ health_cmd="curl --fail localhost || exit 1"
+ )
+ ```
+ Please note that the healthcheck is executed _inside_ the container, so
+ the container above would require a `curl` installed.
+ When using podman containers, the user running the container
+ has lingering enabled, i.e. a long-running user session is started by
+ logind (https://www.freedesktop.org/software/systemd/man/latest/loginctl.html#enable-linger%20USER%E2%80%A6).
"""
# general options
@@ -81,6 +142,9 @@ class Container(Component):
version: str = "latest"
container_name = Attribute(str)
+ health_cmd = Attribute(str, None)
+ user = Attribute(str, None)
+
# Set up monitoring
monitor: bool = True
@@ -95,7 +159,7 @@ class Container(Component):
ports: dict = {}
env: dict = {}
depends_on: list = None
- extra_options: list = None
+ extra_options: list = []
# secrets
registry_address = Attribute(Optional[str], None)
@@ -108,6 +172,12 @@ class Container(Component):
}
def configure(self):
+ self.backend = (
+ "podman"
+ if self.require("oci:podman", strict=False, host=self.host)
+ else "docker"
+ )
+
if (
self.registry_user or self.registry_password
) and not self.registry_address:
@@ -145,6 +215,21 @@ def configure(self):
if self.docker_cmd:
self._docker_cmd_list = shlex.split(self.docker_cmd)
+ if self.backend != "podman":
+ for prop in ["user", "health_cmd"]:
+ assert (
+ getattr(self, prop) is None
+ ), f"Container '{self.container_name}' runs with Docker, so the '{prop}' option is not supported!"
+ else:
+ if self.health_cmd is not None:
+ # `json.dumps` quotes the quoted string in a way that it can
+ # be placed as valid string into a Nix expression.
+ # The bash-quoting is done in the OCI module in NixOS.
+ self.health_cmd = json.dumps(self.health_cmd)
+
+ if self.user is None:
+ self.user = self.host.service_user
+
if not self.depends_on:
self.depends_on = []
@@ -220,7 +305,11 @@ def verify(self):
# query the local digest to compare against upstream
local_digest = self._get_local_digest()
- image_ident = f"{container.image}:{container.version}@{local_digest}"
+ image_ident = (
+ f"{container.image}:{container.version}@{local_digest}"
+ if self.container.backend == "docker"
+ else f"{container.image}@{local_digest}"
+ )
# test whether the ident has been checked already
if image_ident in self._remote_manifest_cache:
@@ -243,7 +332,7 @@ def verify(self):
def update(self):
if self._need_explicit_restart:
self.cmd(
- f"sudo systemctl restart docker-{self.container.container_name}"
+ f"sudo systemctl restart {self.container.backend}-{self.container.container_name}"
)
def _get_running_container_image_id(self):
@@ -254,7 +343,7 @@ def _get_running_container_image_id(self):
image_id, stderr = self.cmd(
dedent(
"""\
- docker container inspect {{component.container.container_name}} \
+ {{ component.container.backend }} container inspect {{component.container.container_name}} \
| jq -r '.[0].Image'
"""
)
@@ -269,7 +358,7 @@ def _get_local_image_id(self):
local_image_id, stderr = self.cmd(
dedent(
"""\
- docker image inspect {{component.container.image}}:{{component.container.version}} \
+ {{ component.container.backend }} image inspect {{component.container.image}}:{{component.container.version}} \
| jq -r '.[0].Id' \
|| echo image not available locally
"""
@@ -281,7 +370,7 @@ def _get_local_digest(self):
local_digest, stderr = self.cmd(
dedent(
"""\
- docker image inspect {{component.container.image}}:{{component.container.version}} \
+ {{ component.container.backend }} image inspect {{component.container.image}}:{{component.container.version}} \
| jq -r 'first | .RepoDigests | first | split("@") | last' \
|| echo image not available locally
"""
@@ -294,7 +383,7 @@ def _docker_login(self):
self.expand(
dedent(
"""\
- docker login \\
+ {{ component.container.backend }} login \\
{%- if component.container.registry_user and component.container.registry_password %}
-u {{component.container.registry_user}} \\
-p {{component.container.registry_password}} \\
@@ -309,7 +398,9 @@ def _validate_remote_image(self, image_ident):
try:
# check if the digest aligns with the remote image
# if it does not, this command will throw an error
- stdout, stderr = self.cmd(f"docker manifest inspect {image_ident}")
+ stdout, stderr = self.cmd(
+ f"{self.container.backend} manifest inspect {image_ident}"
+ )
except CmdExecutionError as e:
error = e.stderr
if error.startswith("unsupported manifest format"): # gitlab
diff --git a/src/batou_ext/resources/oci-template.nix b/src/batou_ext/resources/oci-template.nix
index b5c21c7..9622d88 100644
--- a/src/batou_ext/resources/oci-template.nix
+++ b/src/batou_ext/resources/oci-template.nix
@@ -2,13 +2,13 @@
{
# {% if component.monitor %}
flyingcircus = {
- services.sensu-client.checks."docker-{{component.container_name}}" = {
+ services.sensu-client.checks."{{ component.backend }}-{{component.container_name}}" = {
notification = "Status of container {{component.container_name}}";
command = ''
- if $(systemctl is-active --quiet docker-{{component.container_name}}); then
- echo "docker container {{component.container_name}} is ok"
+ if $(systemctl is-active --quiet {{ component.backend }}-{{component.container_name}}); then
+ echo "{{ component.backend }} container {{component.container_name}} is ok"
else
- echo "docker container {{component.container_name}} is inactive"
+ echo "{{ component.backend }} container {{component.container_name}} is inactive"
exit 2
fi
'';
@@ -17,7 +17,6 @@
# {% endif %}
virtualisation.oci-containers = {
- backend = "docker";
containers."{{component.container_name}}" = {
# {% if component.entrypoint %}
entrypoint = "{{component.entrypoint}}";
@@ -40,9 +39,20 @@
extraOptions = [
"--pull=always"
+ # {% if component.backend == "podman" %}
+ "--cgroups=enabled"
+ "--cidfile=/run/{{component.container_name}}/ctr-id"
+ # {% endif %}
# {% for option in (component.extra_options or []) %}
"{{option}}"
# {% endfor %}
+ # {% if component.backend == "podman" %}
+ "--sdnotify=healthy"
+ # {% endif %}
+ # {% if component.health_cmd %}
+ "--health-cmd"
+ {{ component.health_cmd }}
+ # {% endif %}
];
volumes = [
@@ -68,4 +78,16 @@
];
};
};
+
+ # {% if component.backend == "podman" %}
+ systemd.services."podman-{{ component.container_name }}".serviceConfig = {
+ User = "{{ component.user }}";
+ RuntimeDirectory = "{{component.container_name}}";
+ Delegate = true;
+ NotifyAccess = "all";
+ };
+
+ systemd.services."podman-{{ component.container_name }}".wants = [ "linger-users.service" ];
+ users.users."{{ component.user }}".linger = true;
+ # {% endif %}
}
From c9beca6cc62641b8836d4a07af22ffd7d5fabf18 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <mb@flyingcircus.io>
Date: Wed, 5 Feb 2025 15:16:51 +0100
Subject: [PATCH 2/2] batou_ext.http.HTTPServiceWatchdog: make rebuild optional
---
src/batou_ext/http.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/src/batou_ext/http.py b/src/batou_ext/http.py
index 87ce7b2..bdad592 100644
--- a/src/batou_ext/http.py
+++ b/src/batou_ext/http.py
@@ -61,7 +61,6 @@ def _deploy_customer_http_auth_file(self):
self.path = self._.path
-@batou_ext.nix.rebuild
class HTTPServiceWatchdog(batou.component.Component):
"""
Adds a "watchdog" on top of a systemd service that checks within a given interval
@@ -209,6 +208,8 @@ class HTTPServiceWatchdog(batou.component.Component):
start_timeout = batou.component.Attribute(int, default=64)
watchdog_interval = batou.component.Attribute(int, default=64)
+ rebuild = batou.component.Attribute("literal", default=True)
+
def configure(self):
if self.predefined_service and self.script is not None:
raise ValueError(
@@ -226,6 +227,9 @@ def configure(self):
),
)
+ if self.rebuild:
+ self += batou_ext.nix.Rebuild()
+
class HTTPWatchdogScript(batou.component.Component):
"""