Support CONTAINER_ENGINE=(podman|nerdctl) in addition to docker

- Dockerfile: Use long image names with the `docker.io/` prefix for
  compatibility with Red Hat's containers
- Dockerfile: Avoid using BuildKit features that are unsupported by buildah
- docker-compose.yaml: Remove `tty: true`, as it is unsupported by `nerdctl compose up -d`.
- Makefile: Remove `$(COMPOSE) cp`, as it is unsupported by podman-compose

nerdctl support depends on:
- containerd/nerdctl PR 2507 (milestone: nerdctl v1.5.1)

Signed-off-by: Akihiro Suda <[email protected]>

.github/workflows/main.yaml

@@ -9,6 +9,12 @@ jobs:
     name: "Single node"
     runs-on: ubuntu-22.04
     timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        engine: [docker, podman]
+    env:
+      CONTAINER_ENGINE: "${{ matrix.engine }}"
     steps:
       - uses: actions/checkout@v3
       - name: Set up cgroup v2 delegation
@@ -19,15 +25,26 @@ jobs:
           Delegate=cpu cpuset io memory pids
           EOF
           sudo systemctl daemon-reload
+      - name: Remove preinstalled Moby
+        # Preinstalled Moby does not contain dockerd-rootless-setuptool.sh
+        run: sudo apt-get remove moby-engine-*
       - name: Set up Rootless Docker
+        if: ${{ matrix.engine == 'docker' }}
         run: |
           set -eux -o pipefail
-          sudo apt-get remove moby-engine-*
           curl https://get.docker.com | sudo sh
           sudo systemctl disable --now docker.socket docker.service
           sudo rm -rf /var/run/docker*
           dockerd-rootless-setuptool.sh install
           docker info
+      - name: Set up Rootless Podman
+        if: ${{ matrix.engine == 'podman' }}
+        run: |
+          set -eux -o pipefail
+          # Preinstalled Podman is too old (v3.4.4)
+          sudo apt-get remove podman*
+          sudo ./init-host/init-host.root.d/install-podman.sh
+          podman info
       - run: make up
       - run: sleep 5
       - run: make kubeadm-init

Dockerfile

@@ -1,19 +1,11 @@
-ARG BASE_IMAGE=kindest/node:v1.28.0
-
-# TODO: use `ADD --checksum=sha256...`
-FROM scratch AS cni-plugins-amd64
-ADD https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz /cni-plugins.tgz
-
-FROM scratch AS cni-plugins-arm64
-ADD https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-arm64-v1.3.0.tgz /cni-plugins.tgz
-
-ARG TARGETARCH
-FROM cni-plugins-$TARGETARCH AS cni-plugins
-
-ARG BASE_IMAGE
+ARG BASE_IMAGE=docker.io/kindest/node:v1.28.0
+ARG CNI_PLUGINS_VERSION=v1.3.0
 FROM ${BASE_IMAGE}
-RUN --mount=type=bind,from=cni-plugins,dst=/mnt/tmp \
-  tar Cxzvf /opt/cni/bin /mnt/tmp/cni-plugins.tgz
+# TODO: check SHA256SUMS of cni-plugins
+ARG CNI_PLUGINS_VERSION
+RUN arch="$(uname -m | sed -e s/x86_64/amd64/ -e s/aarch64/arm64/)" && \
+  curl -fsSL https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${arch}-${CNI_PLUGINS_VERSION}.tgz \
+  | tar Cxzv /opt/cni/bin
 # gettext-base: for `envsubst`
 # moreutils: for `sponge`
 # socat: for `socat` (to silence "[WARNING FileExisting-socat]" from kubeadm)

Makefile

@@ -17,11 +17,14 @@ export U7S_NODE_NAME:= $(NODE_NAME)
 # Not accessible from other hosts.
 export U7S_NODE_SUBNET := $(NODE_SUBNET)
 
-DOCKER ?= docker
+CONTAINER_ENGINE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh CONTAINER_ENGINE)
+export CONTAINER_ENGINE := $(CONTAINER_ENGINE)
 
-export DOCKER := $(DOCKER)
+CONTAINER_ENGINE_TYPE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh CONTAINER_ENGINE_TYPE)
+export CONTAINER_ENGINE_TYPE := $(CONTAINER_ENGINE_TYPE)
+
+COMPOSE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh COMPOSE)
 
-COMPOSE := $(DOCKER) compose
 NODE_SERVICE_NAME := node
 NODE_SHELL := $(COMPOSE) exec \
 	-e U7S_HOST_IP=$(U7S_HOST_IP) \
@@ -78,7 +81,7 @@ logs:
 
 .PHONY: kubeconfig
 kubeconfig:
-	$(COMPOSE) cp $(NODE_SERVICE_NAME):/etc/kubernetes/admin.conf ./kubeconfig
+	$(COMPOSE) exec -T $(NODE_SERVICE_NAME) cat /etc/kubernetes/admin.conf >kubeconfig
 	@echo "# Run the following command by yourself:"
 	@echo "export KUBECONFIG=$(shell pwd)/kubeconfig"
 ifeq ($(shell command -v kubectl 2> /dev/null),)
@@ -88,7 +91,7 @@ endif
 
 .PHONY: kubectl
 kubectl:
-	$(COMPOSE) cp $(NODE_SERVICE_NAME):/usr/bin/kubectl ./kubectl
+	$(COMPOSE) exec -T --workdir=/usr/bin $(NODE_SERVICE_NAME) tar c kubectl | tar xv
 	@echo "# Run the following command by yourself:"
 	@echo "export PATH=$(shell pwd):\$$PATH"
 	@echo "source <(kubectl completion bash)"

Makefile.d/check-preflight.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -eu
 
+function INFO() {
+	echo >&2 -e "\e[104m\e[97m[INFO]\e[49m\e[39m $@"
+}
 function WARNING() {
 	echo >&2 -e "\e[101m\e[97m[WARNING]\e[49m\e[39m $@"
 }
@@ -9,12 +12,21 @@ function ERROR() {
 	echo >&2 -e "\e[101m\e[97m[ERROR]\e[49m\e[39m $@"
 }
 
-: "${DOCKER:=docker}"
+script_dir="$(dirname "$0")"
+detect_engine="${script_dir}"/detect_container_engine.sh
+: "${CONTAINER_ENGINE:=$("${detect_engine}" CONTAINER_ENGINE)}"
+: "${CONTAINER_ENGINE_TYPE:=$("${detect_engine}" CONTAINER_ENGINE_TYPE)}"
 : "${QUICK:=0}"
-: "${BUSYBOX_IMAGE:=busybox}"
+: "${BUSYBOX_IMAGE:=docker.io/library/busybox:latest}"
+
+if [ -z "${CONTAINER_ENGINE}" ] || [ -z "${CONTAINER_ENGINE_TYPE}" ]; then
+	ERROR "No container engine was detected"
+	exit 1
+fi
+INFO "Detected container engine type: ${CONTAINER_ENGINE_TYPE}"
 
 # Check hard dependency commands
-for f in make jq "${DOCKER}"; do
+for f in make jq "${CONTAINER_ENGINE}"; do
 	if ! command -v "${f}" >/dev/null 2>&1; then
 		ERROR "Command \"${f}\" is not installed"
 		exit 1
@@ -28,9 +40,22 @@ for f in kubectl; do
 	fi
 done
 
-# Check if Docker is running in Rootless mode
-# TODO: support Podman?
-if "${DOCKER}" info --format '{{json .SecurityOptions}}' | grep -q "name=rootless"; then
+rootless=
+case "${CONTAINER_ENGINE_TYPE}" in
+"podman")
+	if [ "$(${CONTAINER_ENGINE} info --format '{{.Host.Security.Rootless}}')" = "true" ]; then
+		rootless=1
+	fi
+	;;
+*)
+	if ${CONTAINER_ENGINE} info --format '{{json .SecurityOptions}}' | grep -q "name=rootless"; then
+		rootless=1
+	fi
+	;;
+esac
+
+# Check if the container engine is running in Rootless mode
+if [ "${rootless}" = "1" ]; then
 	# Check systemd lingering: https://rootlesscontaine.rs/getting-started/common/login/
 	if command -v loginctl >/dev/null 2>&1; then
 		if [ "$(loginctl list-users --output json | jq ".[] | select(.uid == "${UID}").linger")" != "true" ]; then
@@ -57,7 +82,7 @@ if "${DOCKER}" info --format '{{json .SecurityOptions}}' | grep -q "name=rootles
 		fi
 	fi
 else
-	WARNING "Docker does not seem running in Rootless mode"
+	WARNING "Container engine (${CONTAINER_ENGINE}) does not seem running in Rootless mode"
 fi
 
 # Check kernel modules
@@ -68,10 +93,10 @@ for f in br_netfilter ip6_tables ip6table_nat ip_tables iptable_nat vxlan; do
 done
 
 if [ "$QUICK" != "1" ]; then
-	# Check net.ipv4.conf.default.rp_filter in the daemon's network namespace.
+	# Check net.ipv4.conf.default.rp_filter in the container engine's network namespace. (e.g., netns of dockerd)
 	# The value can be 0 (disabled) or 2 (loose), must not be 1 (strict).
-	if [ "$(${DOCKER} run --rm --net=host "${BUSYBOX_IMAGE}" sysctl -n net.ipv4.conf.default.rp_filter)" == "1" ]; then
-		ERROR "sysctl value \"net.ipv4.conf.default.rp_filter\" must be 0 (disabled) or 2 (loose) in the daemon's network namespace"
+	if [ "$(${CONTAINER_ENGINE} run --rm --net=host "${BUSYBOX_IMAGE}" sysctl -n net.ipv4.conf.default.rp_filter)" == "1" ]; then
+		ERROR "sysctl value \"net.ipv4.conf.default.rp_filter\" must be 0 (disabled) or 2 (loose) in the container engine's network namespace"
 		exit 1
 	fi
 fi

Makefile.d/detect_container_engine.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+set -eu -o pipefail
+: "${CONTAINER_ENGINE:=}"
+: "${COMPOSE:=}"
+
+if [ -z "${CONTAINER_ENGINE}" ]; then
+	if command -v dockerd-rootless.sh >/dev/null 2>&1; then
+		CONTAINER_ENGINE=docker
+	elif command -v containerd-rootless.sh >/dev/null 2>&1; then
+		CONTAINER_ENGINE=nerdctl
+	elif command -v podman 2>&1; then
+		CONTAINER_ENGINE=podman
+	else
+		echo >&2 "$0: no container engine was detected"
+		exit 1
+	fi
+fi
+
+CONTAINER_ENGINE_TYPE=docker
+if [[ "${CONTAINER_ENGINE}" = *"podman"* ]]; then
+	CONTAINER_ENGINE_TYPE=podman
+elif [[ "${CONTAINER_ENGINE}" = *"nerdctl"* ]]; then
+	CONTAINER_ENGINE_TYPE=nerdctl
+fi
+
+if [ -z "${COMPOSE}" ]; then
+	COMPOSE="${CONTAINER_ENGINE} compose"
+	if [ "${CONTAINER_ENGINE_TYPE}" = "podman" ]; then
+		COMPOSE=podman-compose
+	fi
+fi
+
+case "$#" in
+0)
+	echo "CONTAINER_ENGINE=${CONTAINER_ENGINE}"
+	echo "CONTAINER_ENGINE_TYPE=${CONTAINER_ENGINE_TYPE}"
+	echo "COMPOSE=${COMPOSE}"
+	;;
+1)
+	case "$1" in
+	"CONTAINER_ENGINE")
+		echo "${CONTAINER_ENGINE}"
+		;;
+	"CONTAINER_ENGINE_TYPE")
+		echo "${CONTAINER_ENGINE_TYPE}"
+		;;
+	"COMPOSE")
+		echo "${COMPOSE}"
+		;;
+	*)
+		echo >&2 "$0: unknown argument: $1"
+		exit 1
+		;;
+	esac
+	;;
+*)
+	echo >&2 "$0: too many arguments"
+	exit 1
+	;;
+esac

README.md

@@ -27,7 +27,12 @@ but Usernetes (Gen 2) supports creating a cluster with multiple hosts.
   - Rocky Linux 9
   - AlmaLinux 9
 
-- [Rootless Docker](https://rootlesscontaine.rs/getting-started/docker/):
+- One of the following container engines:
+|Container Engine                                                                   |Minimum version|
+|-----------------------------------------------------------------------------------|---------------|
+|[Rootless Docker](https://rootlesscontaine.rs/getting-started/docker/)(recommended)|v20.10         |
+|[Rootless Podman](https://rootlesscontaine.rs/getting-started/podman/)             |v4 (?)         |
+|[Rootless nerdctl](https://rootlesscontaine.rs/getting-started/containerd/)        |v1.5.1         |
 ```bash
 curl -o install.sh -fsSL https://get.docker.com
 sudo sh install.sh
@@ -98,6 +103,9 @@ make down-v
 kubectl taint nodes --all node-role.kubernetes.io/control-plane-
 ```
 
+The container engine defaults to Docker.
+To change the container engine, set `export CONTAINER_ENGINE=podman` or `export CONTAINER_ENGINE=nerdctl`.
+
 ## Limitations
 - Node ports cannot be exposed automatically. Edit [`docker-compose.yaml`](./docker-compose.yaml) for exposing additional node ports.
 - Most of host files are not visible with `hostPath` mounts. Edit [`docker-compose.yaml`](./docker-compose.yaml) for mounting additional files.

docker-compose.yaml

@@ -7,7 +7,6 @@ services:
     hostname: ${U7S_NODE_NAME}
     privileged: true
     restart: on-failure
-    tty: true
     ports:
       # etcd
       - 2379:2379

init-host/init-host.root.d/install-podman.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# This script installs the latest release of Podman.
+# Repository information is from https://podman.io/docs/installation#linux-distributions
+set -eux -o pipefail
+if [ "$(id -u)" != "0" ]; then
+	echo "Must run as the root"
+	exit 1
+fi
+
+mkdir -p /etc/apt/keyrings
+curl -fsSL "https://download.opensuse.org/repositories/devel:kubic:libcontainers:unstable/xUbuntu_$(lsb_release -rs)/Release.key" \
+  | gpg --dearmor \
+  | tee /etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg > /dev/null
+echo \
+  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg]\
+    https://download.opensuse.org/repositories/devel:kubic:libcontainers:unstable/xUbuntu_$(lsb_release -rs)/ /" \
+  | tee /etc/apt/sources.list.d/devel:kubic:libcontainers:unstable.list > /dev/null
+apt-get update -qq
+apt-get -qq -y install podman
+# No dpkg for podman-compose ?
+pip3 install podman-compose

init-host/init-host.root.sh

@@ -40,6 +40,8 @@ if ! command -v dockerd-rootless-setuptool.sh >/dev/null 2>&1; then
 fi
 systemctl disable --now docker
 
+# TODO: install Podman when CONTAINER_ENGINE=podman
+
 if command -v dnf >/dev/null 2>&1; then
 	dnf install -y git shadow-utils make jq
 else