From 79a1922584cd98f792d3811df13cf4c65284afc5 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Sat, 16 Sep 2023 01:02:21 +0900 Subject: [PATCH] Support CONTAINER_ENGINE=(podman|nerdctl) in addition to docker - Dockerfile: Use long image names with the `docker.io/` prefix for compatibility with Red Hat's containers - Dockerfile: Avoid using BuildKit features that are unsupported by buildah - docker-compose.yaml: Remove `tty: true`, as it is unsupported by `nerdctl compose up -d`. - Makefile: Remove `$(COMPOSE) cp`, as it is unsupported by podman-compose nerdctl support depends on: - containerd/nerdctl PR 2507 (milestone: nerdctl v1.5.1) Signed-off-by: Akihiro Suda --- .github/workflows/main.yaml | 19 ++++++++- Dockerfile | 22 ++++------ Makefile | 13 +++--- Makefile.d/check-preflight.sh | 45 +++++++++++++++----- Makefile.d/detect_container_engine.sh | 60 +++++++++++++++++++++++++++ README.md | 10 ++++- docker-compose.yaml | 1 - init-host/init-host.root.sh | 2 + 8 files changed, 139 insertions(+), 33 deletions(-) create mode 100755 Makefile.d/detect_container_engine.sh diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index dc23717..db7dd63 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -9,6 +9,12 @@ jobs: name: "Single node" runs-on: ubuntu-22.04 timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + engine: [docker, podman] + env: + CONTAINER_ENGINE: "${{ matrix.engine }}" steps: - uses: actions/checkout@v3 - name: Set up cgroup v2 delegation @@ -19,15 +25,26 @@ jobs: Delegate=cpu cpuset io memory pids EOF sudo systemctl daemon-reload + - name: Remove preinstalled Moby + # Preinstalled Moby does not contain dockerd-rootless-setuptool.sh + run: sudo apt-get remove moby-engine-* - name: Set up Rootless Docker + if: ${{ matrix.engine == 'docker' }} run: | set -eux -o pipefail - sudo apt-get remove moby-engine-* curl https://get.docker.com | sudo sh sudo systemctl disable --now docker.socket docker.service sudo rm -rf /var/run/docker* dockerd-rootless-setuptool.sh install docker info + - name: Set up Rootless Podman + if: ${{ matrix.engine == 'podman' }} + run: | + set -eux -o pipefail + # Preinstalled Podman is too old (v3.4.4) + sudo apt-get remove podman* + sudo ./init-host/init-host.root.d/install-podman.sh + podman info - run: make up - run: sleep 5 - run: make kubeadm-init diff --git a/Dockerfile b/Dockerfile index 7896c9d..2526e51 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,11 @@ -ARG BASE_IMAGE=kindest/node:v1.28.0 - -# TODO: use `ADD --checksum=sha256...` -FROM scratch AS cni-plugins-amd64 -ADD https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz /cni-plugins.tgz - -FROM scratch AS cni-plugins-arm64 -ADD https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-arm64-v1.3.0.tgz /cni-plugins.tgz - -ARG TARGETARCH -FROM cni-plugins-$TARGETARCH AS cni-plugins - -ARG BASE_IMAGE +ARG BASE_IMAGE=docker.io/kindest/node:v1.28.0 +ARG CNI_PLUGINS_VERSION=v1.3.0 FROM ${BASE_IMAGE} -RUN --mount=type=bind,from=cni-plugins,dst=/mnt/tmp \ - tar Cxzvf /opt/cni/bin /mnt/tmp/cni-plugins.tgz +# TODO: check SHA256SUMS of cni-plugins +ARG CNI_PLUGINS_VERSION +RUN arch="$(uname -m | sed -e s/x86_64/amd64/ -e s/aarch64/arm64/)" && \ + curl -fsSL https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${arch}-${CNI_PLUGINS_VERSION}.tgz \ + | tar Cxzv /opt/cni/bin # gettext-base: for `envsubst` # moreutils: for `sponge` # socat: for `socat` (to silence "[WARNING FileExisting-socat]" from kubeadm) diff --git a/Makefile b/Makefile index 76157e4..95579ba 100644 --- a/Makefile +++ b/Makefile @@ -17,11 +17,14 @@ export U7S_NODE_NAME:= $(NODE_NAME) # Not accessible from other hosts. export U7S_NODE_SUBNET := $(NODE_SUBNET) -DOCKER ?= docker +CONTAINER_ENGINE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh CONTAINER_ENGINE) +export CONTAINER_ENGINE := $(CONTAINER_ENGINE) -export DOCKER := $(DOCKER) +CONTAINER_ENGINE_TYPE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh CONTAINER_ENGINE_TYPE) +export CONTAINER_ENGINE_TYPE := $(CONTAINER_ENGINE_TYPE) + +COMPOSE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh COMPOSE) -COMPOSE := $(DOCKER) compose NODE_SERVICE_NAME := node NODE_SHELL := $(COMPOSE) exec \ -e U7S_HOST_IP=$(U7S_HOST_IP) \ @@ -78,7 +81,7 @@ logs: .PHONY: kubeconfig kubeconfig: - $(COMPOSE) cp $(NODE_SERVICE_NAME):/etc/kubernetes/admin.conf ./kubeconfig + $(COMPOSE) exec -T $(NODE_SERVICE_NAME) cat /etc/kubernetes/admin.conf >kubeconfig @echo "# Run the following command by yourself:" @echo "export KUBECONFIG=$(shell pwd)/kubeconfig" ifeq ($(shell command -v kubectl 2> /dev/null),) @@ -88,7 +91,7 @@ endif .PHONY: kubectl kubectl: - $(COMPOSE) cp $(NODE_SERVICE_NAME):/usr/bin/kubectl ./kubectl + $(COMPOSE) exec -T --workdir=/usr/bin $(NODE_SERVICE_NAME) tar c kubectl | tar xv @echo "# Run the following command by yourself:" @echo "export PATH=$(shell pwd):\$$PATH" @echo "source <(kubectl completion bash)" diff --git a/Makefile.d/check-preflight.sh b/Makefile.d/check-preflight.sh index 534b67f..df4cd29 100755 --- a/Makefile.d/check-preflight.sh +++ b/Makefile.d/check-preflight.sh @@ -1,6 +1,9 @@ #!/bin/bash set -eu +function INFO() { + echo >&2 -e "\e[104m\e[97m[INFO]\e[49m\e[39m $@" +} function WARNING() { echo >&2 -e "\e[101m\e[97m[WARNING]\e[49m\e[39m $@" } @@ -9,12 +12,21 @@ function ERROR() { echo >&2 -e "\e[101m\e[97m[ERROR]\e[49m\e[39m $@" } -: "${DOCKER:=docker}" +script_dir="$(dirname "$0")" +detect_engine="${script_dir}"/detect_container_engine.sh +: "${CONTAINER_ENGINE:=$("${detect_engine}" CONTAINER_ENGINE)}" +: "${CONTAINER_ENGINE_TYPE:=$("${detect_engine}" CONTAINER_ENGINE_TYPE)}" : "${QUICK:=0}" -: "${BUSYBOX_IMAGE:=busybox}" +: "${BUSYBOX_IMAGE:=docker.io/library/busybox:latest}" + +if [ -z "${CONTAINER_ENGINE}" ] || [ -z "${CONTAINER_ENGINE_TYPE}" ]; then + ERROR "No container engine was detected" + exit 1 +fi +INFO "Detected container engine type: ${CONTAINER_ENGINE_TYPE}" # Check hard dependency commands -for f in make jq "${DOCKER}"; do +for f in make jq "${CONTAINER_ENGINE}"; do if ! command -v "${f}" >/dev/null 2>&1; then ERROR "Command \"${f}\" is not installed" exit 1 @@ -28,9 +40,22 @@ for f in kubectl; do fi done -# Check if Docker is running in Rootless mode -# TODO: support Podman? -if "${DOCKER}" info --format '{{json .SecurityOptions}}' | grep -q "name=rootless"; then +rootless= +case "${CONTAINER_ENGINE_TYPE}" in +"podman") + if [ "$(${CONTAINER_ENGINE} info --format '{{.Host.Security.Rootless}}')" = "true" ]; then + rootless=1 + fi + ;; +*) + if ${CONTAINER_ENGINE} info --format '{{json .SecurityOptions}}' | grep -q "name=rootless"; then + rootless=1 + fi + ;; +esac + +# Check if the container engine is running in Rootless mode +if [ "${rootless}" = "1" ]; then # Check systemd lingering: https://rootlesscontaine.rs/getting-started/common/login/ if command -v loginctl >/dev/null 2>&1; then if [ "$(loginctl list-users --output json | jq ".[] | select(.uid == "${UID}").linger")" != "true" ]; then @@ -57,7 +82,7 @@ if "${DOCKER}" info --format '{{json .SecurityOptions}}' | grep -q "name=rootles fi fi else - WARNING "Docker does not seem running in Rootless mode" + WARNING "Container engine (${CONTAINER_ENGINE}) does not seem running in Rootless mode" fi # Check kernel modules @@ -68,10 +93,10 @@ for f in br_netfilter ip6_tables ip6table_nat ip_tables iptable_nat vxlan; do done if [ "$QUICK" != "1" ]; then - # Check net.ipv4.conf.default.rp_filter in the daemon's network namespace. + # Check net.ipv4.conf.default.rp_filter in the container engine's network namespace. (e.g., netns of dockerd) # The value can be 0 (disabled) or 2 (loose), must not be 1 (strict). - if [ "$(${DOCKER} run --rm --net=host "${BUSYBOX_IMAGE}" sysctl -n net.ipv4.conf.default.rp_filter)" == "1" ]; then - ERROR "sysctl value \"net.ipv4.conf.default.rp_filter\" must be 0 (disabled) or 2 (loose) in the daemon's network namespace" + if [ "$(${CONTAINER_ENGINE} run --rm --net=host "${BUSYBOX_IMAGE}" sysctl -n net.ipv4.conf.default.rp_filter)" == "1" ]; then + ERROR "sysctl value \"net.ipv4.conf.default.rp_filter\" must be 0 (disabled) or 2 (loose) in the container engine's network namespace" exit 1 fi fi diff --git a/Makefile.d/detect_container_engine.sh b/Makefile.d/detect_container_engine.sh new file mode 100755 index 0000000..fee6431 --- /dev/null +++ b/Makefile.d/detect_container_engine.sh @@ -0,0 +1,60 @@ +#!/bin/bash +set -eu -o pipefail +: "${CONTAINER_ENGINE:=}" +: "${COMPOSE:=}" + +if [ -z "${CONTAINER_ENGINE}" ]; then + if command -v dockerd-rootless.sh >/dev/null 2>&1; then + CONTAINER_ENGINE=docker + elif command -v containerd-rootless.sh >/dev/null 2>&1; then + CONTAINER_ENGINE=nerdctl + elif command -v podman 2>&1; then + CONTAINER_ENGINE=podman + else + echo >&2 "$0: no container engine was detected" + exit 1 + fi +fi + +CONTAINER_ENGINE_TYPE=docker +if [[ "${CONTAINER_ENGINE}" = *"podman"* ]]; then + CONTAINER_ENGINE_TYPE=podman +elif [[ "${CONTAINER_ENGINE}" = *"nerdctl"* ]]; then + CONTAINER_ENGINE_TYPE=nerdctl +fi + +if [ -z "${COMPOSE}" ]; then + COMPOSE="${CONTAINER_ENGINE} compose" + if [ "${CONTAINER_ENGINE_TYPE}" = "podman" ]; then + COMPOSE=podman-compose + fi +fi + +case "$#" in +0) + echo "CONTAINER_ENGINE=${CONTAINER_ENGINE}" + echo "CONTAINER_ENGINE_TYPE=${CONTAINER_ENGINE_TYPE}" + echo "COMPOSE=${COMPOSE}" + ;; +1) + case "$1" in + "CONTAINER_ENGINE") + echo "${CONTAINER_ENGINE}" + ;; + "CONTAINER_ENGINE_TYPE") + echo "${CONTAINER_ENGINE_TYPE}" + ;; + "COMPOSE") + echo "${COMPOSE}" + ;; + *) + echo >&2 "$0: unknown argument: $1" + exit 1 + ;; + esac + ;; +*) + echo >&2 "$0: too many arguments" + exit 1 + ;; +esac diff --git a/README.md b/README.md index be13bf5..a525acd 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,12 @@ but Usernetes (Gen 2) supports creating a cluster with multiple hosts. - Rocky Linux 9 - AlmaLinux 9 -- [Rootless Docker](https://rootlesscontaine.rs/getting-started/docker/): +- One of the following container engines: +|Container Engine |Minimum version| +|-----------------------------------------------------------------------------------|---------------| +|[Rootless Docker](https://rootlesscontaine.rs/getting-started/docker/)(recommended)|v20.10 | +|[Rootless Podman](https://rootlesscontaine.rs/getting-started/podman/) |v4 (?) | +|[Rootless nerdctl](https://rootlesscontaine.rs/getting-started/containerd/) |v1.5.1 | ```bash curl -o install.sh -fsSL https://get.docker.com sudo sh install.sh @@ -98,6 +103,9 @@ make down-v kubectl taint nodes --all node-role.kubernetes.io/control-plane- ``` +The container engine defaults to Docker. +To change the container engine, set `export CONTAINER_ENGINE=podman` or `export CONTAINER_ENGINE=nerdctl`. + ## Limitations - Node ports cannot be exposed automatically. Edit [`docker-compose.yaml`](./docker-compose.yaml) for exposing additional node ports. - Most of host files are not visible with `hostPath` mounts. Edit [`docker-compose.yaml`](./docker-compose.yaml) for mounting additional files. diff --git a/docker-compose.yaml b/docker-compose.yaml index 0a5ba00..133b113 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,7 +7,6 @@ services: hostname: ${U7S_NODE_NAME} privileged: true restart: on-failure - tty: true ports: # etcd - 2379:2379 diff --git a/init-host/init-host.root.sh b/init-host/init-host.root.sh index 8aca580..59d4577 100755 --- a/init-host/init-host.root.sh +++ b/init-host/init-host.root.sh @@ -40,6 +40,8 @@ if ! command -v dockerd-rootless-setuptool.sh >/dev/null 2>&1; then fi systemctl disable --now docker +# TODO: install Podman when CONTAINER_ENGINE=podman + if command -v dnf >/dev/null 2>&1; then dnf install -y git shadow-utils make jq else