From 337aa10605d2b1073d95d060f85a516b2a7d3101 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Sat, 16 Sep 2023 01:02:21 +0900 Subject: [PATCH] Support CONTAINER_ENGINE=(podman|nerdctl) in addition to docker - Dockerfile: Use long image names with the `docker.io/` prefix for compatibility with Red Hat's containers - Dockerfile: Avoid using BuildKit features that are unsupported by buildah - docker-compose.yaml: Set the restart mode to `always`, as required by `podman-restart.service` - docker-compose.yaml: Remove `tty: true`, as it is unsupported by `nerdctl compose up -d`. - Makefile: Remove `$(COMPOSE) cp`, as it is unsupported by podman-compose nerdctl support depends on: - containerd/nerdctl PR 2507 (milestone: nerdctl v1.6.0) Signed-off-by: Akihiro Suda --- .github/workflows/main.yaml | 30 ++++++++++- Dockerfile | 22 +++----- Makefile | 13 +++-- Makefile.d/check-preflight.sh | 45 ++++++++++++---- Makefile.d/detect_container_engine.sh | 54 ++++++++++++++++++++ README.md | 24 +++++++-- docker-compose.yaml | 3 +- hack/create-cluster-lxd.sh | 12 +++-- hack/create-hosts-lxd.sh | 3 +- init-host/init-host.root.d/install-podman.sh | 25 +++++++++ init-host/init-host.root.sh | 48 ++++++++++++----- init-host/init-host.rootless.sh | 16 +++++- 12 files changed, 236 insertions(+), 59 deletions(-) create mode 100755 Makefile.d/detect_container_engine.sh create mode 100755 init-host/init-host.root.d/install-podman.sh diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index dc23717..a9fbdca 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -9,6 +9,12 @@ jobs: name: "Single node" runs-on: ubuntu-22.04 timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + engine: [docker, podman] + env: + CONTAINER_ENGINE: "${{ matrix.engine }}" steps: - uses: actions/checkout@v3 - name: Set up cgroup v2 delegation @@ -19,15 +25,26 @@ jobs: Delegate=cpu cpuset io memory pids EOF sudo systemctl daemon-reload + - name: Remove preinstalled Moby + # Preinstalled Moby does not contain dockerd-rootless-setuptool.sh + run: sudo apt-get remove moby-engine-* - name: Set up Rootless Docker + if: ${{ matrix.engine == 'docker' }} run: | set -eux -o pipefail - sudo apt-get remove moby-engine-* curl https://get.docker.com | sudo sh sudo systemctl disable --now docker.socket docker.service sudo rm -rf /var/run/docker* dockerd-rootless-setuptool.sh install docker info + - name: Set up Rootless Podman + if: ${{ matrix.engine == 'podman' }} + run: | + set -eux -o pipefail + # Preinstalled Podman is too old (v3.4.4) + sudo apt-get remove podman* + sudo ./init-host/init-host.root.d/install-podman.sh + podman info - run: make up - run: sleep 5 - run: make kubeadm-init @@ -46,6 +63,17 @@ jobs: name: "Multi node (emulated using LXD)" runs-on: ubuntu-22.04 timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - lxc-image: ubuntu:22.04 + engine: docker + - lxc-image: images:fedora/38/cloud + engine: podman + env: + LXC_IMAGE: "${{ matrix.lxc-image }}" + CONTAINER_ENGINE: "${{ matrix.engine }}" steps: - run: sudo modprobe vxlan - uses: actions/checkout@v3 diff --git a/Dockerfile b/Dockerfile index 7896c9d..2526e51 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,11 @@ -ARG BASE_IMAGE=kindest/node:v1.28.0 - -# TODO: use `ADD --checksum=sha256...` -FROM scratch AS cni-plugins-amd64 -ADD https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz /cni-plugins.tgz - -FROM scratch AS cni-plugins-arm64 -ADD https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-arm64-v1.3.0.tgz /cni-plugins.tgz - -ARG TARGETARCH -FROM cni-plugins-$TARGETARCH AS cni-plugins - -ARG BASE_IMAGE +ARG BASE_IMAGE=docker.io/kindest/node:v1.28.0 +ARG CNI_PLUGINS_VERSION=v1.3.0 FROM ${BASE_IMAGE} -RUN --mount=type=bind,from=cni-plugins,dst=/mnt/tmp \ - tar Cxzvf /opt/cni/bin /mnt/tmp/cni-plugins.tgz +# TODO: check SHA256SUMS of cni-plugins +ARG CNI_PLUGINS_VERSION +RUN arch="$(uname -m | sed -e s/x86_64/amd64/ -e s/aarch64/arm64/)" && \ + curl -fsSL https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${arch}-${CNI_PLUGINS_VERSION}.tgz \ + | tar Cxzv /opt/cni/bin # gettext-base: for `envsubst` # moreutils: for `sponge` # socat: for `socat` (to silence "[WARNING FileExisting-socat]" from kubeadm) diff --git a/Makefile b/Makefile index 8d512bf..d458f09 100644 --- a/Makefile +++ b/Makefile @@ -17,11 +17,14 @@ export U7S_NODE_NAME:= $(NODE_NAME) # Not accessible from other hosts. export U7S_NODE_SUBNET := $(NODE_SUBNET) -DOCKER ?= docker +CONTAINER_ENGINE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh CONTAINER_ENGINE) +export CONTAINER_ENGINE := $(CONTAINER_ENGINE) -export DOCKER := $(DOCKER) +CONTAINER_ENGINE_TYPE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh CONTAINER_ENGINE_TYPE) +export CONTAINER_ENGINE_TYPE := $(CONTAINER_ENGINE_TYPE) + +COMPOSE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh COMPOSE) -COMPOSE := $(DOCKER) compose NODE_SERVICE_NAME := node NODE_SHELL := $(COMPOSE) exec \ -e U7S_HOST_IP=$(U7S_HOST_IP) \ @@ -78,7 +81,7 @@ logs: .PHONY: kubeconfig kubeconfig: - $(COMPOSE) cp $(NODE_SERVICE_NAME):/etc/kubernetes/admin.conf ./kubeconfig + $(COMPOSE) exec -T $(NODE_SERVICE_NAME) cat /etc/kubernetes/admin.conf >kubeconfig @echo "# Run the following command by yourself:" @echo "export KUBECONFIG=$(shell pwd)/kubeconfig" ifeq ($(shell command -v kubectl 2> /dev/null),) @@ -88,7 +91,7 @@ endif .PHONY: kubectl kubectl: - $(COMPOSE) cp $(NODE_SERVICE_NAME):/usr/bin/kubectl ./kubectl + $(COMPOSE) exec -T --workdir=/usr/bin $(NODE_SERVICE_NAME) tar c kubectl | tar xv @echo "# Run the following command by yourself:" @echo "export PATH=$(shell pwd):\$$PATH" @echo "source <(kubectl completion bash)" diff --git a/Makefile.d/check-preflight.sh b/Makefile.d/check-preflight.sh index 534b67f..df4cd29 100755 --- a/Makefile.d/check-preflight.sh +++ b/Makefile.d/check-preflight.sh @@ -1,6 +1,9 @@ #!/bin/bash set -eu +function INFO() { + echo >&2 -e "\e[104m\e[97m[INFO]\e[49m\e[39m $@" +} function WARNING() { echo >&2 -e "\e[101m\e[97m[WARNING]\e[49m\e[39m $@" } @@ -9,12 +12,21 @@ function ERROR() { echo >&2 -e "\e[101m\e[97m[ERROR]\e[49m\e[39m $@" } -: "${DOCKER:=docker}" +script_dir="$(dirname "$0")" +detect_engine="${script_dir}"/detect_container_engine.sh +: "${CONTAINER_ENGINE:=$("${detect_engine}" CONTAINER_ENGINE)}" +: "${CONTAINER_ENGINE_TYPE:=$("${detect_engine}" CONTAINER_ENGINE_TYPE)}" : "${QUICK:=0}" -: "${BUSYBOX_IMAGE:=busybox}" +: "${BUSYBOX_IMAGE:=docker.io/library/busybox:latest}" + +if [ -z "${CONTAINER_ENGINE}" ] || [ -z "${CONTAINER_ENGINE_TYPE}" ]; then + ERROR "No container engine was detected" + exit 1 +fi +INFO "Detected container engine type: ${CONTAINER_ENGINE_TYPE}" # Check hard dependency commands -for f in make jq "${DOCKER}"; do +for f in make jq "${CONTAINER_ENGINE}"; do if ! command -v "${f}" >/dev/null 2>&1; then ERROR "Command \"${f}\" is not installed" exit 1 @@ -28,9 +40,22 @@ for f in kubectl; do fi done -# Check if Docker is running in Rootless mode -# TODO: support Podman? -if "${DOCKER}" info --format '{{json .SecurityOptions}}' | grep -q "name=rootless"; then +rootless= +case "${CONTAINER_ENGINE_TYPE}" in +"podman") + if [ "$(${CONTAINER_ENGINE} info --format '{{.Host.Security.Rootless}}')" = "true" ]; then + rootless=1 + fi + ;; +*) + if ${CONTAINER_ENGINE} info --format '{{json .SecurityOptions}}' | grep -q "name=rootless"; then + rootless=1 + fi + ;; +esac + +# Check if the container engine is running in Rootless mode +if [ "${rootless}" = "1" ]; then # Check systemd lingering: https://rootlesscontaine.rs/getting-started/common/login/ if command -v loginctl >/dev/null 2>&1; then if [ "$(loginctl list-users --output json | jq ".[] | select(.uid == "${UID}").linger")" != "true" ]; then @@ -57,7 +82,7 @@ if "${DOCKER}" info --format '{{json .SecurityOptions}}' | grep -q "name=rootles fi fi else - WARNING "Docker does not seem running in Rootless mode" + WARNING "Container engine (${CONTAINER_ENGINE}) does not seem running in Rootless mode" fi # Check kernel modules @@ -68,10 +93,10 @@ for f in br_netfilter ip6_tables ip6table_nat ip_tables iptable_nat vxlan; do done if [ "$QUICK" != "1" ]; then - # Check net.ipv4.conf.default.rp_filter in the daemon's network namespace. + # Check net.ipv4.conf.default.rp_filter in the container engine's network namespace. (e.g., netns of dockerd) # The value can be 0 (disabled) or 2 (loose), must not be 1 (strict). - if [ "$(${DOCKER} run --rm --net=host "${BUSYBOX_IMAGE}" sysctl -n net.ipv4.conf.default.rp_filter)" == "1" ]; then - ERROR "sysctl value \"net.ipv4.conf.default.rp_filter\" must be 0 (disabled) or 2 (loose) in the daemon's network namespace" + if [ "$(${CONTAINER_ENGINE} run --rm --net=host "${BUSYBOX_IMAGE}" sysctl -n net.ipv4.conf.default.rp_filter)" == "1" ]; then + ERROR "sysctl value \"net.ipv4.conf.default.rp_filter\" must be 0 (disabled) or 2 (loose) in the container engine's network namespace" exit 1 fi fi diff --git a/Makefile.d/detect_container_engine.sh b/Makefile.d/detect_container_engine.sh new file mode 100755 index 0000000..61e75f3 --- /dev/null +++ b/Makefile.d/detect_container_engine.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -eu -o pipefail +: "${CONTAINER_ENGINE:=}" +: "${COMPOSE:=}" + +if [ -z "${CONTAINER_ENGINE}" ]; then + if command -v dockerd-rootless.sh >/dev/null 2>&1; then + CONTAINER_ENGINE=docker + elif command -v containerd-rootless.sh >/dev/null 2>&1; then + CONTAINER_ENGINE=nerdctl + elif command -v podman >/dev/null 2>&1; then + CONTAINER_ENGINE=podman + else + echo >&2 "$0: no container engine was detected" + exit 1 + fi +fi + +CONTAINER_ENGINE_TYPE=docker +if [[ "${CONTAINER_ENGINE}" = *"podman"* ]]; then + CONTAINER_ENGINE_TYPE=podman +elif [[ "${CONTAINER_ENGINE}" = *"nerdctl"* ]]; then + CONTAINER_ENGINE_TYPE=nerdctl +fi + +if [ -z "${COMPOSE}" ]; then + COMPOSE="${CONTAINER_ENGINE} compose" + if [ "${CONTAINER_ENGINE_TYPE}" = "podman" ]; then + COMPOSE=podman-compose + fi +fi + +case "$#" in +0) + echo "CONTAINER_ENGINE=${CONTAINER_ENGINE}" + echo "CONTAINER_ENGINE_TYPE=${CONTAINER_ENGINE_TYPE}" + echo "COMPOSE=${COMPOSE}" + ;; +1) + case "$1" in + "CONTAINER_ENGINE" | "CONTAINER_ENGINE_TYPE" | "COMPOSE") + echo "${!1}" + ;; + *) + echo >&2 "$0: unknown argument: $1" + exit 1 + ;; + esac + ;; +*) + echo >&2 "$0: too many arguments" + exit 1 + ;; +esac diff --git a/README.md b/README.md index 9d30c54..c18726e 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,23 @@ but Usernetes (Gen 2) supports creating a cluster with multiple hosts. ## Requirements -- Host OS should be one of the following: - - Ubuntu 22.04 (recommended) - - Rocky Linux 9 - - AlmaLinux 9 +- One of the following host operating system: + +|Host operating system|Minimum version| +|---------------------|---------------| +|Ubuntu (recommended) |22.04 | +|Rocky Linux |9 | +|AlmaLinux |9 | +|Fedora |(?) | + +- One of the following container engines: + +|Container Engine |Minimum version| +|------------------------------------------------------------------------------------|---------------| +|[Rootless Docker](https://rootlesscontaine.rs/getting-started/docker/) (recommended)|v20.10 | +|[Rootless Podman](https://rootlesscontaine.rs/getting-started/podman/) |v4.x | +|[Rootless nerdctl](https://rootlesscontaine.rs/getting-started/containerd/) |v1.6 | -- [Rootless Docker](https://rootlesscontaine.rs/getting-started/docker/): ```bash curl -o install.sh -fsSL https://get.docker.com sudo sh install.sh @@ -98,6 +109,9 @@ make down-v kubectl taint nodes --all node-role.kubernetes.io/control-plane- ``` +The container engine defaults to Docker. +To change the container engine, set `export CONTAINER_ENGINE=podman` or `export CONTAINER_ENGINE=nerdctl`. + ## Limitations - Node ports cannot be exposed automatically. Edit [`docker-compose.yaml`](./docker-compose.yaml) for exposing additional node ports. - Most of host files are not visible with `hostPath` mounts. Edit [`docker-compose.yaml`](./docker-compose.yaml) for mounting additional files. diff --git a/docker-compose.yaml b/docker-compose.yaml index 0a5ba00..2152813 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -6,8 +6,7 @@ services: build: . hostname: ${U7S_NODE_NAME} privileged: true - restart: on-failure - tty: true + restart: always ports: # etcd - 2379:2379 diff --git a/hack/create-cluster-lxd.sh b/hack/create-cluster-lxd.sh index ceecb82..bc82d5f 100755 --- a/hack/create-cluster-lxd.sh +++ b/hack/create-cluster-lxd.sh @@ -1,28 +1,30 @@ #!/bin/bash set -eux -o pipefail +: "${CONTAINER_ENGINE:=docker}" + # Create Rootless Docker hosts ./hack/create-hosts-lxd.sh "${HOME}/.u7s-ci-hosts" host0 host1 SCP="scp -F ${HOME}/.u7s-ci-hosts/ssh_config" SSH="ssh -F ${HOME}/.u7s-ci-hosts/ssh_config" for host in host0 host1; do $SCP -r "$(pwd)" "${host}:~/usernetes" - $SSH "${USER}-sudo@${host}" sudo "~${USER}/usernetes/init-host/init-host.root.sh" + $SSH "${USER}-sudo@${host}" sudo CONTAINER_ENGINE="${CONTAINER_ENGINE}" "~${USER}/usernetes/init-host/init-host.root.sh" $SSH "${USER}-sudo@${host}" sudo loginctl enable-linger "${USER}" - $SSH "${host}" ~/usernetes/init-host/init-host.rootless.sh + $SSH "${host}" CONTAINER_ENGINE="${CONTAINER_ENGINE}" ~/usernetes/init-host/init-host.rootless.sh done # Launch a Kubernetes node inside a Rootless Docker host for host in host0 host1; do - $SSH "${host}" make -C ~/usernetes up + $SSH "${host}" CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes up done # Bootstrap a cluster with host0 -$SSH host0 make -C ~/usernetes kubeadm-init install-flannel kubeconfig join-command +$SSH host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-init install-flannel kubeconfig join-command # Let host1 join the cluster $SCP host0:~/usernetes/join-command host1:~/usernetes/join-command -$SSH host1 make -C ~/usernetes kubeadm-join +$SSH host1 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-join # Enable kubectl $SCP host0:~/usernetes/kubeconfig ./kubeconfig diff --git a/hack/create-hosts-lxd.sh b/hack/create-hosts-lxd.sh index 1965e78..a85f1fc 100755 --- a/hack/create-hosts-lxd.sh +++ b/hack/create-hosts-lxd.sh @@ -8,6 +8,7 @@ dir=$1 shift names=$* +: "${LXC_IMAGE:="ubuntu:22.04"}" LXC="sudo lxc" echo "USER=${USER}" @@ -42,7 +43,7 @@ EOF fi for name in ${names}; do - ${LXC} init ubuntu:22.04 "${name}" -c security.privileged=true -c security.nesting=true + ${LXC} init "${LXC_IMAGE}" "${name}" -c security.privileged=true -c security.nesting=true ${LXC} config device add "${name}" bind-boot disk source=/boot path=/boot readonly=true ${LXC} config set "${name}" user.user-data - <"${userdata}" ${LXC} start "${name}" diff --git a/init-host/init-host.root.d/install-podman.sh b/init-host/init-host.root.d/install-podman.sh new file mode 100755 index 0000000..79604ab --- /dev/null +++ b/init-host/init-host.root.d/install-podman.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# This script installs the latest release of Podman. +# Repository information is from https://podman.io/docs/installation#linux-distributions +set -eux -o pipefail +if [ "$(id -u)" != "0" ]; then + echo "Must run as the root" + exit 1 +fi + +if command -v dnf >/dev/null 2>&1; then + dnf install -y podman podman-compose +else + mkdir -p /etc/apt/keyrings + curl -fsSL "https://download.opensuse.org/repositories/devel:kubic:libcontainers:unstable/xUbuntu_$(lsb_release -rs)/Release.key" | + gpg --dearmor | + tee /etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg >/dev/null + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/devel_kubic_libcontainers_unstable.gpg]\ + https://download.opensuse.org/repositories/devel:kubic:libcontainers:unstable/xUbuntu_$(lsb_release -rs)/ /" | + tee /etc/apt/sources.list.d/devel:kubic:libcontainers:unstable.list >/dev/null + apt-get update -qq + apt-get -qq -y install podman + # No dpkg for podman-compose ? + pip3 install podman-compose +fi diff --git a/init-host/init-host.root.sh b/init-host/init-host.root.sh index 8aca580..a9b762f 100755 --- a/init-host/init-host.root.sh +++ b/init-host/init-host.root.sh @@ -6,6 +6,9 @@ if [ "$(id -u)" != "0" ]; then exit 1 fi +: "${CONTAINER_ENGINE:=docker}" +script_dir="$(dirname "$0")" + if [ ! -e /etc/systemd/system/user@.service.d/delegate.conf ]; then mkdir -p /etc/systemd/system/user@.service.d cat </etc/systemd/system/user@.service.d/delegate.conf @@ -19,7 +22,8 @@ cat >/etc/modules-load.d/usernetes.conf </etc/sysctl.d/99-usernetes.conf </etc/sysctl.d/99-usernetes.conf </dev/null 2>&1; then - if grep -q centos /etc/os-release; then - # Works with Rocky and Alma too - dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - dnf -y install docker-ce - else - curl https://get.docker.com | sh - fi -fi -systemctl disable --now docker +# sysctl may fail inside LXC +sysctl --system || true if command -v dnf >/dev/null 2>&1; then dnf install -y git shadow-utils make jq + # Workaround: SUID bit on newuidmap is dropped on LXC images:fedora/38/cloud, + # so it has to be reinstalled + dnf reinstall -y shadow-utils else + apt-get update apt-get install -y git uidmap make jq fi + +case "${CONTAINER_ENGINE}" in +"docker") + if ! command -v dockerd-rootless-setuptool.sh >/dev/null 2>&1; then + if grep -q centos /etc/os-release; then + # Works with Rocky and Alma too + dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + dnf -y install docker-ce + else + curl https://get.docker.com | sh + fi + fi + systemctl disable --now docker + ;; +"podman") + if ! command -v podman >/dev/null 2>&1; then + "${script_dir}"/init-host.root.d/install-podman.sh + fi + ;; +*) + echo >&2 "Unsupported container engine: ${CONTAINER_ENGINE}" + exit 1 + ;; +esac diff --git a/init-host/init-host.rootless.sh b/init-host/init-host.rootless.sh index bffccdc..096ede4 100755 --- a/init-host/init-host.rootless.sh +++ b/init-host/init-host.rootless.sh @@ -6,5 +6,17 @@ if [ "$(id -u)" == "0" ]; then exit 1 fi -dockerd-rootless-setuptool.sh install -docker info +: "${CONTAINER_ENGINE:=docker}" +case "${CONTAINER_ENGINE}" in +"docker") + dockerd-rootless-setuptool.sh install + ;; +"podman") + systemctl --user enable --now podman-restart + ;; +*) + # NOP + ;; +esac + +${CONTAINER_ENGINE} info