Remove ip addr add "${U7S_HOST_IP}" dev eth0

Dockerfile

@@ -1,17 +1,22 @@
-ARG BASE_IMAGE=docker.io/kindest/node:v1.29.1@sha256:a0cc28af37cf39b019e2b448c54d1a3f789de32536cb5a5db61a49623e527144
-ARG CNI_PLUGINS_VERSION=v1.4.0
+ARG BASE_IMAGE=docker.io/kindest/node:v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245
+ARG CNI_PLUGINS_VERSION=v1.4.1
 FROM ${BASE_IMAGE}
-# TODO: check SHA256SUMS of cni-plugins
+COPY Dockerfile.d/SHA256SUMS.d/ /tmp/SHA256SUMS.d
 ARG CNI_PLUGINS_VERSION
 RUN arch="$(uname -m | sed -e s/x86_64/amd64/ -e s/aarch64/arm64/)" && \
-  curl -fsSL https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${arch}-${CNI_PLUGINS_VERSION}.tgz \
-  | tar Cxzv /opt/cni/bin
+  fname="cni-plugins-linux-${arch}-${CNI_PLUGINS_VERSION}.tgz" && \
+  curl -o "${fname}" -fSL "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/${fname}" && \
+  grep "${fname}" "/tmp/SHA256SUMS.d/cni-plugins-${CNI_PLUGINS_VERSION}" | sha256sum -c && \
+  mkdir -p /opt/cni/bin && \
+  tar xzf "${fname}" -C /opt/cni/bin && \
+  rm -f "${fname}"
 # gettext-base: for `envsubst`
 # moreutils: for `sponge`
 # socat: for `socat` (to silence "[WARNING FileExisting-socat]" from kubeadm)
 RUN apt-get update && apt-get install -y --no-install-recommends \
   gettext-base \
   moreutils \
   socat
+ADD Dockerfile.d/etc_udev_rules.d_90-flannel.rules /etc/udev/rules.d/90-flannel.rules
 ADD Dockerfile.d/u7s-entrypoint.sh /
 ENTRYPOINT ["/u7s-entrypoint.sh", "/usr/local/bin/entrypoint", "/sbin/init"]

Dockerfile.d/SHA256SUMS.d/cni-plugins-v1.4.1

@@ -0,0 +1,2 @@
+1511f6c003ace805eafeb1132727791326283cff88a923d76329e1892bba7a10  cni-plugins-linux-amd64-v1.4.1.tgz
+72644e13557cda8a5b39baf97fc5e93d23fdf7baba7700000e7e9efd8bdf9234  cni-plugins-linux-arm64-v1.4.1.tgz

Dockerfile.d/etc_udev_rules.d_90-flannel.rules

@@ -0,0 +1,5 @@
+# Correct UDP checksums for VXLAN behind NAT
+# https://github.com/flannel-io/flannel/issues/1279
+# https://github.com/kubernetes/kops/pull/9074
+# https://github.com/karmab/kcli/commit/b1a8eff658d17cf4e28162f0fa2c8b2b10e5ad00
+SUBSYSTEM=="net", ACTION=="add|change|move", ENV{INTERFACE}=="flannel.1", RUN+="/usr/sbin/ethtool -K flannel.1 tx-checksum-ip-generic off"

Dockerfile.d/u7s-entrypoint.sh

@@ -1,20 +1,10 @@
 #!/bin/bash
 set -eux -o pipefail
 
-# Append "---node-ip=${U7S_HOST_IP}" to "KUBELET_EXTRA_ARGS=..." in /etc/default/kubelet
-sed -e "s/\(^KUBELET_EXTRA_ARGS=.*\)/\\1 --node-ip=${U7S_HOST_IP}/" </etc/default/kubelet | sponge /etc/default/kubelet
+# Append "KUBELET_EXTRA_ARGS=..." in /etc/default/kubelet
+sed -e "s!\(^KUBELET_EXTRA_ARGS=.*\)!\\1 --cloud-provider=external --node-labels=usernetes/host-ip=${U7S_HOST_IP}!" </etc/default/kubelet | sponge /etc/default/kubelet
 
-# Let kubelet recognize ${U7S_HOST_IP} as its IP:
-# https://github.com/kubernetes/kubernetes/issues/54337#issuecomment-363597985
-ip addr add "${U7S_HOST_IP}" dev eth0
-
-cat <<EOF >/u7s-flanneld-wrapper.sh
-#!/bin/sh
-# Usage: /u7s-flanneld-wrapper.sh /opt/bin/flanneld --ip-masq --kube-subnet-mgr ...
-# This script is expected to be mounted inside a "docker.io/flannel/flannel" container.
-set -eux
-"\$@" --public-ip="${U7S_HOST_IP}"
-EOF
-chmod +x /u7s-flanneld-wrapper.sh
+# Import control plane hosts from previous boot
+[ -e /etc/hosts.u7s ] && cat /etc/hosts.u7s >>/etc/hosts
 
 exec "$@"

Makefile

@@ -7,29 +7,33 @@ export HOSTNAME := $(HOSTNAME)
 
 HOST_IP ?= $(shell ip --json route get 1 | jq -r .[0].prefsrc)
 NODE_NAME ?= u7s-$(HOSTNAME)
-NODE_SUBNET ?= $(shell $(CURDIR)/Makefile.d/node_subnet.sh)
+NODE_SUBNET ?= $(shell $(CURDIR)/Makefile.d/node-subnet.sh)
 # U7S_HOST_IP is the IP address of the physical host. Accessible from other hosts.
 export U7S_HOST_IP := $(HOST_IP)
-# U7S_NODE_NAME is the IP address of the Kubernetes node running in Rootless Docker.
+# U7S_NODE_NAME is the host name of the Kubernetes node running in Rootless Docker.
 # Not accessible from other hosts.
 export U7S_NODE_NAME:= $(NODE_NAME)
 # U7S_NODE_NAME is the subnet of the Kubernetes node running in Rootless Docker.
 # Not accessible from other hosts.
 export U7S_NODE_SUBNET := $(NODE_SUBNET)
+# U7S_NODE_IP is the IP address of the Kubernetes node running in Rootless Docker.
+# Not accessible from other hosts.
+export U7S_NODE_IP := $(subst .0/24,.100,$(U7S_NODE_SUBNET))
 
-CONTAINER_ENGINE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh CONTAINER_ENGINE)
+CONTAINER_ENGINE ?= $(shell $(CURDIR)/Makefile.d/detect-container-engine.sh CONTAINER_ENGINE)
 export CONTAINER_ENGINE := $(CONTAINER_ENGINE)
 
-CONTAINER_ENGINE_TYPE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh CONTAINER_ENGINE_TYPE)
+CONTAINER_ENGINE_TYPE ?= $(shell $(CURDIR)/Makefile.d/detect-container-engine.sh CONTAINER_ENGINE_TYPE)
 export CONTAINER_ENGINE_TYPE := $(CONTAINER_ENGINE_TYPE)
 
-COMPOSE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh COMPOSE)
+COMPOSE ?= $(shell $(CURDIR)/Makefile.d/detect-container-engine.sh COMPOSE)
 
 NODE_SERVICE_NAME := node
 NODE_SHELL := $(COMPOSE) exec \
 	-e U7S_HOST_IP=$(U7S_HOST_IP) \
 	-e U7S_NODE_NAME=$(U7S_NODE_NAME) \
 	-e U7S_NODE_SUBNET=$(U7S_NODE_SUBNET) \
+	-e U7S_NODE_IP=$(U7S_NODE_IP) \
 	$(NODE_SERVICE_NAME)
 
 .PHONY: help
@@ -48,6 +52,7 @@ help:
 	@echo 'make join-command'
 	@echo 'scp join-command another-host:~/usernetes'
 	@echo 'ssh another-host make -C ~/usernetes up kubeadm-join'
+	@echo 'make sync-external-ip'
 	@echo
 	@echo '# Debug'
 	@echo 'make logs'
@@ -81,7 +86,7 @@ logs:
 
 .PHONY: kubeconfig
 kubeconfig:
-	$(COMPOSE) exec -T $(NODE_SERVICE_NAME) cat /etc/kubernetes/admin.conf >kubeconfig
+	$(COMPOSE) exec -T $(NODE_SERVICE_NAME) sed -e "s/$(NODE_NAME)/127.0.0.1/g" /etc/kubernetes/admin.conf >kubeconfig
 	@echo "# Run the following command by yourself:"
 	@echo "export KUBECONFIG=$(shell pwd)/kubeconfig"
 ifeq ($(shell command -v kubectl 2> /dev/null),)
@@ -98,19 +103,35 @@ kubectl:
 
 .PHONY: join-command
 join-command:
-	$(NODE_SHELL) kubeadm token create --print-join-command | tr -d '\r' >join-command
-	@echo "# Copy the 'join-command' file to another host, and run 'make kubeadm-join' on that host (not on this host)"
+	echo "#!/bin/bash" >join-command
+	echo "set -eux -o pipefail" >>join-command
+	echo "echo \"$(HOST_IP)  $(NODE_NAME)\" >/etc/hosts.u7s" >>join-command
+	echo "cat /etc/hosts.u7s >>/etc/hosts" >>join-command
+	$(NODE_SHELL) kubeadm token create --print-join-command | tr -d '\r' >>join-command
+	chmod +x join-command
+	@echo "# Copy the 'join-command' file to another host, and run the following commands:"
+	@echo "# On the other host (the new worker):"
+	@echo "#   make kubeadm-join"
+	@echo "# On this host (the control plane):"
+	@echo "#   make sync-external-ip"
 
 .PHONY: kubeadm-init
 kubeadm-init:
 	$(NODE_SHELL) sh -euc "envsubst </usernetes/kubeadm-config.yaml >/tmp/kubeadm-config.yaml"
 	$(NODE_SHELL) kubeadm init --config /tmp/kubeadm-config.yaml --skip-token-print
+	$(MAKE) sync-external-ip
 	@echo "# Run 'make join-command' to print the join command"
 
+.PHONY: sync-external-ip
+sync-external-ip:
+	$(NODE_SHELL) /usernetes/Makefile.d/sync-external-ip.sh
+
 .PHONY: kubeadm-join
 kubeadm-join:
-	$(NODE_SHELL) sh -euc '$$(cat /usernetes/join-command)'
+	$(NODE_SHELL) sh -euc "envsubst </usernetes/kubeadm-config.yaml >/tmp/kubeadm-config.yaml"
+	$(NODE_SHELL) /usernetes/join-command
+	@echo "# Run 'make sync-external-ip' on the control plane"
 
 .PHONY: install-flannel
 install-flannel:
-	$(NODE_SHELL) kubectl apply -f /usernetes/manifests/kube-flannel.yml
+	$(NODE_SHELL) kubectl apply -f https://github.com/flannel-io/flannel/releases/download/v0.24.4/kube-flannel.yml

Makefile.d/check-preflight.sh

@@ -13,7 +13,7 @@ function ERROR() {
 }
 
 script_dir="$(dirname "$0")"
-detect_engine="${script_dir}"/detect_container_engine.sh
+detect_engine="${script_dir}"/detect-container-engine.sh
 : "${CONTAINER_ENGINE:=$("${detect_engine}" CONTAINER_ENGINE)}"
 : "${CONTAINER_ENGINE_TYPE:=$("${detect_engine}" CONTAINER_ENGINE_TYPE)}"
 : "${QUICK:=0}"

Makefile.d/sync-external-ip.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -eu -o pipefail
+
+for node in $(kubectl get nodes -o name); do
+	# Set ExternalIP
+	host_ip="$(kubectl get "${node}" -o jsonpath='{.metadata.labels.usernetes/host-ip}')"
+	kubectl patch "${node}" --type=merge --subresource status --patch \
+		"\"status\": {\"addresses\": [{\"type\":\"ExternalIP\", \"address\": \"${host_ip}\"}]}"
+
+	# Propagate ExternalIP to flannel
+	# https://github.com/flannel-io/flannel/blob/v0.24.4/Documentation/kubernetes.md#annotations
+	kubectl annotate "${node}" flannel.alpha.coreos.com/public-ip-overwrite=${host_ip}
+
+	# Remove taints
+	taints="$(kubectl get "${node}" -o jsonpath='{.spec.taints}')"
+	if echo "${taints}" | grep -q node.cloudprovider.kubernetes.io/uninitialized; then
+		kubectl taint nodes "${node}" node.cloudprovider.kubernetes.io/uninitialized-
+	fi
+done

README.md

@@ -101,6 +101,7 @@ kubectl get pods -A
 make join-command
 scp join-command another-host:~/usernetes
 ssh another-host make -C ~/usernetes up kubeadm-join
+make sync-external-ip
 
 # Debug
 make logs
@@ -117,9 +118,6 @@ To change the container engine, set `export CONTAINER_ENGINE=podman` or `export
 - Most of host files are not visible with `hostPath` mounts. Edit [`docker-compose.yaml`](./docker-compose.yaml) for mounting additional files.
 - Some [volume drivers](https://kubernetes.io/docs/concepts/storage/volumes/) such as `nfs` do not work.
 
-<!--
 ## Advanced topics
 - Although Usernetes (Gen2) is designed to be used with Rootless Docker, it should work with the regular "rootful" Docker too.
   This might be useful for some people who are looking for "multi-host" version of [`kind`](https://kind.sigs.k8s.io/) and [minikube](https://minikube.sigs.k8s.io/).
--->
-<!-- ↑FIXME: "rootful" support is broken: https://github.com/rootless-containers/usernetes/issues/297 -->

docker-compose.yaml

@@ -7,6 +7,9 @@ services:
     hostname: ${U7S_NODE_NAME}
     privileged: true
     restart: always
+    networks:
+      default:
+        ipv4_address: ${U7S_NODE_IP}
     ports:
       # etcd
       - 2379:2379

hack/create-cluster-lxd.sh

@@ -25,9 +25,11 @@ $SSH host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-in
 # Let host1 join the cluster
 $SCP host0:~/usernetes/join-command host1:~/usernetes/join-command
 $SSH host1 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-join
+$SSH host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes sync-external-ip
 
 # Enable kubectl
 $SCP host0:~/usernetes/kubeconfig ./kubeconfig
+sed -i -e "s/127.0.0.1/$($SSH host0 ip --json route get 1 | jq -r .[0].prefsrc)/g" ./kubeconfig
 KUBECONFIG="$(pwd)/kubeconfig"
 export KUBECONFIG
 kubectl get nodes -o wide

init-host/init-host.root.sh

@@ -40,7 +40,7 @@ if command -v dnf >/dev/null 2>&1; then
 	# so it has to be reinstalled
 	dnf reinstall -y shadow-utils
 else
-  apt-get update
+	apt-get update
 	apt-get install -y git uidmap make jq
 fi
 

kubeadm-config.yaml

@@ -1,16 +1,27 @@
 ---
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: InitConfiguration
-localAPIEndpoint:
-  advertiseAddress: "${U7S_HOST_IP}"
-  bindPort: 6443
 ---
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: ClusterConfiguration
 networking:
   serviceSubnet: "10.96.0.0/16"
   podSubnet: "10.244.0.0/16"
-controlPlaneEndpoint: "${U7S_HOST_IP}:6443"
+controlPlaneEndpoint: "${U7S_NODE_NAME}:6443"
+apiServer:
+  extraArgs:
+    advertise-address: "${U7S_HOST_IP}"
+    cloud-provider: external
+    # Default: "Hostname,InternalDNS,InternalIP,ExternalDNS,ExternalIP"
+    kubelet-preferred-address-types: "ExternalIP"
+  certSANs:
+    - localhost
+    - 127.0.0.1
+    - "${U7S_NODE_NAME}"
+    - "${U7S_HOST_IP}"
+controllerManager:
+  extraArgs:
+    cloud-provider: external
 ---
 kind: KubeletConfiguration
 apiVersion: kubelet.config.k8s.io/v1beta1