Remove ip addr add "${U7S_HOST_IP}" dev eth0

Fix issue 296

Also fix issue 293, 297

Signed-off-by: Akihiro Suda <[email protected]>

.github/workflows/main.yaml

@@ -58,6 +58,18 @@ jobs:
           make up
           sleep 30
           ./hack/test-smoke.sh
+      - name: "Debug"
+        if: failure()
+        run: |
+          podman exec usernetes_node_1 journalctl --since="1 day ago"
+      - name: "Debug 2"
+        if: failure()
+        run: |
+          podman exec usernetes_node_1 sh -euxc "tail -n 1000 /var/log/containers/kube-apiserver-*"
+      - name: "Debug 3"
+        if: failure()
+        run: |
+          podman exec usernetes_node_1 sh -euxc "tail -n 1000 /var/log/containers/etcd*"
 
   multi-node:
     name: "Multi node (emulated using LXD)"

Dockerfile

@@ -13,5 +13,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   gettext-base \
   moreutils \
   socat
+ADD Dockerfile.d/etc_udev_rules.d_90-flannel.rules /etc/udev/rules.d/90-flannel.rules
 ADD Dockerfile.d/u7s-entrypoint.sh /
 ENTRYPOINT ["/u7s-entrypoint.sh", "/usr/local/bin/entrypoint", "/sbin/init"]

Dockerfile.d/etc_udev_rules.d_90-flannel.rules

@@ -0,0 +1,5 @@
+# Correct UDP checksums for VXLAN behind NAT
+# https://github.com/flannel-io/flannel/issues/1279
+# https://github.com/kubernetes/kops/pull/9074
+# https://github.com/karmab/kcli/commit/b1a8eff658d17cf4e28162f0fa2c8b2b10e5ad00
+SUBSYSTEM=="net", ACTION=="add|change|move", ENV{INTERFACE}=="flannel.1", RUN+="/usr/sbin/ethtool -K flannel.1 tx-checksum-ip-generic off"

Dockerfile.d/u7s-entrypoint.sh

@@ -1,20 +1,10 @@
 #!/bin/bash
 set -eux -o pipefail
 
-# Append "---node-ip=${U7S_HOST_IP}" to "KUBELET_EXTRA_ARGS=..." in /etc/default/kubelet
-sed -e "s/\(^KUBELET_EXTRA_ARGS=.*\)/\\1 --node-ip=${U7S_HOST_IP}/" </etc/default/kubelet | sponge /etc/default/kubelet
+# Append "KUBELET_EXTRA_ARGS=..." in /etc/default/kubelet
+sed -e "s!\(^KUBELET_EXTRA_ARGS=.*\)!\\1 --cloud-provider=external --node-labels=usernetes/host-ip=${U7S_HOST_IP}!" </etc/default/kubelet | sponge /etc/default/kubelet
 
-# Let kubelet recognize ${U7S_HOST_IP} as its IP:
-# https://github.com/kubernetes/kubernetes/issues/54337#issuecomment-363597985
-ip addr add "${U7S_HOST_IP}" dev eth0
-
-cat <<EOF >/u7s-flanneld-wrapper.sh
-#!/bin/sh
-# Usage: /u7s-flanneld-wrapper.sh /opt/bin/flanneld --ip-masq --kube-subnet-mgr ...
-# This script is expected to be mounted inside a "docker.io/flannel/flannel" container.
-set -eux
-"\$@" --public-ip="${U7S_HOST_IP}"
-EOF
-chmod +x /u7s-flanneld-wrapper.sh
+# Import control plane hosts from previous boot
+[ -e /etc/hosts.u7s ] && cat /etc/hosts.u7s >>/etc/hosts
 
 exec "$@"

Makefile

@@ -10,12 +10,15 @@ NODE_NAME ?= u7s-$(HOSTNAME)
 NODE_SUBNET ?= $(shell $(CURDIR)/Makefile.d/node_subnet.sh)
 # U7S_HOST_IP is the IP address of the physical host. Accessible from other hosts.
 export U7S_HOST_IP := $(HOST_IP)
-# U7S_NODE_NAME is the IP address of the Kubernetes node running in Rootless Docker.
+# U7S_NODE_NAME is the host name of the Kubernetes node running in Rootless Docker.
 # Not accessible from other hosts.
 export U7S_NODE_NAME:= $(NODE_NAME)
 # U7S_NODE_NAME is the subnet of the Kubernetes node running in Rootless Docker.
 # Not accessible from other hosts.
 export U7S_NODE_SUBNET := $(NODE_SUBNET)
+# U7S_NODE_IP is the IP address of the Kubernetes node running in Rootless Docker.
+# Not accessible from other hosts.
+export U7S_NODE_IP := $(subst .0/24,.100,$(U7S_NODE_SUBNET))
 
 CONTAINER_ENGINE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh CONTAINER_ENGINE)
 export CONTAINER_ENGINE := $(CONTAINER_ENGINE)
@@ -30,6 +33,7 @@ NODE_SHELL := $(COMPOSE) exec \
 	-e U7S_HOST_IP=$(U7S_HOST_IP) \
 	-e U7S_NODE_NAME=$(U7S_NODE_NAME) \
 	-e U7S_NODE_SUBNET=$(U7S_NODE_SUBNET) \
+	-e U7S_NODE_IP=$(U7S_NODE_IP) \
 	$(NODE_SERVICE_NAME)
 
 .PHONY: help
@@ -48,6 +52,7 @@ help:
 	@echo 'make join-command'
 	@echo 'scp join-command another-host:~/usernetes'
 	@echo 'ssh another-host make -C ~/usernetes up kubeadm-join'
+	@echo 'make sync-external-ip'
 	@echo
 	@echo '# Debug'
 	@echo 'make logs'
@@ -81,7 +86,7 @@ logs:
 
 .PHONY: kubeconfig
 kubeconfig:
-	$(COMPOSE) exec -T $(NODE_SERVICE_NAME) cat /etc/kubernetes/admin.conf >kubeconfig
+	$(COMPOSE) exec -T $(NODE_SERVICE_NAME) sed -e "s/$(NODE_NAME)/127.0.0.1/g" /etc/kubernetes/admin.conf >kubeconfig
 	@echo "# Run the following command by yourself:"
 	@echo "export KUBECONFIG=$(shell pwd)/kubeconfig"
 ifeq ($(shell command -v kubectl 2> /dev/null),)
@@ -98,19 +103,35 @@ kubectl:
 
 .PHONY: join-command
 join-command:
-	$(NODE_SHELL) kubeadm token create --print-join-command | tr -d '\r' >join-command
-	@echo "# Copy the 'join-command' file to another host, and run 'make kubeadm-join' on that host (not on this host)"
+	echo "#!/bin/bash" >join-command
+	echo "set -eux -o pipefail" >>join-command
+	echo "echo \"$(HOST_IP)  $(NODE_NAME)\" >/etc/hosts.u7s" >>join-command
+	echo "cat /etc/hosts.u7s >>/etc/hosts" >>join-command
+	$(NODE_SHELL) kubeadm token create --print-join-command | tr -d '\r' >>join-command
+	chmod +x join-command
+	@echo "# Copy the 'join-command' file to another host, and run the following commands:"
+	@echo "# On the other host (the new worker):"
+	@echo "#   make kubeadm-join"
+	@echo "# On this host (the control plane):"
+	@echo "#   make sync-external-ip"
 
 .PHONY: kubeadm-init
 kubeadm-init:
 	$(NODE_SHELL) sh -euc "envsubst </usernetes/kubeadm-config.yaml >/tmp/kubeadm-config.yaml"
 	$(NODE_SHELL) kubeadm init --config /tmp/kubeadm-config.yaml --skip-token-print
+	$(MAKE) sync-external-ip
 	@echo "# Run 'make join-command' to print the join command"
 
+.PHONY: sync-external-ip
+sync-external-ip:
+	$(NODE_SHELL) /usernetes/Makefile.d/sync-external-ip.sh
+
 .PHONY: kubeadm-join
 kubeadm-join:
-	$(NODE_SHELL) sh -euc '$$(cat /usernetes/join-command)'
+	$(NODE_SHELL) sh -euc "envsubst </usernetes/kubeadm-config.yaml >/tmp/kubeadm-config.yaml"
+	$(NODE_SHELL) /usernetes/join-command
+	@echo "# Run 'make sync-external-ip' on the control plane"
 
 .PHONY: install-flannel
 install-flannel:
-	$(NODE_SHELL) kubectl apply -f /usernetes/manifests/kube-flannel.yml
+	$(NODE_SHELL) kubectl apply -f https://github.com/flannel-io/flannel/releases/download/v0.24.4/kube-flannel.yml

Makefile.d/sync-external-ip.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -eu -o pipefail
+
+for node in $(kubectl get nodes -o name); do
+	# Set ExternalIP
+	host_ip="$(kubectl get "${node}" -o jsonpath='{.metadata.labels.usernetes/host-ip}')"
+	kubectl patch "${node}" --type=merge --subresource status --patch \
+		"\"status\": {\"addresses\": [{\"type\":\"ExternalIP\", \"address\": \"${host_ip}\"}]}"
+
+	# Propagate ExternalIP to flannel
+	# https://github.com/flannel-io/flannel/blob/v0.24.4/Documentation/kubernetes.md#annotations
+	kubectl annotate "${node}" flannel.alpha.coreos.com/public-ip-overwrite=${host_ip}
+
+	# Remove taints
+	taints="$(kubectl get "${node}" -o jsonpath='{.spec.taints}')"
+	if echo "${taints}" | grep -q node.cloudprovider.kubernetes.io/uninitialized; then
+		kubectl taint nodes "${node}" node.cloudprovider.kubernetes.io/uninitialized-
+	fi
+done

README.md

@@ -101,6 +101,7 @@ kubectl get pods -A
 make join-command
 scp join-command another-host:~/usernetes
 ssh another-host make -C ~/usernetes up kubeadm-join
+make sync-external-ip
 
 # Debug
 make logs
@@ -117,9 +118,6 @@ To change the container engine, set `export CONTAINER_ENGINE=podman` or `export
 - Most of host files are not visible with `hostPath` mounts. Edit [`docker-compose.yaml`](./docker-compose.yaml) for mounting additional files.
 - Some [volume drivers](https://kubernetes.io/docs/concepts/storage/volumes/) such as `nfs` do not work.
 
-<!--
 ## Advanced topics
 - Although Usernetes (Gen2) is designed to be used with Rootless Docker, it should work with the regular "rootful" Docker too.
   This might be useful for some people who are looking for "multi-host" version of [`kind`](https://kind.sigs.k8s.io/) and [minikube](https://minikube.sigs.k8s.io/).
--->
-<!-- ↑FIXME: "rootful" support is broken: https://github.com/rootless-containers/usernetes/issues/297 -->

docker-compose.yaml

@@ -7,6 +7,9 @@ services:
     hostname: ${U7S_NODE_NAME}
     privileged: true
     restart: always
+    networks:
+      default:
+        ipv4_address: ${U7S_NODE_IP}
     ports:
       # etcd
       - 2379:2379

hack/create-cluster-lxd.sh

@@ -25,9 +25,11 @@ $SSH host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-in
 # Let host1 join the cluster
 $SCP host0:~/usernetes/join-command host1:~/usernetes/join-command
 $SSH host1 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-join
+$SSH host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes sync-external-ip
 
 # Enable kubectl
 $SCP host0:~/usernetes/kubeconfig ./kubeconfig
+sed -i -e "s/127.0.0.1/$($SSH host0 ip --json route get 1 | jq -r .[0].prefsrc)/g" ./kubeconfig
 KUBECONFIG="$(pwd)/kubeconfig"
 export KUBECONFIG
 kubectl get nodes -o wide

kubeadm-config.yaml

@@ -1,16 +1,27 @@
 ---
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: InitConfiguration
-localAPIEndpoint:
-  advertiseAddress: "${U7S_HOST_IP}"
-  bindPort: 6443
 ---
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: ClusterConfiguration
 networking:
   serviceSubnet: "10.96.0.0/16"
   podSubnet: "10.244.0.0/16"
-controlPlaneEndpoint: "${U7S_HOST_IP}:6443"
+controlPlaneEndpoint: "${U7S_NODE_NAME}:6443"
+apiServer:
+  extraArgs:
+    advertise-address: "${U7S_HOST_IP}"
+    cloud-provider: external
+    # Default: "Hostname,InternalDNS,InternalIP,ExternalDNS,ExternalIP"
+    kubelet-preferred-address-types: "ExternalIP"
+  certSANs:
+    - localhost
+    - 127.0.0.1
+    - "${U7S_NODE_NAME}"
+    - "${U7S_HOST_IP}"
+controllerManager:
+  extraArgs:
+    cloud-provider: external
 ---
 kind: KubeletConfiguration
 apiVersion: kubelet.config.k8s.io/v1beta1