[WIP] Remove ip addr add "${U7S_HOST_IP}" dev eth0

Fix issue 296

Also fix issue 293, 297

Signed-off-by: Akihiro Suda <[email protected]>

Dockerfile.d/u7s-entrypoint.sh

@@ -1,20 +1,7 @@
 #!/bin/bash
 set -eux -o pipefail
 
-# Append "---node-ip=${U7S_HOST_IP}" to "KUBELET_EXTRA_ARGS=..." in /etc/default/kubelet
-sed -e "s/\(^KUBELET_EXTRA_ARGS=.*\)/\\1 --node-ip=${U7S_HOST_IP}/" </etc/default/kubelet | sponge /etc/default/kubelet
-
-# Let kubelet recognize ${U7S_HOST_IP} as its IP:
-# https://github.com/kubernetes/kubernetes/issues/54337#issuecomment-363597985
-ip addr add "${U7S_HOST_IP}" dev eth0
-
-cat <<EOF >/u7s-flanneld-wrapper.sh
-#!/bin/sh
-# Usage: /u7s-flanneld-wrapper.sh /opt/bin/flanneld --ip-masq --kube-subnet-mgr ...
-# This script is expected to be mounted inside a "docker.io/flannel/flannel" container.
-set -eux
-"\$@" --public-ip="${U7S_HOST_IP}"
-EOF
-chmod +x /u7s-flanneld-wrapper.sh
+# Append "KUBELET_EXTRA_ARGS=..." in /etc/default/kubelet
+sed -e "s!\(^KUBELET_EXTRA_ARGS=.*\)!\\1 --cloud-provider=external --node-labels=usernetes/host-ip=${U7S_HOST_IP}!" </etc/default/kubelet | sponge /etc/default/kubelet
 
 exec "$@"

Makefile

@@ -48,6 +48,7 @@ help:
 	@echo 'make join-command'
 	@echo 'scp join-command another-host:~/usernetes'
 	@echo 'ssh another-host make -C ~/usernetes up kubeadm-join'
+	@echo 'make sync-external-ip'
 	@echo
 	@echo '# Debug'
 	@echo 'make logs'
@@ -81,7 +82,7 @@ logs:
 
 .PHONY: kubeconfig
 kubeconfig:
-	$(COMPOSE) exec -T $(NODE_SERVICE_NAME) cat /etc/kubernetes/admin.conf >kubeconfig
+	$(COMPOSE) exec -T $(NODE_SERVICE_NAME) sed -e "s/$(NODE_NAME)/localhost/g" /etc/kubernetes/admin.conf >kubeconfig
 	@echo "# Run the following command by yourself:"
 	@echo "export KUBECONFIG=$(shell pwd)/kubeconfig"
 ifeq ($(shell command -v kubectl 2> /dev/null),)
@@ -98,19 +99,37 @@ kubectl:
 
 .PHONY: join-command
 join-command:
-	$(NODE_SHELL) kubeadm token create --print-join-command | tr -d '\r' >join-command
-	@echo "# Copy the 'join-command' file to another host, and run 'make kubeadm-join' on that host (not on this host)"
+	echo "#!/bin/bash" >join-command
+	echo "set -eux -o pipefail" >>join-command
+	echo "grep -q -w $(NODE_NAME) /etc/hosts || echo \"$(HOST_IP)  $(NODE_NAME)\" >>/etc/hosts" >>join-command
+	$(NODE_SHELL) kubeadm token create --print-join-command | tr -d '\r' >>join-command
+	chmod +x join-command
+	@echo "# Copy the 'join-command' file to another host, and run the following commands:"
+	@echo "# On the other host (the new worker):"
+	@echo "#   make kubeadm-join"
+	@echo "# On this host (the control plane):"
+	@echo "#   make sync-external-ip"
 
 .PHONY: kubeadm-init
 kubeadm-init:
 	$(NODE_SHELL) sh -euc "envsubst </usernetes/kubeadm-config.yaml >/tmp/kubeadm-config.yaml"
 	$(NODE_SHELL) kubeadm init --config /tmp/kubeadm-config.yaml --skip-token-print
+	$(MAKE) sync-external-ip
 	@echo "# Run 'make join-command' to print the join command"
 
+.PHONY: sync-external-ip
+sync-external-ip:
+	$(NODE_SHELL) /usernetes/Makefile.d/sync-external-ip.sh
+
 .PHONY: kubeadm-join
 kubeadm-join:
-	$(NODE_SHELL) sh -euc '$$(cat /usernetes/join-command)'
+	$(NODE_SHELL) sh -euc "envsubst </usernetes/kubeadm-config.yaml >/tmp/kubeadm-config.yaml"
+	$(NODE_SHELL) /usernetes/join-command
+	@echo "# Run 'make sync-external-ip' on the control plane"
 
 .PHONY: install-flannel
 install-flannel:
-	$(NODE_SHELL) kubectl apply -f /usernetes/manifests/kube-flannel.yml
+	$(NODE_SHELL) kubectl apply -f https://github.com/flannel-io/flannel/releases/download/v0.24.4/kube-flannel.yml
+	# flanneld attempts to retrieve the Kubernetes objects from 10.96.0.1:443, but it is not accessible until flannel is properly configured
+	$(NODE_SHELL) kubectl -n kube-flannel set env daemonset.apps/kube-flannel-ds KUBERNETES_SERVICE_HOST=$(HOST_IP)
+	$(NODE_SHELL) kubectl -n kube-flannel set env daemonset.apps/kube-flannel-ds KUBERNETES_SERVICE_PORT=6443

Makefile.d/sync-external-ip.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -eu -o pipefail
+
+for node in $(kubectl get nodes -o name); do
+	# Set ExternalIP
+	host_ip="$(kubectl get "${node}" -o jsonpath='{.metadata.labels.usernetes/host-ip}')"
+	kubectl patch "${node}" --type=merge --subresource status --patch \
+		"\"status\": {\"addresses\": [{\"type\":\"ExternalIP\", \"address\": \"${host_ip}\"}]}"
+
+	# Propagate ExternalIP to flannel
+	# https://github.com/flannel-io/flannel/blob/v0.24.4/Documentation/kubernetes.md#annotations
+	kubectl annotate "${node}" flannel.alpha.coreos.com/public-ip-overwrite=${host_ip}
+
+	# Remove taints
+	taints="$(kubectl get "${node}" -o jsonpath='{.spec.taints}')"
+	if echo "${taints}" | grep -q node.cloudprovider.kubernetes.io/uninitialized; then
+		kubectl taint nodes "${node}" node.cloudprovider.kubernetes.io/uninitialized-
+	fi
+done

README.md

@@ -101,6 +101,7 @@ kubectl get pods -A
 make join-command
 scp join-command another-host:~/usernetes
 ssh another-host make -C ~/usernetes up kubeadm-join
+make sync-external-ip
 
 # Debug
 make logs
@@ -117,9 +118,6 @@ To change the container engine, set `export CONTAINER_ENGINE=podman` or `export
 - Most of host files are not visible with `hostPath` mounts. Edit [`docker-compose.yaml`](./docker-compose.yaml) for mounting additional files.
 - Some [volume drivers](https://kubernetes.io/docs/concepts/storage/volumes/) such as `nfs` do not work.
 
-<!--
 ## Advanced topics
 - Although Usernetes (Gen2) is designed to be used with Rootless Docker, it should work with the regular "rootful" Docker too.
   This might be useful for some people who are looking for "multi-host" version of [`kind`](https://kind.sigs.k8s.io/) and [minikube](https://minikube.sigs.k8s.io/).
--->
-<!-- ↑FIXME: "rootful" support is broken: https://github.com/rootless-containers/usernetes/issues/297 -->

hack/create-cluster-lxd.sh

@@ -25,8 +25,10 @@ $SSH host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-in
 # Let host1 join the cluster
 $SCP host0:~/usernetes/join-command host1:~/usernetes/join-command
 $SSH host1 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-join
+$SSH host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes sync-external-ip
 
 # Enable kubectl
+$SSH -fNT -L 6443:localhost:6443
 $SCP host0:~/usernetes/kubeconfig ./kubeconfig
 KUBECONFIG="$(pwd)/kubeconfig"
 export KUBECONFIG

kubeadm-config.yaml

@@ -1,16 +1,19 @@
 ---
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: InitConfiguration
-localAPIEndpoint:
-  advertiseAddress: "${U7S_HOST_IP}"
-  bindPort: 6443
 ---
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: ClusterConfiguration
 networking:
   serviceSubnet: "10.96.0.0/16"
   podSubnet: "10.244.0.0/16"
-controlPlaneEndpoint: "${U7S_HOST_IP}:6443"
+controlPlaneEndpoint: "${U7S_NODE_NAME}:6443"
+apiServer:
+  certSANs:
+    - localhost
+    - 127.0.0.1
+    - "${U7S_NODE_NAME}"
+    - "${U7S_HOST_IP}"
 ---
 kind: KubeletConfiguration
 apiVersion: kubelet.config.k8s.io/v1beta1