[WIP] Remove ip addr add "${U7S_HOST_IP}" dev eth0

Fix issue 296

Also fix issue 293, 297

Signed-off-by: Akihiro Suda <[email protected]>

Dockerfile

@@ -7,11 +7,7 @@ RUN arch="$(uname -m | sed -e s/x86_64/amd64/ -e s/aarch64/arm64/)" && \
   curl -fsSL https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${arch}-${CNI_PLUGINS_VERSION}.tgz \
   | tar Cxzv /opt/cni/bin
 # gettext-base: for `envsubst`
-# moreutils: for `sponge`
 # socat: for `socat` (to silence "[WARNING FileExisting-socat]" from kubeadm)
 RUN apt-get update && apt-get install -y --no-install-recommends \
   gettext-base \
-  moreutils \
   socat
-ADD Dockerfile.d/u7s-entrypoint.sh /
-ENTRYPOINT ["/u7s-entrypoint.sh", "/usr/local/bin/entrypoint", "/sbin/init"]

Makefile

@@ -48,6 +48,7 @@ help:
 	@echo 'make join-command'
 	@echo 'scp join-command another-host:~/usernetes'
 	@echo 'ssh another-host make -C ~/usernetes up kubeadm-join'
+	@echo 'make sync-external-ip'
 	@echo
 	@echo '# Debug'
 	@echo 'make logs'
@@ -81,7 +82,7 @@ logs:
 
 .PHONY: kubeconfig
 kubeconfig:
-	$(COMPOSE) exec -T $(NODE_SERVICE_NAME) cat /etc/kubernetes/admin.conf >kubeconfig
+	$(COMPOSE) exec -T $(NODE_SERVICE_NAME) sed -e "s/$(NODE_NAME)/localhost/g" /etc/kubernetes/admin.conf >kubeconfig
 	@echo "# Run the following command by yourself:"
 	@echo "export KUBECONFIG=$(shell pwd)/kubeconfig"
 ifeq ($(shell command -v kubectl 2> /dev/null),)
@@ -98,19 +99,31 @@ kubectl:
 
 .PHONY: join-command
 join-command:
-	$(NODE_SHELL) kubeadm token create --print-join-command | tr -d '\r' >join-command
-	@echo "# Copy the 'join-command' file to another host, and run 'make kubeadm-join' on that host (not on this host)"
+	echo "grep -q -w $(NODE_NAME) /etc/hosts || echo \"$(HOST_IP)  $(NODE_NAME)\" >>/etc/hosts" >join-command
+	$(NODE_SHELL) kubeadm token create --print-join-command | tr -d '\r' | sed -e "s/$(NODE_NAME)/$(HOST_IP)/g" -e "s@\$$@--config /tmp/kubeadm-config.yaml@" >>join-command
+	@echo "# Copy the 'join-command' file to another host, and run the following commands:"
+	@echo "# On the other host (the new worker):"
+	@echo "#   make kubeadm-join"
+	@echo "# On this host (the control plane):"
+	@echo "#   make sync-external-ip"
 
 .PHONY: kubeadm-init
 kubeadm-init:
 	$(NODE_SHELL) sh -euc "envsubst </usernetes/kubeadm-config.yaml >/tmp/kubeadm-config.yaml"
 	$(NODE_SHELL) kubeadm init --config /tmp/kubeadm-config.yaml --skip-token-print
+	$(MAKE) sync-external-ip
 	@echo "# Run 'make join-command' to print the join command"
 
+.PHONY: sync-external-ip
+sync-external-ip:
+	$(NODE_SHELL) /usernetes/Makefile.d/sync-external-ip.sh
+
 .PHONY: kubeadm-join
 kubeadm-join:
-	$(NODE_SHELL) sh -euc '$$(cat /usernetes/join-command)'
+	$(NODE_SHELL) sh -euc "envsubst </usernetes/kubeadm-config.yaml >/tmp/kubeadm-config.yaml"
+	$(NODE_SHELL) sh -euc /usernetes/join-command
+	@echo "# Run 'make sync-external-ip' on the control plane"
 
 .PHONY: install-flannel
 install-flannel:
-	$(NODE_SHELL) kubectl apply -f /usernetes/manifests/kube-flannel.yml
+	$(NODE_SHELL) kubectl apply -f https://github.com/flannel-io/flannel/releases/download/v0.24.4/kube-flannel.yml

Makefile.d/sync-external-ip.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -eu -o pipefail
+
+for node in $(kubectl get nodes -o name); do
+	host_ip="$(kubectl get "${node}" -o jsonpath='{.metadata.labels.usernetes/host-ip}')"
+	kubectl patch "${node}" --type=merge --subresource status --patch \
+		"\"status\": {\"addresses\": [{\"type\":\"ExternalIP\", \"address\": \"${host_ip}\"}]}"
+	taints="$(kubectl get "${node}" -o jsonpath='{.spec.taints}')"
+	if echo "${taints}" | grep -q node.cloudprovider.kubernetes.io/uninitialized; then
+		kubectl taint nodes "${node}" node.cloudprovider.kubernetes.io/uninitialized-
+	fi
+done

README.md

@@ -101,6 +101,7 @@ kubectl get pods -A
 make join-command
 scp join-command another-host:~/usernetes
 ssh another-host make -C ~/usernetes up kubeadm-join
+make sync-external-ip
 
 # Debug
 make logs
@@ -117,9 +118,6 @@ To change the container engine, set `export CONTAINER_ENGINE=podman` or `export
 - Most of host files are not visible with `hostPath` mounts. Edit [`docker-compose.yaml`](./docker-compose.yaml) for mounting additional files.
 - Some [volume drivers](https://kubernetes.io/docs/concepts/storage/volumes/) such as `nfs` do not work.
 
-<!--
 ## Advanced topics
 - Although Usernetes (Gen2) is designed to be used with Rootless Docker, it should work with the regular "rootful" Docker too.
   This might be useful for some people who are looking for "multi-host" version of [`kind`](https://kind.sigs.k8s.io/) and [minikube](https://minikube.sigs.k8s.io/).
--->
-<!-- ↑FIXME: "rootful" support is broken: https://github.com/rootless-containers/usernetes/issues/297 -->

hack/create-cluster-lxd.sh

@@ -25,6 +25,7 @@ $SSH host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-in
 # Let host1 join the cluster
 $SCP host0:~/usernetes/join-command host1:~/usernetes/join-command
 $SSH host1 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-join
+$SSH host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes sync-external-ip
 
 # Enable kubectl
 $SCP host0:~/usernetes/kubeconfig ./kubeconfig

kubeadm-config.yaml

@@ -1,16 +1,30 @@
 ---
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: InitConfiguration
-localAPIEndpoint:
-  advertiseAddress: "${U7S_HOST_IP}"
-  bindPort: 6443
+nodeRegistration:
+  kubeletExtraArgs:
+    cloud-provider: external
+    node-labels: "usernetes/host-ip=${U7S_HOST_IP}"
+---
+apiVersion: kubeadm.k8s.io/v1beta3
+kind: JoinConfiguration
+nodeRegistration:
+  kubeletExtraArgs:
+    cloud-provider: external
+    node-labels: "usernetes/host-ip=${U7S_HOST_IP}"
 ---
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: ClusterConfiguration
 networking:
   serviceSubnet: "10.96.0.0/16"
   podSubnet: "10.244.0.0/16"
-controlPlaneEndpoint: "${U7S_HOST_IP}:6443"
+controlPlaneEndpoint: "${U7S_NODE_NAME}:6443"
+apiServer:
+  certSANs:
+    - localhost
+    - 127.0.0.1
+    - "${U7S_NODE_NAME}"
+    - "${U7S_HOST_IP}"
 ---
 kind: KubeletConfiguration
 apiVersion: kubelet.config.k8s.io/v1beta1