diff --git a/Dockerfile b/Dockerfile
index c97356e..c255d86 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,11 +1,15 @@
-ARG BASE_IMAGE=docker.io/kindest/node:v1.29.1@sha256:a0cc28af37cf39b019e2b448c54d1a3f789de32536cb5a5db61a49623e527144
-ARG CNI_PLUGINS_VERSION=v1.4.0
+ARG BASE_IMAGE=docker.io/kindest/node:v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245
+ARG CNI_PLUGINS_VERSION=v1.4.1
FROM ${BASE_IMAGE}
-# TODO: check SHA256SUMS of cni-plugins
+COPY Dockerfile.d/SHA256SUMS.d/ /tmp/SHA256SUMS.d
ARG CNI_PLUGINS_VERSION
RUN arch="$(uname -m | sed -e s/x86_64/amd64/ -e s/aarch64/arm64/)" && \
- curl -fsSL https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${arch}-${CNI_PLUGINS_VERSION}.tgz \
- | tar Cxzv /opt/cni/bin
+ fname="cni-plugins-linux-${arch}-${CNI_PLUGINS_VERSION}.tgz" && \
+ curl -o "${fname}" -fSL "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/${fname}" && \
+ grep "${fname}" "/tmp/SHA256SUMS.d/cni-plugins-${CNI_PLUGINS_VERSION}" | sha256sum -c && \
+ mkdir -p /opt/cni/bin && \
+ tar xzf "${fname}" -C /opt/cni/bin && \
+ rm -f "${fname}"
# gettext-base: for `envsubst`
# moreutils: for `sponge`
# socat: for `socat` (to silence "[WARNING FileExisting-socat]" from kubeadm)
@@ -13,5 +17,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
gettext-base \
moreutils \
socat
+ADD Dockerfile.d/etc_udev_rules.d_90-flannel.rules /etc/udev/rules.d/90-flannel.rules
ADD Dockerfile.d/u7s-entrypoint.sh /
ENTRYPOINT ["/u7s-entrypoint.sh", "/usr/local/bin/entrypoint", "/sbin/init"]
diff --git a/Dockerfile.d/SHA256SUMS.d/cni-plugins-v1.4.1 b/Dockerfile.d/SHA256SUMS.d/cni-plugins-v1.4.1
new file mode 100644
index 0000000..0baa626
--- /dev/null
+++ b/Dockerfile.d/SHA256SUMS.d/cni-plugins-v1.4.1
@@ -0,0 +1,2 @@
+1511f6c003ace805eafeb1132727791326283cff88a923d76329e1892bba7a10 cni-plugins-linux-amd64-v1.4.1.tgz
+72644e13557cda8a5b39baf97fc5e93d23fdf7baba7700000e7e9efd8bdf9234 cni-plugins-linux-arm64-v1.4.1.tgz
diff --git a/Dockerfile.d/etc_udev_rules.d_90-flannel.rules b/Dockerfile.d/etc_udev_rules.d_90-flannel.rules
new file mode 100644
index 0000000..f1478aa
--- /dev/null
+++ b/Dockerfile.d/etc_udev_rules.d_90-flannel.rules
@@ -0,0 +1,5 @@
+# Correct UDP checksums for VXLAN behind NAT
+# https://github.com/flannel-io/flannel/issues/1279
+# https://github.com/kubernetes/kops/pull/9074
+# https://github.com/karmab/kcli/commit/b1a8eff658d17cf4e28162f0fa2c8b2b10e5ad00
+SUBSYSTEM=="net", ACTION=="add|change|move", ENV{INTERFACE}=="flannel.1", RUN+="/usr/sbin/ethtool -K flannel.1 tx-checksum-ip-generic off"
diff --git a/Dockerfile.d/u7s-entrypoint.sh b/Dockerfile.d/u7s-entrypoint.sh
index 537707c..36072a8 100755
--- a/Dockerfile.d/u7s-entrypoint.sh
+++ b/Dockerfile.d/u7s-entrypoint.sh
@@ -1,20 +1,10 @@
#!/bin/bash
set -eux -o pipefail
-# Append "---node-ip=${U7S_HOST_IP}" to "KUBELET_EXTRA_ARGS=..." in /etc/default/kubelet
-sed -e "s/\(^KUBELET_EXTRA_ARGS=.*\)/\\1 --node-ip=${U7S_HOST_IP}/" /u7s-flanneld-wrapper.sh
-#!/bin/sh
-# Usage: /u7s-flanneld-wrapper.sh /opt/bin/flanneld --ip-masq --kube-subnet-mgr ...
-# This script is expected to be mounted inside a "docker.io/flannel/flannel" container.
-set -eux
-"\$@" --public-ip="${U7S_HOST_IP}"
-EOF
-chmod +x /u7s-flanneld-wrapper.sh
+# Import control plane hosts from previous boot
+[ -e /etc/hosts.u7s ] && cat /etc/hosts.u7s >>/etc/hosts
exec "$@"
diff --git a/Makefile b/Makefile
index d458f09..62339a6 100644
--- a/Makefile
+++ b/Makefile
@@ -7,29 +7,33 @@ export HOSTNAME := $(HOSTNAME)
HOST_IP ?= $(shell ip --json route get 1 | jq -r .[0].prefsrc)
NODE_NAME ?= u7s-$(HOSTNAME)
-NODE_SUBNET ?= $(shell $(CURDIR)/Makefile.d/node_subnet.sh)
+NODE_SUBNET ?= $(shell $(CURDIR)/Makefile.d/node-subnet.sh)
# U7S_HOST_IP is the IP address of the physical host. Accessible from other hosts.
export U7S_HOST_IP := $(HOST_IP)
-# U7S_NODE_NAME is the IP address of the Kubernetes node running in Rootless Docker.
+# U7S_NODE_NAME is the host name of the Kubernetes node running in Rootless Docker.
# Not accessible from other hosts.
export U7S_NODE_NAME:= $(NODE_NAME)
# U7S_NODE_NAME is the subnet of the Kubernetes node running in Rootless Docker.
# Not accessible from other hosts.
export U7S_NODE_SUBNET := $(NODE_SUBNET)
+# U7S_NODE_IP is the IP address of the Kubernetes node running in Rootless Docker.
+# Not accessible from other hosts.
+export U7S_NODE_IP := $(subst .0/24,.100,$(U7S_NODE_SUBNET))
-CONTAINER_ENGINE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh CONTAINER_ENGINE)
+CONTAINER_ENGINE ?= $(shell $(CURDIR)/Makefile.d/detect-container-engine.sh CONTAINER_ENGINE)
export CONTAINER_ENGINE := $(CONTAINER_ENGINE)
-CONTAINER_ENGINE_TYPE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh CONTAINER_ENGINE_TYPE)
+CONTAINER_ENGINE_TYPE ?= $(shell $(CURDIR)/Makefile.d/detect-container-engine.sh CONTAINER_ENGINE_TYPE)
export CONTAINER_ENGINE_TYPE := $(CONTAINER_ENGINE_TYPE)
-COMPOSE ?= $(shell $(CURDIR)/Makefile.d/detect_container_engine.sh COMPOSE)
+COMPOSE ?= $(shell $(CURDIR)/Makefile.d/detect-container-engine.sh COMPOSE)
NODE_SERVICE_NAME := node
NODE_SHELL := $(COMPOSE) exec \
-e U7S_HOST_IP=$(U7S_HOST_IP) \
-e U7S_NODE_NAME=$(U7S_NODE_NAME) \
-e U7S_NODE_SUBNET=$(U7S_NODE_SUBNET) \
+ -e U7S_NODE_IP=$(U7S_NODE_IP) \
$(NODE_SERVICE_NAME)
.PHONY: help
@@ -48,6 +52,7 @@ help:
@echo 'make join-command'
@echo 'scp join-command another-host:~/usernetes'
@echo 'ssh another-host make -C ~/usernetes up kubeadm-join'
+ @echo 'make sync-external-ip'
@echo
@echo '# Debug'
@echo 'make logs'
@@ -81,7 +86,7 @@ logs:
.PHONY: kubeconfig
kubeconfig:
- $(COMPOSE) exec -T $(NODE_SERVICE_NAME) cat /etc/kubernetes/admin.conf >kubeconfig
+ $(COMPOSE) exec -T $(NODE_SERVICE_NAME) sed -e "s/$(NODE_NAME)/127.0.0.1/g" /etc/kubernetes/admin.conf >kubeconfig
@echo "# Run the following command by yourself:"
@echo "export KUBECONFIG=$(shell pwd)/kubeconfig"
ifeq ($(shell command -v kubectl 2> /dev/null),)
@@ -98,19 +103,35 @@ kubectl:
.PHONY: join-command
join-command:
- $(NODE_SHELL) kubeadm token create --print-join-command | tr -d '\r' >join-command
- @echo "# Copy the 'join-command' file to another host, and run 'make kubeadm-join' on that host (not on this host)"
+ echo "#!/bin/bash" >join-command
+ echo "set -eux -o pipefail" >>join-command
+ echo "echo \"$(HOST_IP) $(NODE_NAME)\" >/etc/hosts.u7s" >>join-command
+ echo "cat /etc/hosts.u7s >>/etc/hosts" >>join-command
+ $(NODE_SHELL) kubeadm token create --print-join-command | tr -d '\r' >>join-command
+ chmod +x join-command
+ @echo "# Copy the 'join-command' file to another host, and run the following commands:"
+ @echo "# On the other host (the new worker):"
+ @echo "# make kubeadm-join"
+ @echo "# On this host (the control plane):"
+ @echo "# make sync-external-ip"
.PHONY: kubeadm-init
kubeadm-init:
$(NODE_SHELL) sh -euc "envsubst /tmp/kubeadm-config.yaml"
$(NODE_SHELL) kubeadm init --config /tmp/kubeadm-config.yaml --skip-token-print
+ $(MAKE) sync-external-ip
@echo "# Run 'make join-command' to print the join command"
+.PHONY: sync-external-ip
+sync-external-ip:
+ $(NODE_SHELL) /usernetes/Makefile.d/sync-external-ip.sh
+
.PHONY: kubeadm-join
kubeadm-join:
- $(NODE_SHELL) sh -euc '$$(cat /usernetes/join-command)'
+ $(NODE_SHELL) sh -euc "envsubst /tmp/kubeadm-config.yaml"
+ $(NODE_SHELL) /usernetes/join-command
+ @echo "# Run 'make sync-external-ip' on the control plane"
.PHONY: install-flannel
install-flannel:
- $(NODE_SHELL) kubectl apply -f /usernetes/manifests/kube-flannel.yml
+ $(NODE_SHELL) kubectl apply -f https://github.com/flannel-io/flannel/releases/download/v0.24.4/kube-flannel.yml
diff --git a/Makefile.d/check-preflight.sh b/Makefile.d/check-preflight.sh
index 065307f..623e9f5 100755
--- a/Makefile.d/check-preflight.sh
+++ b/Makefile.d/check-preflight.sh
@@ -13,7 +13,7 @@ function ERROR() {
}
script_dir="$(dirname "$0")"
-detect_engine="${script_dir}"/detect_container_engine.sh
+detect_engine="${script_dir}"/detect-container-engine.sh
: "${CONTAINER_ENGINE:=$("${detect_engine}" CONTAINER_ENGINE)}"
: "${CONTAINER_ENGINE_TYPE:=$("${detect_engine}" CONTAINER_ENGINE_TYPE)}"
: "${QUICK:=0}"
diff --git a/Makefile.d/detect_container_engine.sh b/Makefile.d/detect-container-engine.sh
similarity index 100%
rename from Makefile.d/detect_container_engine.sh
rename to Makefile.d/detect-container-engine.sh
diff --git a/Makefile.d/node_subnet.sh b/Makefile.d/node-subnet.sh
similarity index 100%
rename from Makefile.d/node_subnet.sh
rename to Makefile.d/node-subnet.sh
diff --git a/Makefile.d/sync-external-ip.sh b/Makefile.d/sync-external-ip.sh
new file mode 100755
index 0000000..2b4e8be
--- /dev/null
+++ b/Makefile.d/sync-external-ip.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+set -eu -o pipefail
+
+for node in $(kubectl get nodes -o name); do
+ # Set ExternalIP
+ host_ip="$(kubectl get "${node}" -o jsonpath='{.metadata.labels.usernetes/host-ip}')"
+ kubectl patch "${node}" --type=merge --subresource status --patch \
+ "\"status\": {\"addresses\": [{\"type\":\"ExternalIP\", \"address\": \"${host_ip}\"}]}"
+
+ # Propagate ExternalIP to flannel
+ # https://github.com/flannel-io/flannel/blob/v0.24.4/Documentation/kubernetes.md#annotations
+ kubectl annotate "${node}" flannel.alpha.coreos.com/public-ip-overwrite=${host_ip}
+
+ # Remove taints
+ taints="$(kubectl get "${node}" -o jsonpath='{.spec.taints}')"
+ if echo "${taints}" | grep -q node.cloudprovider.kubernetes.io/uninitialized; then
+ kubectl taint nodes "${node}" node.cloudprovider.kubernetes.io/uninitialized-
+ fi
+done
diff --git a/README.md b/README.md
index c18726e..cd585a2 100644
--- a/README.md
+++ b/README.md
@@ -101,6 +101,7 @@ kubectl get pods -A
make join-command
scp join-command another-host:~/usernetes
ssh another-host make -C ~/usernetes up kubeadm-join
+make sync-external-ip
# Debug
make logs
@@ -117,9 +118,6 @@ To change the container engine, set `export CONTAINER_ENGINE=podman` or `export
- Most of host files are not visible with `hostPath` mounts. Edit [`docker-compose.yaml`](./docker-compose.yaml) for mounting additional files.
- Some [volume drivers](https://kubernetes.io/docs/concepts/storage/volumes/) such as `nfs` do not work.
-
-
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 2152813..2ae7291 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -7,6 +7,9 @@ services:
hostname: ${U7S_NODE_NAME}
privileged: true
restart: always
+ networks:
+ default:
+ ipv4_address: ${U7S_NODE_IP}
ports:
# etcd
- 2379:2379
diff --git a/hack/create-cluster-lxd.sh b/hack/create-cluster-lxd.sh
index bc82d5f..e4a17b7 100755
--- a/hack/create-cluster-lxd.sh
+++ b/hack/create-cluster-lxd.sh
@@ -25,9 +25,11 @@ $SSH host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-in
# Let host1 join the cluster
$SCP host0:~/usernetes/join-command host1:~/usernetes/join-command
$SSH host1 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-join
+$SSH host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes sync-external-ip
# Enable kubectl
$SCP host0:~/usernetes/kubeconfig ./kubeconfig
+sed -i -e "s/127.0.0.1/$($SSH host0 ip --json route get 1 | jq -r .[0].prefsrc)/g" ./kubeconfig
KUBECONFIG="$(pwd)/kubeconfig"
export KUBECONFIG
kubectl get nodes -o wide
diff --git a/init-host/init-host.root.sh b/init-host/init-host.root.sh
index a9b762f..5151eb1 100755
--- a/init-host/init-host.root.sh
+++ b/init-host/init-host.root.sh
@@ -40,7 +40,7 @@ if command -v dnf >/dev/null 2>&1; then
# so it has to be reinstalled
dnf reinstall -y shadow-utils
else
- apt-get update
+ apt-get update
apt-get install -y git uidmap make jq
fi
diff --git a/kubeadm-config.yaml b/kubeadm-config.yaml
index dd2d9e2..cc09f9a 100644
--- a/kubeadm-config.yaml
+++ b/kubeadm-config.yaml
@@ -1,16 +1,27 @@
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
-localAPIEndpoint:
- advertiseAddress: "${U7S_HOST_IP}"
- bindPort: 6443
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
networking:
serviceSubnet: "10.96.0.0/16"
podSubnet: "10.244.0.0/16"
-controlPlaneEndpoint: "${U7S_HOST_IP}:6443"
+controlPlaneEndpoint: "${U7S_NODE_NAME}:6443"
+apiServer:
+ extraArgs:
+ advertise-address: "${U7S_HOST_IP}"
+ cloud-provider: external
+ # Default: "Hostname,InternalDNS,InternalIP,ExternalDNS,ExternalIP"
+ kubelet-preferred-address-types: "ExternalIP"
+ certSANs:
+ - localhost
+ - 127.0.0.1
+ - "${U7S_NODE_NAME}"
+ - "${U7S_HOST_IP}"
+controllerManager:
+ extraArgs:
+ cloud-provider: external
---
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
diff --git a/manifests/kube-flannel.yml b/manifests/kube-flannel.yml
deleted file mode 100644
index a349311..0000000
--- a/manifests/kube-flannel.yml
+++ /dev/null
@@ -1,236 +0,0 @@
-#
-# Forked from https://github.com/flannel-io/flannel/releases/download/v0.24.2/kube-flannel.yml ,
-# to specify a custom `--public-ip` value via `/u7s-flanneld-wrapper.sh.
-#
-apiVersion: v1
-kind: Namespace
-metadata:
- labels:
- k8s-app: flannel
- pod-security.kubernetes.io/enforce: privileged
- name: kube-flannel
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- labels:
- k8s-app: flannel
- name: flannel
- namespace: kube-flannel
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- labels:
- k8s-app: flannel
- name: flannel
-rules:
-- apiGroups:
- - ""
- resources:
- - pods
- verbs:
- - get
-- apiGroups:
- - ""
- resources:
- - nodes
- verbs:
- - get
- - list
- - watch
-- apiGroups:
- - ""
- resources:
- - nodes/status
- verbs:
- - patch
-- apiGroups:
- - networking.k8s.io
- resources:
- - clustercidrs
- verbs:
- - list
- - watch
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
- labels:
- k8s-app: flannel
- name: flannel
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: flannel
-subjects:
-- kind: ServiceAccount
- name: flannel
- namespace: kube-flannel
----
-apiVersion: v1
-data:
- cni-conf.json: |
- {
- "name": "cbr0",
- "cniVersion": "0.3.1",
- "plugins": [
- {
- "type": "flannel",
- "delegate": {
- "hairpinMode": true,
- "isDefaultGateway": true
- }
- },
- {
- "type": "portmap",
- "capabilities": {
- "portMappings": true
- }
- }
- ]
- }
- net-conf.json: |
- {
- "Network": "10.244.0.0/16",
- "Backend": {
- "Type": "vxlan"
- }
- }
-kind: ConfigMap
-metadata:
- labels:
- app: flannel
- k8s-app: flannel
- tier: node
- name: kube-flannel-cfg
- namespace: kube-flannel
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
- labels:
- app: flannel
- k8s-app: flannel
- tier: node
- name: kube-flannel-ds
- namespace: kube-flannel
-spec:
- selector:
- matchLabels:
- app: flannel
- k8s-app: flannel
- template:
- metadata:
- labels:
- app: flannel
- k8s-app: flannel
- tier: node
- spec:
- affinity:
- nodeAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- nodeSelectorTerms:
- - matchExpressions:
- - key: kubernetes.io/os
- operator: In
- values:
- - linux
- containers:
- - args:
-#
- - /opt/bin/flanneld
-#
- - --ip-masq
- - --kube-subnet-mgr
- command:
-#
- - /u7s-flanneld-wrapper.sh
-#
- env:
- - name: POD_NAME
- valueFrom:
- fieldRef:
- fieldPath: metadata.name
- - name: POD_NAMESPACE
- valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
- - name: EVENT_QUEUE_DEPTH
- value: "5000"
- image: docker.io/flannel/flannel:v0.24.2
- name: kube-flannel
- resources:
- requests:
- cpu: 100m
- memory: 50Mi
- securityContext:
- capabilities:
- add:
- - NET_ADMIN
- - NET_RAW
- privileged: false
- volumeMounts:
- - mountPath: /run/flannel
- name: run
- - mountPath: /etc/kube-flannel/
- name: flannel-cfg
- - mountPath: /run/xtables.lock
- name: xtables-lock
-#
- - mountPath: /u7s-flanneld-wrapper.sh
- name: u7s-flanneld-wrapper
-#
- hostNetwork: true
- initContainers:
- - args:
- - -f
- - /flannel
- - /opt/cni/bin/flannel
- command:
- - cp
- image: docker.io/flannel/flannel-cni-plugin:v1.4.0-flannel1
- name: install-cni-plugin
- volumeMounts:
- - mountPath: /opt/cni/bin
- name: cni-plugin
- - args:
- - -f
- - /etc/kube-flannel/cni-conf.json
- - /etc/cni/net.d/10-flannel.conflist
- command:
- - cp
- image: docker.io/flannel/flannel:v0.24.2
- name: install-cni
- volumeMounts:
- - mountPath: /etc/cni/net.d
- name: cni
- - mountPath: /etc/kube-flannel/
- name: flannel-cfg
- priorityClassName: system-node-critical
- serviceAccountName: flannel
- tolerations:
- - effect: NoSchedule
- operator: Exists
- volumes:
- - hostPath:
- path: /run/flannel
- name: run
- - hostPath:
- path: /opt/cni/bin
- name: cni-plugin
- - hostPath:
- path: /etc/cni/net.d
- name: cni
- - configMap:
- name: kube-flannel-cfg
- name: flannel-cfg
- - hostPath:
- path: /run/xtables.lock
- type: FileOrCreate
- name: xtables-lock
-#
- - hostPath:
- path: /u7s-flanneld-wrapper.sh
- name: u7s-flanneld-wrapper
-#