diff --git a/Dockerfile.d/u7s-entrypoint.sh b/Dockerfile.d/u7s-entrypoint.sh
index 537707c..4927745 100755
--- a/Dockerfile.d/u7s-entrypoint.sh
+++ b/Dockerfile.d/u7s-entrypoint.sh
@@ -1,20 +1,7 @@
#!/bin/bash
set -eux -o pipefail
-# Append "---node-ip=${U7S_HOST_IP}" to "KUBELET_EXTRA_ARGS=..." in /etc/default/kubelet
-sed -e "s/\(^KUBELET_EXTRA_ARGS=.*\)/\\1 --node-ip=${U7S_HOST_IP}/" /u7s-flanneld-wrapper.sh
-#!/bin/sh
-# Usage: /u7s-flanneld-wrapper.sh /opt/bin/flanneld --ip-masq --kube-subnet-mgr ...
-# This script is expected to be mounted inside a "docker.io/flannel/flannel" container.
-set -eux
-"\$@" --public-ip="${U7S_HOST_IP}"
-EOF
-chmod +x /u7s-flanneld-wrapper.sh
+# Append "KUBELET_EXTRA_ARGS=..." in /etc/default/kubelet
+sed -e "s!\(^KUBELET_EXTRA_ARGS=.*\)!\\1 --cloud-provider=external --node-labels=usernetes/host-ip=${U7S_HOST_IP}!" kubeconfig
+ $(COMPOSE) exec -T $(NODE_SERVICE_NAME) sed -e "s/$(NODE_NAME)/localhost/g" /etc/kubernetes/admin.conf >kubeconfig
@echo "# Run the following command by yourself:"
@echo "export KUBECONFIG=$(shell pwd)/kubeconfig"
ifeq ($(shell command -v kubectl 2> /dev/null),)
@@ -98,19 +99,34 @@ kubectl:
.PHONY: join-command
join-command:
- $(NODE_SHELL) kubeadm token create --print-join-command | tr -d '\r' >join-command
- @echo "# Copy the 'join-command' file to another host, and run 'make kubeadm-join' on that host (not on this host)"
+ echo "#!/bin/bash" >join-command
+ echo "set -eux -o pipefail" >>join-command
+ echo "grep -q -w $(NODE_NAME) /etc/hosts || echo \"$(HOST_IP) $(NODE_NAME)\" >>/etc/hosts" >>join-command
+ $(NODE_SHELL) kubeadm token create --print-join-command | tr -d '\r' >>join-command
+ chmod +x join-command
+ @echo "# Copy the 'join-command' file to another host, and run the following commands:"
+ @echo "# On the other host (the new worker):"
+ @echo "# make kubeadm-join"
+ @echo "# On this host (the control plane):"
+ @echo "# make sync-external-ip"
.PHONY: kubeadm-init
kubeadm-init:
$(NODE_SHELL) sh -euc "envsubst /tmp/kubeadm-config.yaml"
$(NODE_SHELL) kubeadm init --config /tmp/kubeadm-config.yaml --skip-token-print
+ $(MAKE) sync-external-ip
@echo "# Run 'make join-command' to print the join command"
+.PHONY: sync-external-ip
+sync-external-ip:
+ $(NODE_SHELL) /usernetes/Makefile.d/sync-external-ip.sh
+
.PHONY: kubeadm-join
kubeadm-join:
- $(NODE_SHELL) sh -euc '$$(cat /usernetes/join-command)'
+ $(NODE_SHELL) sh -euc "envsubst /tmp/kubeadm-config.yaml"
+ $(NODE_SHELL) /usernetes/join-command
+ @echo "# Run 'make sync-external-ip' on the control plane"
.PHONY: install-flannel
install-flannel:
- $(NODE_SHELL) kubectl apply -f /usernetes/manifests/kube-flannel.yml
+ $(NODE_SHELL) kubectl apply -f https://github.com/flannel-io/flannel/releases/download/v0.24.4/kube-flannel.yml
diff --git a/Makefile.d/sync-external-ip.sh b/Makefile.d/sync-external-ip.sh
new file mode 100755
index 0000000..2b4e8be
--- /dev/null
+++ b/Makefile.d/sync-external-ip.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+set -eu -o pipefail
+
+for node in $(kubectl get nodes -o name); do
+ # Set ExternalIP
+ host_ip="$(kubectl get "${node}" -o jsonpath='{.metadata.labels.usernetes/host-ip}')"
+ kubectl patch "${node}" --type=merge --subresource status --patch \
+ "\"status\": {\"addresses\": [{\"type\":\"ExternalIP\", \"address\": \"${host_ip}\"}]}"
+
+ # Propagate ExternalIP to flannel
+ # https://github.com/flannel-io/flannel/blob/v0.24.4/Documentation/kubernetes.md#annotations
+ kubectl annotate "${node}" flannel.alpha.coreos.com/public-ip-overwrite=${host_ip}
+
+ # Remove taints
+ taints="$(kubectl get "${node}" -o jsonpath='{.spec.taints}')"
+ if echo "${taints}" | grep -q node.cloudprovider.kubernetes.io/uninitialized; then
+ kubectl taint nodes "${node}" node.cloudprovider.kubernetes.io/uninitialized-
+ fi
+done
diff --git a/README.md b/README.md
index c18726e..cd585a2 100644
--- a/README.md
+++ b/README.md
@@ -101,6 +101,7 @@ kubectl get pods -A
make join-command
scp join-command another-host:~/usernetes
ssh another-host make -C ~/usernetes up kubeadm-join
+make sync-external-ip
# Debug
make logs
@@ -117,9 +118,6 @@ To change the container engine, set `export CONTAINER_ENGINE=podman` or `export
- Most of host files are not visible with `hostPath` mounts. Edit [`docker-compose.yaml`](./docker-compose.yaml) for mounting additional files.
- Some [volume drivers](https://kubernetes.io/docs/concepts/storage/volumes/) such as `nfs` do not work.
-
-
diff --git a/hack/create-cluster-lxd.sh b/hack/create-cluster-lxd.sh
index bc82d5f..17a009c 100755
--- a/hack/create-cluster-lxd.sh
+++ b/hack/create-cluster-lxd.sh
@@ -25,6 +25,7 @@ $SSH host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-in
# Let host1 join the cluster
$SCP host0:~/usernetes/join-command host1:~/usernetes/join-command
$SSH host1 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-join
+$SSH host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes sync-external-ip
# Enable kubectl
$SCP host0:~/usernetes/kubeconfig ./kubeconfig
diff --git a/kubeadm-config.yaml b/kubeadm-config.yaml
index dd2d9e2..0817b1a 100644
--- a/kubeadm-config.yaml
+++ b/kubeadm-config.yaml
@@ -1,16 +1,19 @@
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
-localAPIEndpoint:
- advertiseAddress: "${U7S_HOST_IP}"
- bindPort: 6443
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
networking:
serviceSubnet: "10.96.0.0/16"
podSubnet: "10.244.0.0/16"
-controlPlaneEndpoint: "${U7S_HOST_IP}:6443"
+controlPlaneEndpoint: "${U7S_NODE_NAME}:6443"
+apiServer:
+ certSANs:
+ - localhost
+ - 127.0.0.1
+ - "${U7S_NODE_NAME}"
+ - "${U7S_HOST_IP}"
---
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
diff --git a/manifests/kube-flannel.yml b/manifests/kube-flannel.yml
deleted file mode 100644
index a349311..0000000
--- a/manifests/kube-flannel.yml
+++ /dev/null
@@ -1,236 +0,0 @@
-#
-# Forked from https://github.com/flannel-io/flannel/releases/download/v0.24.2/kube-flannel.yml ,
-# to specify a custom `--public-ip` value via `/u7s-flanneld-wrapper.sh.
-#
-apiVersion: v1
-kind: Namespace
-metadata:
- labels:
- k8s-app: flannel
- pod-security.kubernetes.io/enforce: privileged
- name: kube-flannel
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- labels:
- k8s-app: flannel
- name: flannel
- namespace: kube-flannel
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- labels:
- k8s-app: flannel
- name: flannel
-rules:
-- apiGroups:
- - ""
- resources:
- - pods
- verbs:
- - get
-- apiGroups:
- - ""
- resources:
- - nodes
- verbs:
- - get
- - list
- - watch
-- apiGroups:
- - ""
- resources:
- - nodes/status
- verbs:
- - patch
-- apiGroups:
- - networking.k8s.io
- resources:
- - clustercidrs
- verbs:
- - list
- - watch
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
- labels:
- k8s-app: flannel
- name: flannel
-roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: flannel
-subjects:
-- kind: ServiceAccount
- name: flannel
- namespace: kube-flannel
----
-apiVersion: v1
-data:
- cni-conf.json: |
- {
- "name": "cbr0",
- "cniVersion": "0.3.1",
- "plugins": [
- {
- "type": "flannel",
- "delegate": {
- "hairpinMode": true,
- "isDefaultGateway": true
- }
- },
- {
- "type": "portmap",
- "capabilities": {
- "portMappings": true
- }
- }
- ]
- }
- net-conf.json: |
- {
- "Network": "10.244.0.0/16",
- "Backend": {
- "Type": "vxlan"
- }
- }
-kind: ConfigMap
-metadata:
- labels:
- app: flannel
- k8s-app: flannel
- tier: node
- name: kube-flannel-cfg
- namespace: kube-flannel
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
- labels:
- app: flannel
- k8s-app: flannel
- tier: node
- name: kube-flannel-ds
- namespace: kube-flannel
-spec:
- selector:
- matchLabels:
- app: flannel
- k8s-app: flannel
- template:
- metadata:
- labels:
- app: flannel
- k8s-app: flannel
- tier: node
- spec:
- affinity:
- nodeAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- nodeSelectorTerms:
- - matchExpressions:
- - key: kubernetes.io/os
- operator: In
- values:
- - linux
- containers:
- - args:
-#
- - /opt/bin/flanneld
-#
- - --ip-masq
- - --kube-subnet-mgr
- command:
-#
- - /u7s-flanneld-wrapper.sh
-#
- env:
- - name: POD_NAME
- valueFrom:
- fieldRef:
- fieldPath: metadata.name
- - name: POD_NAMESPACE
- valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
- - name: EVENT_QUEUE_DEPTH
- value: "5000"
- image: docker.io/flannel/flannel:v0.24.2
- name: kube-flannel
- resources:
- requests:
- cpu: 100m
- memory: 50Mi
- securityContext:
- capabilities:
- add:
- - NET_ADMIN
- - NET_RAW
- privileged: false
- volumeMounts:
- - mountPath: /run/flannel
- name: run
- - mountPath: /etc/kube-flannel/
- name: flannel-cfg
- - mountPath: /run/xtables.lock
- name: xtables-lock
-#
- - mountPath: /u7s-flanneld-wrapper.sh
- name: u7s-flanneld-wrapper
-#
- hostNetwork: true
- initContainers:
- - args:
- - -f
- - /flannel
- - /opt/cni/bin/flannel
- command:
- - cp
- image: docker.io/flannel/flannel-cni-plugin:v1.4.0-flannel1
- name: install-cni-plugin
- volumeMounts:
- - mountPath: /opt/cni/bin
- name: cni-plugin
- - args:
- - -f
- - /etc/kube-flannel/cni-conf.json
- - /etc/cni/net.d/10-flannel.conflist
- command:
- - cp
- image: docker.io/flannel/flannel:v0.24.2
- name: install-cni
- volumeMounts:
- - mountPath: /etc/cni/net.d
- name: cni
- - mountPath: /etc/kube-flannel/
- name: flannel-cfg
- priorityClassName: system-node-critical
- serviceAccountName: flannel
- tolerations:
- - effect: NoSchedule
- operator: Exists
- volumes:
- - hostPath:
- path: /run/flannel
- name: run
- - hostPath:
- path: /opt/cni/bin
- name: cni-plugin
- - hostPath:
- path: /etc/cni/net.d
- name: cni
- - configMap:
- name: kube-flannel-cfg
- name: flannel-cfg
- - hostPath:
- path: /run/xtables.lock
- type: FileOrCreate
- name: xtables-lock
-#
- - hostPath:
- path: /u7s-flanneld-wrapper.sh
- name: u7s-flanneld-wrapper
-#