From 897a2a404443b6c07d3ab3970f7f9b527384c44b Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Sun, 31 Mar 2024 22:54:12 +0900 Subject: [PATCH] Remove `ip addr add "${U7S_HOST_IP}" dev eth0` Fix issue 296 Also fix issue 293, 297 Signed-off-by: Akihiro Suda --- Dockerfile | 1 + .../etc_udev_rules.d_90-flannel.rules | 5 + Dockerfile.d/u7s-entrypoint.sh | 18 +- Makefile | 27 +- Makefile.d/sync-external-ip.sh | 19 ++ README.md | 4 +- hack/create-cluster-lxd.sh | 2 + kubeadm-config.yaml | 19 +- manifests/kube-flannel.yml | 236 ------------------ 9 files changed, 69 insertions(+), 262 deletions(-) create mode 100644 Dockerfile.d/etc_udev_rules.d_90-flannel.rules create mode 100755 Makefile.d/sync-external-ip.sh delete mode 100644 manifests/kube-flannel.yml diff --git a/Dockerfile b/Dockerfile index c97356e..82c23ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,5 +13,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gettext-base \ moreutils \ socat +ADD Dockerfile.d/etc_udev_rules.d_90-flannel.rules /etc/udev/rules.d/90-flannel.rules ADD Dockerfile.d/u7s-entrypoint.sh / ENTRYPOINT ["/u7s-entrypoint.sh", "/usr/local/bin/entrypoint", "/sbin/init"] diff --git a/Dockerfile.d/etc_udev_rules.d_90-flannel.rules b/Dockerfile.d/etc_udev_rules.d_90-flannel.rules new file mode 100644 index 0000000..f1478aa --- /dev/null +++ b/Dockerfile.d/etc_udev_rules.d_90-flannel.rules @@ -0,0 +1,5 @@ +# Correct UDP checksums for VXLAN behind NAT +# https://github.com/flannel-io/flannel/issues/1279 +# https://github.com/kubernetes/kops/pull/9074 +# https://github.com/karmab/kcli/commit/b1a8eff658d17cf4e28162f0fa2c8b2b10e5ad00 +SUBSYSTEM=="net", ACTION=="add|change|move", ENV{INTERFACE}=="flannel.1", RUN+="/usr/sbin/ethtool -K flannel.1 tx-checksum-ip-generic off" diff --git a/Dockerfile.d/u7s-entrypoint.sh b/Dockerfile.d/u7s-entrypoint.sh index 537707c..36072a8 100755 --- a/Dockerfile.d/u7s-entrypoint.sh +++ b/Dockerfile.d/u7s-entrypoint.sh @@ -1,20 +1,10 @@ #!/bin/bash set -eux -o pipefail -# Append "---node-ip=${U7S_HOST_IP}" to "KUBELET_EXTRA_ARGS=..." in /etc/default/kubelet -sed -e "s/\(^KUBELET_EXTRA_ARGS=.*\)/\\1 --node-ip=${U7S_HOST_IP}/" /u7s-flanneld-wrapper.sh -#!/bin/sh -# Usage: /u7s-flanneld-wrapper.sh /opt/bin/flanneld --ip-masq --kube-subnet-mgr ... -# This script is expected to be mounted inside a "docker.io/flannel/flannel" container. -set -eux -"\$@" --public-ip="${U7S_HOST_IP}" -EOF -chmod +x /u7s-flanneld-wrapper.sh +# Import control plane hosts from previous boot +[ -e /etc/hosts.u7s ] && cat /etc/hosts.u7s >>/etc/hosts exec "$@" diff --git a/Makefile b/Makefile index d458f09..845826a 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,7 @@ help: @echo 'make join-command' @echo 'scp join-command another-host:~/usernetes' @echo 'ssh another-host make -C ~/usernetes up kubeadm-join' + @echo 'make sync-external-ip' @echo @echo '# Debug' @echo 'make logs' @@ -81,7 +82,7 @@ logs: .PHONY: kubeconfig kubeconfig: - $(COMPOSE) exec -T $(NODE_SERVICE_NAME) cat /etc/kubernetes/admin.conf >kubeconfig + $(COMPOSE) exec -T $(NODE_SERVICE_NAME) sed -e "s/$(NODE_NAME)/127.0.0.1/g" /etc/kubernetes/admin.conf >kubeconfig @echo "# Run the following command by yourself:" @echo "export KUBECONFIG=$(shell pwd)/kubeconfig" ifeq ($(shell command -v kubectl 2> /dev/null),) @@ -98,19 +99,35 @@ kubectl: .PHONY: join-command join-command: - $(NODE_SHELL) kubeadm token create --print-join-command | tr -d '\r' >join-command - @echo "# Copy the 'join-command' file to another host, and run 'make kubeadm-join' on that host (not on this host)" + echo "#!/bin/bash" >join-command + echo "set -eux -o pipefail" >>join-command + echo "echo \"$(HOST_IP) $(NODE_NAME)\" >/etc/hosts.u7s" >>join-command + echo "cat /etc/hosts.u7s >>/etc/hosts" >>join-command + $(NODE_SHELL) kubeadm token create --print-join-command | tr -d '\r' >>join-command + chmod +x join-command + @echo "# Copy the 'join-command' file to another host, and run the following commands:" + @echo "# On the other host (the new worker):" + @echo "# make kubeadm-join" + @echo "# On this host (the control plane):" + @echo "# make sync-external-ip" .PHONY: kubeadm-init kubeadm-init: $(NODE_SHELL) sh -euc "envsubst /tmp/kubeadm-config.yaml" $(NODE_SHELL) kubeadm init --config /tmp/kubeadm-config.yaml --skip-token-print + $(MAKE) sync-external-ip @echo "# Run 'make join-command' to print the join command" +.PHONY: sync-external-ip +sync-external-ip: + $(NODE_SHELL) /usernetes/Makefile.d/sync-external-ip.sh + .PHONY: kubeadm-join kubeadm-join: - $(NODE_SHELL) sh -euc '$$(cat /usernetes/join-command)' + $(NODE_SHELL) sh -euc "envsubst /tmp/kubeadm-config.yaml" + $(NODE_SHELL) /usernetes/join-command + @echo "# Run 'make sync-external-ip' on the control plane" .PHONY: install-flannel install-flannel: - $(NODE_SHELL) kubectl apply -f /usernetes/manifests/kube-flannel.yml + $(NODE_SHELL) kubectl apply -f https://github.com/flannel-io/flannel/releases/download/v0.24.4/kube-flannel.yml diff --git a/Makefile.d/sync-external-ip.sh b/Makefile.d/sync-external-ip.sh new file mode 100755 index 0000000..2b4e8be --- /dev/null +++ b/Makefile.d/sync-external-ip.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -eu -o pipefail + +for node in $(kubectl get nodes -o name); do + # Set ExternalIP + host_ip="$(kubectl get "${node}" -o jsonpath='{.metadata.labels.usernetes/host-ip}')" + kubectl patch "${node}" --type=merge --subresource status --patch \ + "\"status\": {\"addresses\": [{\"type\":\"ExternalIP\", \"address\": \"${host_ip}\"}]}" + + # Propagate ExternalIP to flannel + # https://github.com/flannel-io/flannel/blob/v0.24.4/Documentation/kubernetes.md#annotations + kubectl annotate "${node}" flannel.alpha.coreos.com/public-ip-overwrite=${host_ip} + + # Remove taints + taints="$(kubectl get "${node}" -o jsonpath='{.spec.taints}')" + if echo "${taints}" | grep -q node.cloudprovider.kubernetes.io/uninitialized; then + kubectl taint nodes "${node}" node.cloudprovider.kubernetes.io/uninitialized- + fi +done diff --git a/README.md b/README.md index c18726e..cd585a2 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ kubectl get pods -A make join-command scp join-command another-host:~/usernetes ssh another-host make -C ~/usernetes up kubeadm-join +make sync-external-ip # Debug make logs @@ -117,9 +118,6 @@ To change the container engine, set `export CONTAINER_ENGINE=podman` or `export - Most of host files are not visible with `hostPath` mounts. Edit [`docker-compose.yaml`](./docker-compose.yaml) for mounting additional files. - Some [volume drivers](https://kubernetes.io/docs/concepts/storage/volumes/) such as `nfs` do not work. - - diff --git a/hack/create-cluster-lxd.sh b/hack/create-cluster-lxd.sh index bc82d5f..ca0d0fe 100755 --- a/hack/create-cluster-lxd.sh +++ b/hack/create-cluster-lxd.sh @@ -25,9 +25,11 @@ $SSH host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-in # Let host1 join the cluster $SCP host0:~/usernetes/join-command host1:~/usernetes/join-command $SSH host1 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes kubeadm-join +$SSH host0 CONTAINER_ENGINE="${CONTAINER_ENGINE}" make -C ~/usernetes sync-external-ip # Enable kubectl $SCP host0:~/usernetes/kubeconfig ./kubeconfig +sed -i -e "s/127.0.0.1/$(ip --json route get 1 | jq -r .[0].prefsrc)/g" ./kubeconfig KUBECONFIG="$(pwd)/kubeconfig" export KUBECONFIG kubectl get nodes -o wide diff --git a/kubeadm-config.yaml b/kubeadm-config.yaml index dd2d9e2..cc09f9a 100644 --- a/kubeadm-config.yaml +++ b/kubeadm-config.yaml @@ -1,16 +1,27 @@ --- apiVersion: kubeadm.k8s.io/v1beta3 kind: InitConfiguration -localAPIEndpoint: - advertiseAddress: "${U7S_HOST_IP}" - bindPort: 6443 --- apiVersion: kubeadm.k8s.io/v1beta3 kind: ClusterConfiguration networking: serviceSubnet: "10.96.0.0/16" podSubnet: "10.244.0.0/16" -controlPlaneEndpoint: "${U7S_HOST_IP}:6443" +controlPlaneEndpoint: "${U7S_NODE_NAME}:6443" +apiServer: + extraArgs: + advertise-address: "${U7S_HOST_IP}" + cloud-provider: external + # Default: "Hostname,InternalDNS,InternalIP,ExternalDNS,ExternalIP" + kubelet-preferred-address-types: "ExternalIP" + certSANs: + - localhost + - 127.0.0.1 + - "${U7S_NODE_NAME}" + - "${U7S_HOST_IP}" +controllerManager: + extraArgs: + cloud-provider: external --- kind: KubeletConfiguration apiVersion: kubelet.config.k8s.io/v1beta1 diff --git a/manifests/kube-flannel.yml b/manifests/kube-flannel.yml deleted file mode 100644 index a349311..0000000 --- a/manifests/kube-flannel.yml +++ /dev/null @@ -1,236 +0,0 @@ -# -# Forked from https://github.com/flannel-io/flannel/releases/download/v0.24.2/kube-flannel.yml , -# to specify a custom `--public-ip` value via `/u7s-flanneld-wrapper.sh. -# -apiVersion: v1 -kind: Namespace -metadata: - labels: - k8s-app: flannel - pod-security.kubernetes.io/enforce: privileged - name: kube-flannel ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - k8s-app: flannel - name: flannel - namespace: kube-flannel ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - k8s-app: flannel - name: flannel -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - get -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch -- apiGroups: - - networking.k8s.io - resources: - - clustercidrs - verbs: - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - k8s-app: flannel - name: flannel -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flannel -subjects: -- kind: ServiceAccount - name: flannel - namespace: kube-flannel ---- -apiVersion: v1 -data: - cni-conf.json: | - { - "name": "cbr0", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "flannel", - "delegate": { - "hairpinMode": true, - "isDefaultGateway": true - } - }, - { - "type": "portmap", - "capabilities": { - "portMappings": true - } - } - ] - } - net-conf.json: | - { - "Network": "10.244.0.0/16", - "Backend": { - "Type": "vxlan" - } - } -kind: ConfigMap -metadata: - labels: - app: flannel - k8s-app: flannel - tier: node - name: kube-flannel-cfg - namespace: kube-flannel ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app: flannel - k8s-app: flannel - tier: node - name: kube-flannel-ds - namespace: kube-flannel -spec: - selector: - matchLabels: - app: flannel - k8s-app: flannel - template: - metadata: - labels: - app: flannel - k8s-app: flannel - tier: node - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - linux - containers: - - args: -# - - /opt/bin/flanneld -# - - --ip-masq - - --kube-subnet-mgr - command: -# - - /u7s-flanneld-wrapper.sh -# - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: EVENT_QUEUE_DEPTH - value: "5000" - image: docker.io/flannel/flannel:v0.24.2 - name: kube-flannel - resources: - requests: - cpu: 100m - memory: 50Mi - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - privileged: false - volumeMounts: - - mountPath: /run/flannel - name: run - - mountPath: /etc/kube-flannel/ - name: flannel-cfg - - mountPath: /run/xtables.lock - name: xtables-lock -# - - mountPath: /u7s-flanneld-wrapper.sh - name: u7s-flanneld-wrapper -# - hostNetwork: true - initContainers: - - args: - - -f - - /flannel - - /opt/cni/bin/flannel - command: - - cp - image: docker.io/flannel/flannel-cni-plugin:v1.4.0-flannel1 - name: install-cni-plugin - volumeMounts: - - mountPath: /opt/cni/bin - name: cni-plugin - - args: - - -f - - /etc/kube-flannel/cni-conf.json - - /etc/cni/net.d/10-flannel.conflist - command: - - cp - image: docker.io/flannel/flannel:v0.24.2 - name: install-cni - volumeMounts: - - mountPath: /etc/cni/net.d - name: cni - - mountPath: /etc/kube-flannel/ - name: flannel-cfg - priorityClassName: system-node-critical - serviceAccountName: flannel - tolerations: - - effect: NoSchedule - operator: Exists - volumes: - - hostPath: - path: /run/flannel - name: run - - hostPath: - path: /opt/cni/bin - name: cni-plugin - - hostPath: - path: /etc/cni/net.d - name: cni - - configMap: - name: kube-flannel-cfg - name: flannel-cfg - - hostPath: - path: /run/xtables.lock - type: FileOrCreate - name: xtables-lock -# - - hostPath: - path: /u7s-flanneld-wrapper.sh - name: u7s-flanneld-wrapper -#