From 337e2c53656ca830f8409cad3a9f75ea7f018d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B7=AE=E5=8F=B3?= Date: Tue, 27 Dec 2022 16:42:29 +0800 Subject: [PATCH] Feature/v1 20 11 (#167) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * rename clean.sh to uninstall-docker.sh: delete some clean kube logic (#144) * modify plugin file (#150) * modify plugin file * modify Kubefile * modify plugin and build.sh * modify plugin and build.sh * modify plugin * add Clusterfile;modify script,plugin * modify Kubefile,script,plugin * modify Kubefile,script,plugin * update plugins、scripts、imagesList * Fix plugins. Signed-off-by: huaiyou * Fix bins path. Signed-off-by: huaiyou * If another ip family ip not found, will not crash. Support configure RegistryIP dns resolve. Use hack sealer. HACK: add adp images. Signed-off-by: huaiyou * fix helm values; fix docker clean volumes. Signed-off-by: huaiyou * fix. Signed-off-by: huaiyou * Fix etcd ephemeral-storage patch; Configure registry domain; Rename kube-prometheus-stack to kube-prometheus-crds; Support use & to join multi storage device. Signed-off-by: huaiyou * Set registry url for nvidia pod. disable serializeImagePulls. Signed-off-by: huaiyou * Update imageList. Signed-off-by: huaiyou * Support process DeployMode. Signed-off-by: huaiyou * Support PlatformType Support install component in managedcluster. Signed-off-by: huaiyou * Hack label for infra; Split coredns host to another configmap; Support generate clusterinfo; Remove useless images. Signed-off-by: huaiyou * Split plugin to post scale up and post install; Fix set log rotate should run on all hosts; Support ingressInternalIP etc. Signed-off-by: huaiyou * Mark managed by ackd label. Signed-off-by: huaiyou * Dump ca cert into clusterinfo. Signed-off-by: huaiyou * Support set PlatformCA. Signed-off-by: huaiyou * Fix. Signed-off-by: huaiyou * Fix clusterinfo. Signed-off-by: huaiyou * Fix. Signed-off-by: huaiyou * Label all node with cnstack-infra if not enterprise mode. Enable IPv6DualStack by default. Signed-off-by: huaiyou * Use new oss. Signed-off-by: huaiyou * Fix cidr. Signed-off-by: huaiyou * Fix image. Signed-off-by: huaiyou * Update. Signed-off-by: huaiyou * Support vip for apiserver. Improve helm install retry. Signed-off-by: huaiyou * Update to 1.20.11; Fix depreciated api verison; Remove useless ingress; Ignore check for hybridnet subnet. Signed-off-by: huaiyou * Set metrics server replicas. Signed-off-by: huaiyou * Remove allocate node cidr. Signed-off-by: huaiyou * Fix. Signed-off-by: huaiyou * Support gatewayExposeMode Signed-off-by: huaiyou * Check mountpoint before disk init. Signed-off-by: huaiyou * Fix node port for apiserver lb. Signed-off-by: huaiyou * update storage class annotations (#166) * Support deployMode,gatewayDomain,registryExternalURL; Add push-to-registry.sh etcdctl_v3.sh Signed-off-by: huaiyou Signed-off-by: huaiyou Co-authored-by: kakaZhou719 <83740799+kakaZhou719@users.noreply.github.com> Co-authored-by: kakzhou719 Co-authored-by: Steven.Wang <65949637+Stevent-fei@users.noreply.github.com> Co-authored-by: Zhiheng Sun --- build/build.sh | 2 - build/etc/kubeadm.yml.tmpl | 4 ++ build/imageList | 1 - build/plugins/post_install_shell_plugin.yaml | 42 ++++++++++----- build/plugins/post_scaleup_shell_plugin.yaml | 4 ++ build/plugins/pre_init_host_shell_plugin.yaml | 28 +++++++++- build/scripts/disk_init_v2.sh | 54 ++++++++++++------- build/scripts/etcdctl_v3.sh | 1 + build/scripts/push-to-registry.sh | 46 ++++++++++++++++ build/yamls/apiserver-lb-svc.yaml | 1 + docs/user-guide/requirements.md | 1 - docs/user-guide/requirements_zh.md | 1 - 12 files changed, 145 insertions(+), 40 deletions(-) create mode 100644 build/scripts/etcdctl_v3.sh create mode 100644 build/scripts/push-to-registry.sh diff --git a/build/build.sh b/build/build.sh index 986b885b..b4661d37 100755 --- a/build/build.sh +++ b/build/build.sh @@ -52,8 +52,6 @@ if [ "$SKIP_DOWNLOAD_BINS" != "true" ];then wget https://ack-a-aecp.oss-cn-hangzhou.aliyuncs.com/ack-distro/bin/${arch}/${bin} -O ${arch}/bin/${bin} done - wget "https://ack-a-aecp.oss-cn-hangzhou.aliyuncs.com/ack-distro/bin/${arch}/seautil-new" -O ${arch}/bin/seautil - wget "https://ack-a-aecp.oss-cn-hangzhou.aliyuncs.com/trident/release/trident_license_off-linux-${arch}_${trident_version}.bin" -O ${arch}/bin/trident if [ "$arch" == "amd64" ];then diff --git a/build/etc/kubeadm.yml.tmpl b/build/etc/kubeadm.yml.tmpl index 47a10388..5270248f 100644 --- a/build/etc/kubeadm.yml.tmpl +++ b/build/etc/kubeadm.yml.tmpl @@ -16,6 +16,9 @@ kind: ClusterConfiguration kubernetesVersion: v1.20.11-aliyun.1 controlPlaneEndpoint: "apiserver.cluster.local:6443" imageRepository: {{or .RegistryURL "sea.hub:5000"}}/ecp_builder +dns: + imageRepository: {{or .RegistryURL "sea.hub:5000"}}/acs + imageTag: v1.8.4.5-2ce07fd2-aliyun networking: dnsDomain: {{or .DNSDomain "cluster.local"}} {{ if .PodCIDR }} @@ -40,6 +43,7 @@ apiServer: - 10.96.0.1 - 10.103.97.2 - 1248:4003:10bb:6a01:83b9:6360:c66d:0002 + - cnstack.local extraArgs: feature-gates: TTLAfterFinished=true,EphemeralContainers=true,IPv6DualStack=true service-node-port-range: {{or .ServiceNodePortRange "30000-32767"}} diff --git a/build/imageList b/build/imageList index 23cfabe6..83202fbe 100644 --- a/build/imageList +++ b/build/imageList @@ -5,7 +5,6 @@ ack-agility-registry.cn-shanghai.cr.aliyuncs.com/ecp_builder/kube-scheduler:v1.2 ack-agility-registry.cn-shanghai.cr.aliyuncs.com/ecp_builder/etcd:3.5.4 ack-agility-registry.cn-shanghai.cr.aliyuncs.com/ecp_builder/metrics-server:v0.3.8.5-307cf45-aliyun ack-agility-registry.cn-shanghai.cr.aliyuncs.com/ecp_builder/kube-proxy:v1.20.11-aliyun.1 -ack-agility-registry.cn-shanghai.cr.aliyuncs.com/ecp_builder/coredns:1.7.0 ack-agility-registry.cn-shanghai.cr.aliyuncs.com/ecp_builder/l-zero:v1.1.0-6829307 ack-agility-registry.cn-shanghai.cr.aliyuncs.com/ecp_builder/httpd:alpine3.15 ack-agility-registry.cn-shanghai.cr.aliyuncs.com/ecp_builder/opsbase-alpine:v0.2 diff --git a/build/plugins/post_install_shell_plugin.yaml b/build/plugins/post_install_shell_plugin.yaml index 0e659214..ad8d2033 100644 --- a/build/plugins/post_install_shell_plugin.yaml +++ b/build/plugins/post_install_shell_plugin.yaml @@ -27,23 +27,24 @@ spec: export ComponentToInstall=${ComponentToInstall} export GenerateClusterInfo=${GenerateClusterInfo:-true} export ParalbHostInterface=${ParalbHostInterface} - export ingressAddress=${ingressAddress:-ingress.cnstack.local} - export ingressInternalIP=${ingressInternalIP:-${Master0IP}} - export ingressExternalIP=${ingressExternalIP:-${Master0IP}} - export ingressHttpPort=${ingressHttpPort:-80} - export ingressHttpsPort=${ingressHttpsPort:-443} - export scale=${scale:-small} - export harborAddress=${harborAddress:-harbor.cnstack.local} - export vcnsOssAddress=${vcnsOssAddress:-vcns-oss.cnstack.local} - export apiServerInternalIP=${apiServerInternalIP} - export apiServerInternalPort=${apiServerInternalPort} + export deployMode=${deployMode:-offline} + export gatewayDomain=${gatewayDomain:-cnstack.local} if [ "$DisableGateway" != "true" ];then - export gatewayAddress=${gatewayAddress:-cnstack.local} + export gatewayExposeMode=${gatewayExposeMode:-ip_domain} export gatewayInternalIP=${gatewayInternalIP:-${Master0IP}} export gatewayExternalIP=${gatewayExternalIP:-${Master0IP}} export gatewayPort=${gatewayPort:-30383} export gatewayAPIServerPort=${gatewayAPIServerPort:-30384} fi + export ingressAddress=${ingressAddress:-ingress.${gatewayDomain}} + export ingressInternalIP=${ingressInternalIP:-${Master0IP}} + export ingressExternalIP=${ingressExternalIP:-${Master0IP}} + export ingressHttpPort=${ingressHttpPort:-80} + export ingressHttpsPort=${ingressHttpsPort:-443} + export harborAddress=${harborAddress:-harbor.${gatewayDomain}} + export vcnsOssAddress=${vcnsOssAddress:-vcns-oss.${gatewayDomain}} + export apiServerInternalIP=${apiServerInternalIP} + export apiServerInternalPort=${apiServerInternalPort} export KUBECONFIG=/etc/kubernetes/admin.conf if [ "$HostIPFamily" = "6" ];then @@ -63,19 +64,34 @@ spec: kubectl label node node-role.kubernetes.io/cnstack-infra="" --all kubectl label node node-role.kubernetes.io/proxy="" --all fi + if [ "${deployMode}" == "online" ];then + gatewayExposeMode=ip + fi + + gatewayAddress=${gatewayDomain} + if [ "$gatewayExposeMode" == "ip" ];then + if [[ ${gatewayExternalIP} =~ ":" ]];then + gatewayAddress=[${gatewayExternalIP}] + else + gatewayAddress=${gatewayExternalIP} + fi + fi bash scripts/install_addons.sh if [ $? -ne 0 ];then exit 1 fi - + # generate cluster info if [ "$GenerateClusterInfo" == "true" ];then cat >/tmp/clusterinfo-cm.yaml < /root/.sealer/Clusterfile + fi trident on-sealer -f /root/.sealer/Clusterfile --sealer --dump-managed-cluster ${ComponentToInstallFlag} if [ $? -ne 0 ];then exit 1 diff --git a/build/plugins/pre_init_host_shell_plugin.yaml b/build/plugins/pre_init_host_shell_plugin.yaml index 64f197f0..b81a31d2 100644 --- a/build/plugins/pre_init_host_shell_plugin.yaml +++ b/build/plugins/pre_init_host_shell_plugin.yaml @@ -5,7 +5,7 @@ metadata: spec: type: SHELL action: pre-init-host - scope: master | node + scope: node data: | set -x export HostIPFamily=${HostIPFamily:-4} @@ -15,4 +15,28 @@ spec: bash scripts/pre_init_host.sh if [ $? -ne 0 ];then exit 1 - fi \ No newline at end of file + fi +--- +apiVersion: sealer.aliyun.com/v1alpha1 +kind: Plugin +metadata: + name: pre_init_host_master # Specify this plugin name,will dump in $rootfs/plugin dir. +spec: + type: SHELL + action: pre-init-host + scope: master + data: | + set -x + export HostIPFamily=${HostIPFamily:-4} + export HostIP=${HostIP} + export IPv6DualStack=${IPv6DualStack:-true} + + bash scripts/pre_init_host.sh + if [ $? -ne 0 ];then + exit 1 + fi + + cp -f scripts/push-to-registry.sh /usr/local/bin/push-to-registry.sh + chmod +x /usr/local/bin/push-to-registry.sh + cp -f scripts/etcdctl_v3.sh /usr/local/bin/etcdctl_v3 + chmod +x /usr/local/bin/etcdctl_v3 \ No newline at end of file diff --git a/build/scripts/disk_init_v2.sh b/build/scripts/disk_init_v2.sh index da0238b3..54eec6af 100644 --- a/build/scripts/disk_init_v2.sh +++ b/build/scripts/disk_init_v2.sh @@ -26,28 +26,29 @@ if [ -z "$file_system" ]; then fi mkfsForce() { - if [ "$file_system" = "ext4" ];then - mkfs.ext4 -F "$1" - elif [ "$file_system" = "xfs" ];then - mkfs.xfs -f "$1" - else - panic "file system $file_system is not supported now" - fi + if [ "$file_system" = "ext4" ];then + mkfs.ext4 -F "$1" + elif [ "$file_system" = "xfs" ];then + mkfs.xfs -f "$1" + else + panic "file system $file_system is not supported now" + fi +} + +checkMountOK() { + mountPoint=${1} + nowDev=`mount | awk -v mp="$mountPoint" '{if($3 == mp)print $1}'` + if [ "${nowDev}" != "" ];then + utils_info "${mountPoint} has already been mounted by ${nowDev}" + return 0 + fi + + return 1 } mountEtcd() { - if [[ $etcdDev == *"nvme"* ]]; then - mount |grep ^$etcdDev[p0-9]*|grep /var/lib/etcd - if [ "$?" == "0" ]; then - utils_info "$etcdDev has been mounted already, and in correct way~" - return - fi - else - mount |grep ^$etcdDev[0-9]*|grep /var/lib/etcd - if [ "$?" == "0" ]; then - utils_info "$etcdDev has been mounted already, and in correct way~" - return - fi + if checkMountOK /var/lib/etcd;then + return 0 fi mkfsForce $etcdDev @@ -78,6 +79,20 @@ if [ -z "$kubelet_size" ]; then utils_info "set partition /var/lib/kubelet size to default size - 100G" fi +checkMountOK /var/lib/kubelet +check1=$? +checkMountOK /var/lib/${container_runtime} +check2=$? +if [ "${check1}" == "0" ] && [ "${check2}" == "0" ];then + exit 0 +fi +if [ "${check1}" == "0" ] && [ "${check2}" != "0" ];then + panic "mount for /var/lib/kubelet found, but not /var/lib/${container_runtime}, if you are scaling this node and some error occurs before, you can try delete it and try again" +fi +if [ "${check1}" != "0" ] && [ "${check2}" == "0" ];then + panic "mount for /var/lib/${container_runtime} found, but not /var/lib/kubelet, if you are scaling this node and some error occurs before, you can try delete it and try again" +fi + # Step 2: create vg devPrefix="/dev/" vgName="ackdistro-pool" @@ -114,7 +129,6 @@ fi # Step 3: create lv sed -i "/\\/var\\/lib\\/kubelet/d" /etc/fstab sed -i "/\\/var\\/lib\\/${container_runtime}/d" /etc/fstab -sed -i "/\\/var\\/lib\\/${container_runtime}\\/logs/d" /etc/fstab lv_container_name="container" lv_kubelet_name="kubelet" diff --git a/build/scripts/etcdctl_v3.sh b/build/scripts/etcdctl_v3.sh new file mode 100644 index 00000000..ffe162af --- /dev/null +++ b/build/scripts/etcdctl_v3.sh @@ -0,0 +1 @@ +ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key $@ \ No newline at end of file diff --git a/build/scripts/push-to-registry.sh b/build/scripts/push-to-registry.sh new file mode 100644 index 00000000..8933e487 --- /dev/null +++ b/build/scripts/push-to-registry.sh @@ -0,0 +1,46 @@ +#! /bin/bash + +set -e + +if [ "$1" == "-h" ] || [ "$1" == "--help" ]; then + echo "此脚本的入参为一个本地存在的docker镜像的tar包路径或者一个本地存在的docker镜像名" + echo "此脚本会将传入的镜像tar包或者镜像名，转存到sealer registry当中" + echo "Usage: $0 /root/image.tar #这是一个docker镜像save的tar包" + echo " $0 image.tgz #这是对一个docker镜像save的tar包进行gzip压缩的包" + echo " $0 nginx:latest #这是一个docker镜像名称" + exit 0 +fi + +split_image_name() { + ImageUrl=$1 + res="${ImageUrl//[^\/]}" + PartNum=${#res} + if [ ${PartNum} -eq 2 ];then + Domain=$(echo $ImageUrl | cut -d'/' -f 2) + Image=$(echo $ImageUrl | cut -d'/' -f 3) + elif [ ${PartNum} -eq 1 ];then + Domain=default + Image=$(echo $ImageUrl | cut -d'/' -f 2) + elif [ ${PartNum} -eq 0 ];then + Domain=default + Image=$ImageUrl + fi +} + +if echo "$1" | grep -q -E '\.tar$';then + FullName=`docker load -i ${1} | cut -d' ' -f 3` +elif echo "$1" | grep -q -E '\.tgz$';then + image=`tar -xvf $1` + FullName=`docker load -i ${image} | cut -d' ' -f 3` +else + FullName="$1" +fi + +split_image_name $FullName + +for m in `kubectl get no -owide |grep master|awk '{print $6}'`;do + docker tag $ImageUrl ${m}:5000/$Domain/$Image + docker push ${m}:5000/$Domain/$Image +done + +echo "已成功转存到 registry-internal.adp.aliyuncs.com:5000/$Domain/$Image" \ No newline at end of file diff --git a/build/yamls/apiserver-lb-svc.yaml b/build/yamls/apiserver-lb-svc.yaml index 21e48e18..695763cf 100644 --- a/build/yamls/apiserver-lb-svc.yaml +++ b/build/yamls/apiserver-lb-svc.yaml @@ -13,5 +13,6 @@ spec: - name: https port: 6443 targetPort: 6443 + nodePort: 30057 selector: component: kube-apiserver \ No newline at end of file diff --git a/docs/user-guide/requirements.md b/docs/user-guide/requirements.md index 4c3633de..b50bb2ca 100644 --- a/docs/user-guide/requirements.md +++ b/docs/user-guide/requirements.md @@ -24,7 +24,6 @@ To successfully install ACK Distro, you need to ensure that the following requir ### Kernel： -- 4.18.* - 4.19.* - 3.10.* (must >=3.10.0-1160) diff --git a/docs/user-guide/requirements_zh.md b/docs/user-guide/requirements_zh.md index 580a2ddd..07560b56 100644 --- a/docs/user-guide/requirements_zh.md +++ b/docs/user-guide/requirements_zh.md @@ -25,7 +25,6 @@ ### Kernel： -- 4.18.* - 4.19.* - 3.10.* (must >=3.10.0-1160)