osp-rhel.yaml

# Copyright 2021 The Operating System Manager contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: operatingsystemmanager.k8c.io/v1alpha1
kind: OperatingSystemProfile
metadata:
  name: osp-rhel
  namespace: kube-system
spec:
  osName: "rhel"
  osVersion: "8.4"
  version: "v0.1.2"
  supportedCloudProviders:
    - name: "aws"
    - name: "azure"
    - name: "kubevirt"
    - name: "openstack"
    - name: "vsphere"
  supportedContainerRuntimes:
    - name: containerd
      files:
        - path: "/etc/crictl.yaml"
          content:
            inline:
              data: |
                runtime-endpoint: unix:///run/containerd/containerd.sock

        - path: "/etc/systemd/system/containerd.service.d/environment.conf"
          content:
            inline:
              data: |
                [Service]
                Restart=always
                EnvironmentFile=-/etc/environment

        - path: "/etc/containerd/config.toml"
          permissions: 0600
          content:
            inline:
              encoding: b64
              data: |
                {{ .ContainerRuntimeConfig }}
      templates:
        containerRuntimeInstallation: |-
          yum install -y yum-utils
          yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
          {{- /*
              Due to DNF modules we have to do this on docker-ce repo
              More info at: https://bugzilla.redhat.com/show_bug.cgi?id=1756473
          */}}
          yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true

          yum install -y containerd.io-1.4* yum-plugin-versionlock
          yum versionlock add containerd.io

          systemctl daemon-reload
          systemctl enable --now containerd

    - name: docker
      files:
        - path: "/etc/systemd/system/containerd.service.d/environment.conf"
          content:
            inline:
              data: |
                [Service]
                Restart=always
                EnvironmentFile=-/etc/environment

        - path: "/etc/systemd/system/docker.service.d/environment.conf"
          content:
            inline:
              data: |
                [Service]
                Restart=always
                EnvironmentFile=-/etc/environment

        - path: /etc/docker/daemon.json
          permissions: 0644
          content:
            inline:
              encoding: b64
              data: |-
                {{ .ContainerRuntimeConfig }}
      templates:
        containerRuntimeInstallation: |-
          yum install -y yum-utils
          yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
          yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true

          yum install -y \
              docker-ce-cli-19.03* \
              containerd.io-1.4* \
              docker-ce-19.03* \
              yum-plugin-versionlock
          yum versionlock add docker-ce* containerd.io

          systemctl daemon-reload
          systemctl enable --now docker

  templates:
    safeDownloadBinariesScript: |-
      {{- /* setup some common directories */}}
      opt_bin=/opt/bin
      usr_local_bin=/usr/local/bin
      cni_bin_dir=/opt/cni/bin

      {{- /* create all the necessary dirs */}}
      mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir"

      {{- /* HOST_ARCH can be defined outside of machine-controller (in kubeone for example) */}}
      arch=${HOST_ARCH-}
      if [ -z "$arch" ]
      then
      case $(uname -m) in
      x86_64)
          arch="amd64"
          ;;
      aarch64)
          arch="arm64"
          ;;
      *)
          echo "unsupported CPU architecture, exiting"
          exit 1
          ;;
      esac
      fi

      {{- /* # CNI variables */}}
      CNI_VERSION="${CNI_VERSION:-v0.8.7}"
      cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION"
      cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz"

      {{- /* download CNI */}}
      curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename"

      {{- /* download CNI checksum */}}
      cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256")
      cd "$cni_bin_dir"

      {{- /* verify CNI checksum */}}
      sha256sum -c <<<"$cni_sum"

      {{- /* unpack CNI */}}
      tar xvf "$cni_filename"
      rm -f "$cni_filename"
      cd -

      {{- /* # cri-tools variables */}}
      CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}"
      cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}"
      cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz"

      {{- /* download cri-tools */}}
      curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename"

      {{- /* download cri-tools checksum */}}
      {{- /* the cri-tools checksum file has a filename prefix that breaks sha256sum so we need to drop it with sed */}}
      cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///')
      cd "$opt_bin"

      {{- /* verify cri-tools checksum */}}
      sha256sum -c <<<"$cri_tools_sum"

      {{- /* unpack cri-tools and symlink to path so it's available to all users */}}
      tar xvf "$cri_tools_filename"
      rm -f "$cri_tools_filename"
      ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped"
      cd -

      {{- /* kubelet */}}
      KUBE_VERSION="${KUBE_VERSION:-{{ .KubeVersion }}}"
      kube_dir="$opt_bin/kubernetes-$KUBE_VERSION"
      kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch"
      kube_sum_file="$kube_dir/sha256"

      {{- /* create versioned kube dir */}}
      mkdir -p "$kube_dir"
      : >"$kube_sum_file"

      for bin in kubelet kubeadm kubectl; do
          {{- /* download kube binary */}}
          curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin"
          chmod +x "$kube_dir/$bin"

          {{- /* download kube binary checksum */}}
          sum=$(curl -Lf "$kube_base_url/$bin.sha256")

          {{- /* save kube binary checksum */}}
          echo "$sum  $kube_dir/$bin" >>"$kube_sum_file"
      done

      {{- /* check kube binaries checksum */}}
      sha256sum -c "$kube_sum_file"

      for bin in kubelet kubeadm kubectl; do
          {{- /* link kube binaries from verioned dir to $opt_bin */}}
          ln -sf "$kube_dir/$bin" "$opt_bin"/$bin
      done

    configureProxyScript: |-
      {{- if .HTTPProxy }}
      cat <<EOF | tee -a /etc/environment
      HTTP_PROXY={{ .HTTPProxy }}
      http_proxy={{ .HTTPProxy }}
      HTTPS_PROXY={{ .HTTPProxy }}
      https_proxy={{ .HTTPProxy }}
      NO_PROXY={{ .NoProxy }}
      no_proxy={{ .NoProxy }}
      EOF
      {{- end }}

  files:
    - path: "/opt/bin/health-monitor.sh"
      permissions: 0755
      content:
        inline:
          encoding: b64
          data: |
            #!/usr/bin/env bash

            # Copyright 2016 The Kubernetes Authors.
            #
            # Licensed under the Apache License, Version 2.0 (the "License");
            # you may not use this file except in compliance with the License.
            # You may obtain a copy of the License at
            #
            #     http://www.apache.org/licenses/LICENSE-2.0
            #
            # Unless required by applicable law or agreed to in writing, software
            # distributed under the License is distributed on an "AS IS" BASIS,
            # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
            # See the License for the specific language governing permissions and
            # limitations under the License.

            # This script is for master and node instance health monitoring, which is
            # packed in kube-manifest tarball. It is executed through a systemd service
            # in cluster/gce/gci/<master/node>.yaml. The env variables come from an env
            # file provided by the systemd service.

            # This script is a slightly adjusted version of
            # https://github.com/kubernetes/kubernetes/blob/e1a1aa211224fcd9b213420b80b2ae680669683d/cluster/gce/gci/health-monitor.sh
            # Adjustments are:
            # * Kubelet health port is 10248 not 10255
            # * Removal of all all references to the KUBE_ENV file

            set -o nounset
            set -o pipefail

            # We simply kill the process when there is a failure. Another systemd service will
            # automatically restart the process.
            function container_runtime_monitoring() {
              local -r max_attempts=5
              local attempt=1
              local -r container_runtime_name="${CONTAINER_RUNTIME_NAME:-docker}"
              # We still need to use 'docker ps' when container runtime is "docker". This is because
              # dockershim is still part of kubelet today. When kubelet is down, crictl pods
              # will also fail, and docker will be killed. This is undesirable especially when
              # docker live restore is disabled.
              local healthcheck_command="docker ps"
              if [[ "${CONTAINER_RUNTIME:-docker}" != "docker" ]]; then
                healthcheck_command="crictl pods"
              fi
              # Container runtime startup takes time. Make initial attempts before starting
              # killing the container runtime.
              until timeout 60 ${healthcheck_command} > /dev/null; do
                if ((attempt == max_attempts)); then
                  echo "Max attempt ${max_attempts} reached! Proceeding to monitor container runtime healthiness."
                  break
                fi
                echo "$attempt initial attempt \"${healthcheck_command}\"! Trying again in $attempt seconds..."
                sleep "$((2 ** attempt++))"
              done
              while true; do
                if ! timeout 60 ${healthcheck_command} > /dev/null; then
                  echo "Container runtime ${container_runtime_name} failed!"
                  if [[ "$container_runtime_name" == "docker" ]]; then
                    # Dump stack of docker daemon for investigation.
                    # Log file name looks like goroutine-stacks-TIMESTAMP and will be saved to
                    # the exec root directory, which is /var/run/docker/ on Ubuntu and COS.
                    pkill -SIGUSR1 dockerd
                  fi
                  systemctl kill --kill-who=main "${container_runtime_name}"
                  # Wait for a while, as we don't want to kill it again before it is really up.
                  sleep 120
                else
                  sleep "${SLEEP_SECONDS}"
                fi
              done
            }

            function kubelet_monitoring() {
              echo "Wait for 2 minutes for kubelet to be functional"
              sleep 120
              local -r max_seconds=10
              local output=""
              while true; do
                local failed=false

                if journalctl -u kubelet -n 1 | grep -q "use of closed network connection"; then
                  failed=true
                  echo "Kubelet stopped posting node status. Restarting"
                elif ! output=$(curl -m "${max_seconds}" -f -s -S http://127.0.0.1:10248/healthz 2>&1); then
                  failed=true
                  # Print the response and/or errors.
                  echo "$output"
                fi

                if [[ "$failed" == "true" ]]; then
                  echo "Kubelet is unhealthy!"
                  systemctl kill kubelet
                  # Wait for a while, as we don't want to kill it again before it is really up.
                  sleep 60
                else
                  sleep "${SLEEP_SECONDS}"
                fi
              done
            }

            ############## Main Function ################
            if [[ "$#" -ne 1 ]]; then
              echo "Usage: health-monitor.sh <container-runtime/kubelet>"
              exit 1
            fi

            SLEEP_SECONDS=10
            component=$1
            echo "Start kubernetes health monitoring for ${component}"
            if [[ "${component}" == "container-runtime" ]]; then
              container_runtime_monitoring
            elif [[ "${component}" == "kubelet" ]]; then
              kubelet_monitoring
            else
              echo "Health monitoring for component ${component} is not supported!"
            fi

    - path: "/etc/systemd/journald.conf.d/max_disk_use.conf"
      content:
        inline:
          encoding: b64
          data: |
            [Journal]
            SystemMaxUse=5G

    - path: "/opt/load-kernel-modules.sh"
      permissions: 0755
      content:
        inline:
          encoding: b64
          data: |
            #!/usr/bin/env bash
            set -euo pipefail

            modprobe ip_vs
            modprobe ip_vs_rr
            modprobe ip_vs_wrr
            modprobe ip_vs_sh

            if modinfo nf_conntrack_ipv4 &> /dev/null; then
              modprobe nf_conntrack_ipv4
            else
              modprobe nf_conntrack
            fi

    - path: "/etc/sysctl.d/k8s.conf"
      content:
        inline:
          encoding: b64
          data: |
            net.bridge.bridge-nf-call-ip6tables = 1
            net.bridge.bridge-nf-call-iptables = 1
            kernel.panic_on_oops = 1
            kernel.panic = 10
            net.ipv4.ip_forward = 1
            vm.overcommit_memory = 1
            fs.inotify.max_user_watches = 1048576
            fs.inotify.max_user_instances = 8192

    - path: "/etc/selinux/config"
      content:
        inline:
          encoding: b64
          data: |
            # This file controls the state of SELinux on the system.
            # SELINUX= can take one of these three values:
            #     enforcing - SELinux security policy is enforced.
            #     permissive - SELinux prints warnings instead of enforcing.
            #     disabled - No SELinux policy is loaded.
            SELINUX=permissive
            # SELINUXTYPE= can take one of three two values:
            #     targeted - Targeted processes are protected,
            #     minimum - Modification of targeted policy. Only selected processes are protected.
            #     mls - Multi Level Security protection.
            SELINUXTYPE=targeted

    - path: "/opt/bin/setup"
      permissions: 0755
      content:
        inline:
          encoding: b64
          data: |
            #!/bin/bash
            set -xeuo pipefail

            setenforce 0 || true

            {{- /* As we added some modules and don't want to reboot, restart the service */}}
            systemctl restart systemd-modules-load.service
            sysctl --system

            {{- /* Make sure we always disable swap - Otherwise the kubelet won't start'. */}}
            sed -i.orig '/.*swap.*/d' /etc/fstab
            swapoff -a

            {{ if eq .CloudProviderName "azure" }}
            yum update -y --disablerepo='*' --enablerepo='*microsoft*'
            {{ end }}
            yum install -y \
              device-mapper-persistent-data \
              lvm2 \
              ebtables \
              ethtool \
              nfs-utils \
              bash-completion \
              sudo \
              socat \
              wget \
              curl \
              {{- if eq .CloudProviderName "vsphere" }}
              open-vm-tools \
              {{- end }}
              {{- if eq .CloudProviderName "nutanix" }}
              iscsi-initiator-utils \
              {{- end }}
              ipvsadm

            systemctl disable --now firewalld || true

            {{- /* iscsid service is required on Nutanix machines for CSI driver to attach volumes. */}}
            {{- if eq .CloudProviderName "nutanix" }}
            systemctl enable --now iscsid
            {{ end }}

            {{- template "containerRuntimeInstallation" }}

            {{- template "safeDownloadBinariesScript" }}

            {{- template "configureProxyScript" }}

            mkdir -p /etc/systemd/system/kubelet.service.d/
            # set kubelet nodeip environment variable
            /opt/bin/setup_net_env.sh

            {{ if eq .CloudProviderName "vsphere" }}
            systemctl enable --now vmtoolsd.service
            {{ end -}}
            systemctl enable --now kubelet
            systemctl enable --now --no-block kubelet-healthcheck.service

    - path: "/opt/bin/supervise.sh"
      permissions: 0755
      content:
        inline:
          encoding: b64
          data: |
            #!/bin/bash
            set -xeuo pipefail
            while ! "$@"; do
              sleep 1
            done

    - path: "/etc/systemd/system/kubelet.service"
      content:
        inline:
          encoding: b64
          data: |
            [Unit]
            After={{ .ContainerRuntime }}.service
            Requires={{ .ContainerRuntime }}.service

            Description=kubelet: The Kubernetes Node Agent
            Documentation=https://kubernetes.io/docs/home/

            [Service]
            Restart=always
            StartLimitInterval=0
            RestartSec=10
            CPUAccounting=true
            MemoryAccounting=true

            Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/"
            EnvironmentFile=-/etc/environment

            ExecStartPre=/bin/bash /opt/load-kernel-modules.sh
            ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh
            ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \
              --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
              --kubeconfig=/var/lib/kubelet/kubeconfig \
              --config=/etc/kubernetes/kubelet.conf \
              --network-plugin=cni \
              --cert-dir=/etc/kubernetes/pki \
              {{- if .ExternalCloudProvider }}
              --cloud-provider=external \
              {{- else if .InTreeCCMAvailable }}
              --cloud-provider={{- .CloudProviderName }} \
              --cloud-config=/etc/kubernetes/cloud-config \
              {{- end }}
              {{- if ne .CloudProviderName "aws" }}
              --hostname-override=${KUBELET_HOSTNAME} \
              {{- else if and (eq .CloudProviderName "aws") (.ExternalCloudProvider) }}
              --hostname-override=${KUBELET_HOSTNAME} \
              {{- end }}
              --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \
              --feature-gates=DynamicKubeletConfig=true \
              --exit-on-lock-contention \
              --lock-file=/tmp/kubelet.lock \
              {{- if .PauseImage }}
              --pod-infra-container-image={{ .PauseImage }} \
              {{- end }}
              {{- if .InitialTaints }}
              --register-with-taints={{- .InitialTaints }} \
              {{- end }}
              {{- if eq .ContainerRuntime "containerd" }}
              --container-runtime=remote \
              --container-runtime-endpoint=unix:///run/containerd/containerd.sock \
              {{- end }}
              {{- if eq .ContainerRuntime "docker" }}
              --container-runtime=docker \
              --container-runtime-endpoint=unix:///var/run/dockershim.sock \
              {{- end }}
              --node-ip ${KUBELET_NODE_IP}

            [Install]
            WantedBy=multi-user.target

    - path: "/etc/kubernetes/cloud-config"
      permissions: 0600
      content:
        inline:
          encoding: b64
          data: |
            {{ .CloudConfig }}

    - path: "/opt/bin/setup_net_env.sh"
      permissions: 0755
      content:
        inline:
          encoding: b64
          data: |
            #!/usr/bin/env bash
            echodate() {
              echo "[$(date -Is)]" "$@"
            }

            # get the default interface IP address
            DEFAULT_IFC_IP=$(ip -o  route get 1 | grep -oP "src \K\S+")

            if [ -z "${DEFAULT_IFC_IP}" ]
            then
              echodate "Failed to get IP address for the default route interface"
              exit 1
            fi

             # get the full hostname
            FULL_HOSTNAME=$(hostname -f)

            # write the nodeip_env file
            # we need the line below because flatcar has the same string "coreos" in that file
            if grep -q coreos /etc/os-release
            then
              echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf
            elif [ ! -d /etc/systemd/system/kubelet.service.d ]
            then
              echodate "Can't find kubelet service extras directory"
              exit 1
            else
              echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf
            fi

    - path: "/etc/kubernetes/pki/ca.crt"
      content:
        inline:
          encoding: b64
          data: |
            {{ .KubernetesCACert }}

    - path: "/etc/systemd/system/setup.service"
      permissions: 0644
      content:
        inline:
          encoding: b64
          data: |
            [Install]
            WantedBy=multi-user.target

            [Unit]
            Requires=network-online.target
            After=network-online.target

            [Service]
            Type=oneshot
            RemainAfterExit=true
            EnvironmentFile=-/etc/environment
            ExecStart=/opt/bin/supervise.sh /opt/bin/setup

    - path: "/etc/profile.d/opt-bin-path.sh"
      permissions: 0644
      content:
        inline:
          encoding: b64
          data: |
            export PATH="/opt/bin:$PATH"

    - path: "/etc/kubernetes/kubelet.conf"
      content:
        inline:
          encoding: b64
          data: |
            apiVersion: kubelet.config.k8s.io/v1beta1
            kind: KubeletConfiguration
            authentication:
              anonymous:
                enabled: false
              webhook:
                enabled: true
              x509:
                clientCAFile: /etc/kubernetes/pki/ca.crt
            authorization:
              mode: Webhook
            cgroupDriver: systemd
            clusterDNS:
            {{- range .ClusterDNSIPs }}
            - "{{ . }}"
            {{- end }}
            clusterDomain: cluster.local
            {{- /* containerLogMaxSize and containerLogMaxFiles have no effect for docker  */}}
            {{- if ne .ContainerRuntime "docker" }}
            {{- if .ContainerLogMaxSize }}
            containerLogMaxSize: {{ .ContainerLogMaxSize }}
            {{- else }}
            containerLogMaxSize: 100Mi
            {{- end }}
            {{- if .ContainerLogMaxFiles }}
            containerLogMaxFiles: {{ .ContainerLogMaxFiles }}
            {{- else }}
            containerLogMaxFiles: 5
            {{- end }}
            {{- end }}
            featureGates:
            {{- if .KubeletFeatureGates -}}
              {{ range $key, $val := .KubeletFeatureGates }}
              {{ $key }}: {{ $val }}
              {{- end -}}
            {{- end }}
            protectKernelDefaults: true
            readOnlyPort: 0
            rotateCertificates: true
            serverTLSBootstrap: true
            staticPodPath: /etc/kubernetes/manifests
            kubeReserved:
            {{- if .KubeReserved -}}
              {{ range $key, $val := .KubeReserved }}
              {{ $key }}: {{ $val }}
              {{- end -}}
            {{- else }}
              cpu: 200m
              ephemeral-storage: 1Gi
              memory: 200Mi
            {{- end }}
            systemReserved:
            {{- if .SystemReserved -}}
              {{ range $key, $val := .SystemReserved }}
              {{ $key }}: {{ $val }}
              {{- end -}}
            {{- else }}
              cpu: 200m
              ephemeral-storage: 1Gi
              memory: 200Mi
            {{- end }}
            evictionHard:
            {{- if .EvictionHard -}}
              {{ range $key, $val := .EvictionHard }}
              {{ $key }}: {{ $val }}
              {{- end -}}
            {{- else }}
              imagefs.available: 15%
              memory.available: 100Mi
              nodefs.available: 10%
              nodefs.inodesFree: 5%
            {{- end }}
            tlsCipherSuites:
            - TLS_AES_128_GCM_SHA256
            - TLS_AES_256_GCM_SHA384
            - TLS_CHACHA20_POLY1305_SHA256
            - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
            - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
            - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
            - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
            - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
            - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
            volumePluginDir: /var/lib/kubelet/volumeplugins

    - path: /etc/systemd/system/kubelet-healthcheck.service
      permissions: 0644
      content:
        inline:
          encoding: b64
          data: |
            [Unit]
            Requires=kubelet.service
            After=kubelet.service

            [Service]
            ExecStart=/opt/bin/health-monitor.sh kubelet

            [Install]
            WantedBy=multi-user.target
  modules:
    bootcmd:
      - modprobe ip_tables