[WIP] k8s

providers/amazon/images/ivy-kubernetes/provision.yml

@@ -0,0 +1,8 @@
+---
+- name: Provision a machine
+  hosts: 127.0.0.1
+  user: ec2-user
+  connection: local
+  roles:
+    - kubernetes
+    - vault

roles/kubernetes/defaults/main.yml

@@ -0,0 +1,39 @@
+---
+# Versions
+etcd_version: "v3.4.9"
+cfssl_version: "1.4.1"
+kubernetes_version: "v1.18.5"
+containerd_version: "1.3.4"
+cni_plugins_version: "v0.8.6"
+# Download dirs
+tmp_dir: "/tmp"
+bin_dir: "/usr/local/bin"
+# ETCD
+etcd_url: "https://github.com/etcd-io/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
+etcd_dest: "{{ tmp_dir }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
+# cfssl
+cfssl_url: "https://github.com/cloudflare/cfssl/releases/download/v{{ cfssl_version }}/cfssl_{{ cfssl_version }}_linux_amd64"
+cfssl_dest: "{{ bin_dir }}/cfssl"
+cfssljson_url: "https://github.com/cloudflare/cfssl/releases/download/v{{ cfssl_version }}/cfssljson_{{ cfssl_version }}_linux_amd64"
+cfssljson_dest: "{{ bin_dir }}/cfssljson"
+# Kubernetes URL
+kubernetes_base_url: "https://storage.googleapis.com/kubernetes-release/release"
+# kubernetes master
+kubernetes_api_url: "{{ kubernetes_base_url }}/{{ kubernetes_version }}/bin/linux/amd64/kube-apiserver"
+kubernetes_api_dest: "{{ bin_dir }}/kube-apiserver"
+kubernetes_controller_manager_url: "{{ kubernetes_base_url }}/{{ kubernetes_version }}/bin/linux/amd64/kube-controller-manager"
+kubernetes_controller_manager_dest: "{{ bin_dir }}/kube-controller-manager"
+kubernetes_scheduler_url: "{{ kubernetes_base_url }}/{{ kubernetes_version }}/bin/linux/amd64/kube-scheduler"
+kubernetes_scheduler_dest: "{{ bin_dir }}/kube-scheduler"
+# kubernetes worker
+kubernetes_proxy_url: "{{ kubernetes_base_url }}/{{ kubernetes_version }}/bin/linux/amd64/kube-proxy"
+kubernetes_proxy_dest: "{{ bin_dir }}/kube-proxy"
+kubelet_url: "{{ kubernetes_base_url }}/{{ kubernetes_version }}/bin/linux/amd64/kubelet"
+kubelet_dest: "{{ bin_dir }}/kubelet"
+# all kubernetes hosts
+kubectl_url: "{{ kubernetes_base_url }}/{{ kubernetes_version }}/bin/linux/amd64/kubectl"
+kubectl_dest: "{{ bin_dir }}/kubectl"
+containerd_url: "https://storage.googleapis.com/cri-containerd-release/cri-containerd-{{ containerd_version }}.linux-amd64.tar.gz"
+containerd_dest: "{{ tmp_dir }}/cri-containerd-{{ containerd_version }}.linux-amd64.tar.gz"
+cni_plugins_url: "https://github.com/containernetworking/plugins/releases/download/{{ cni_plugins_version }}/cni-plugins-linux-amd64-{{ cni_plugins_version }}.tgz"
+cni_plugins_dest: "{{ tmp_dir }}/cni-plugins-linux-amd64-{{ cni_plugins_version }}.tgz"

roles/kubernetes/files/etc/etcd/etcdctl.env

@@ -0,0 +1,7 @@
+export ETCDCTL_API=3
+
+export ETCDCTL_CACERT=/etc/etcd/pki/ca.crt
+export ETCDCTL_CERT=/etc/etcd/pki/etcdctl-etcd-client.crt
+export ETCDCTL_KEY=/etc/etcd/pki/etcdctl-etcd-client.key
+
+export ETCDCTL_DIAL_TIMEOUT=3s

roles/kubernetes/files/etc/systemd/system/sys-fs-bpf.mount

@@ -0,0 +1,15 @@
+[Unit]
+Description=Cilium BPF mounts
+Documentation=https://docs.cilium.io/
+DefaultDependencies=no
+Before=local-fs.target umount.target
+After=swap.target
+
+[Mount]
+What=bpffs
+Where=/sys/fs/bpf
+Type=bpf
+Options=rw,nosuid,nodev,noexec,relatime,mode=700
+
+[Install]
+WantedBy=multi-user.target

roles/kubernetes/files/opt/ivy/kubernetes/generate-admin-config.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+CLUSTER_NAME="${1:-ivynetes}"
+CERTIFICATE_AUTHORITY_LOCATION="${2:-/var/lib/kubernetes/ca.pem}"
+EMBED_CERTS="${3:-true}"
+API_SERVER_ENDPOINT="${4:-https://127.0.0.1:6443}"
+ADMIN_CERTIFICATE_LOCATION="${5:-/var/lib/kubernetes/admin.pem}"
+ADMIN_KEY_LOCATION="${6:-/var/lib/kubernetes/admin-key.pem}"
+
+kubectl config set-cluster "${CLUSTER_NAME}" \
+  --certificate-authority="${CERTIFICATE_AUTHORITY_LOCATION}" \
+  --embed-certs="${EMBED_CERTS}" \
+  --server="${API_SERVER_ENDPOINT}" \
+  --kubeconfig=admin.kubeconfig
+
+kubectl config set-credentials admin \
+  --client-certificate="${ADMIN_CERTIFICATE_LOCATION}" \
+  --client-key="${ADMIN_KEY_LOCATION}" \
+  --embed-certs="${EMBED_CERTS}" \
+  --kubeconfig=admin.kubeconfig
+
+kubectl config set-context default \
+  --cluster="${CLUSTER_NAME}" \
+  --user=admin \
+  --kubeconfig=admin.kubeconfig
+
+kubectl config use-context default --kubeconfig=admin.kubeconfig

roles/kubernetes/files/opt/ivy/kubernetes/generate-encryption-config.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+ENCRYPTION_KEY=$(head -c 32 /dev/urandom | base64)
+ENCRYPTION_LOCATION="${1:-encryption-config.yaml}"
+
+cat > "${ENCRYPTION_LOCATION}" <<EOF
+kind: EncryptionConfig
+apiVersion: v1
+resources:
+  - resources:
+      - secrets
+    providers:
+      - aescbc:
+          keys:
+            - name: key1
+              secret: ${ENCRYPTION_KEY}
+      - identity: {}
+EOF

roles/kubernetes/files/opt/ivy/kubernetes/generate-kube-controller-manager-config.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+CLUSTER_NAME="${1:-ivynetes}"
+CERTIFICATE_AUTHORITY_LOCATION="${2:-/var/lib/kubernetes/ca.pem}"
+EMBED_CERTS="${3:-true}"
+API_SERVER_ENDPOINT="${4:-https://127.0.0.1:6443}"
+KUBE_CONTROLLER_MANAGER_CERTIFICATE_LOCATION="${5:-/var/lib/kubernetes/kube-controller-manager.pem}"
+KUBE_CONTROLLER_MANAGER_KEY_LOCATION="${6:-/var/lib/kubernetes/kube-controller-manager-key.pem}"
+
+kubectl config set-cluster "${CLUSTER_NAME}" \
+  --certificate-authority="${CERTIFICATE_AUTHORITY_LOCATION}" \
+  --embed-certs="${EMBED_CERTS}" \
+  --server="${API_SERVER_ENDPOINT}" \
+  --kubeconfig=kube-controller-manager.kubeconfig
+
+kubectl config set-credentials system:kube-controller-manager \
+  --client-certificate="${KUBE_CONTROLLER_MANAGER_CERTIFICATE_LOCATION}" \
+  --client-key="${KUBE_CONTROLLER_MANAGER_KEY_LOCATION}" \
+  --embed-certs="${EMBED_CERTS}" \
+  --kubeconfig=kube-controller-manager.kubeconfig
+
+kubectl config set-context default \
+  --cluster="${CLUSTER_NAME}" \
+  --user=system:kube-controller-manager \
+  --kubeconfig=kube-controller-manager.kubeconfig
+
+kubectl config use-context default --kubeconfig=kube-controller-manager.kubeconfig

roles/kubernetes/files/opt/ivy/kubernetes/generate-kube-proxy-config.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+CLUSTER_NAME="${1:-ivynetes}"
+CERTIFICATE_AUTHORITY_LOCATION="${2:-/var/lib/kubernetes/ca.pem}"
+EMBED_CERTS="${3:-true}"
+API_SERVER_ENDPOINT="${4}" # Load balancer endpoint
+KUBE_PROXY_CERTIFICATE_LOCATION="${5:-/var/lib/kubernetes/kube-proxy.pem}"
+KUBE_PROXY_KEY_LOCATION="${6:-/var/lib/kubernetes/kube-proxy-key.pem}"
+
+kubectl config set-cluster "${CLUSTER_NAME}" \
+  --certificate-authority="${CERTIFICATE_AUTHORITY_LOCATION}" \
+  --embed-certs="${EMBED_CERTS}" \
+  --server="${API_SERVER_ENDPOINT}" \
+  --kubeconfig=kube-proxy.kubeconfig
+
+kubectl config set-credentials system:kube-proxy \
+  --client-certificate="${KUBE_PROXY_CERTIFICATE_LOCATION}" \
+  --client-key="${KUBE_PROXY_KEY_LOCATION}" \
+  --embed-certs="${EMBED_CERTS}" \
+  --kubeconfig=kube-proxy.kubeconfig
+
+kubectl config set-context default \
+  --cluster="${CLUSTER_NAME}" \
+  --user=system:kube-proxy \
+  --kubeconfig=kube-proxy.kubeconfig
+
+kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

roles/kubernetes/files/opt/ivy/kubernetes/generate-kube-scheduler-config.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+CLUSTER_NAME="${1:-ivynetes}"
+CERTIFICATE_AUTHORITY_LOCATION="${2:-/var/lib/kubernetes/ca.pem}"
+EMBED_CERTS="${3:-true}"
+API_SERVER_ENDPOINT="${4:-https://127.0.0.1:6443}"
+KUBE_SCHEDULER_CERTIFICATE_LOCATION="${5:-/var/lib/kubernetes/kube-scheduler.pem}"
+KUBE_SCHEDULER_KEY_LOCATION="${6:-/var/lib/kubernetes/kube-scheduler-key.pem}"
+
+kubectl config set-cluster "${CLUSTER_NAME}" \
+  --certificate-authority="${CERTIFICATE_AUTHORITY_LOCATION}" \
+  --embed-certs="${EMBED_CERTS}" \
+  --server="${API_SERVER_ENDPOINT}" \
+  --kubeconfig=kube-scheduler.kubeconfig
+
+kubectl config set-credentials system:kube-scheduler \
+  --client-certificate="${KUBE_SCHEDULER_CERTIFICATE_LOCATION}" \
+  --client-key="${KUBE_SCHEDULER_KEY_LOCATION}" \
+  --embed-certs="${EMBED_CERTS}" \
+  --kubeconfig=kube-scheduler.kubeconfig
+
+kubectl config set-context default \
+  --cluster="${CLUSTER_NAME}" \
+  --user=system:kube-scheduler \
+  --kubeconfig=kube-scheduler.kubeconfig
+
+kubectl config use-context default --kubeconfig=kube-scheduler.kubeconfig

roles/kubernetes/files/opt/ivy/kubernetes/generate-kubelet-config.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+# Note: It may be a better idea to create 1 key and certificate for all kubelets
+
+CLUSTER_NAME="${1:-ivynetes}"
+CERTIFICATE_AUTHORITY_LOCATION="${2:-/var/lib/kubernetes/ca.pem}"
+EMBED_CERTS="${3:-true}"
+API_SERVER_ENDPOINT="${4}" # Load balancer endpoint
+KUBELET_CERTIFICATE_LOCATION="${5:-/var/lib/kubernetes/kubelet.pem}"
+KUBELET_KEY_LOCATION="${6:-/var/lib/kubernetes/kubelet-key.pem}"
+KUBELET_ID="${7:-kubelet}"
+
+kubectl config set-cluster "${CLUSTER_NAME}" \
+  --certificate-authority="${CERTIFICATE_AUTHORITY_LOCATION}" \
+  --embed-certs="${EMBED_CERTS}" \
+  --server="${API_SERVER_ENDPOINT}" \
+  --kubeconfig=kubelet.kubeconfig
+
+kubectl config set-credentials "system:node:${KUBELET_ID}" \
+  --client-certificate="${KUBELET_CERTIFICATE_LOCATION}" \
+  --client-key="${KUBELET_KEY_LOCATION}" \
+  --embed-certs="${EMBED_CERTS}" \
+  --kubeconfig=kubelet.kubeconfig
+
+kubectl config set-context default \
+  --cluster="${CLUSTER_NAME}" \
+  --user="system:node:${KUBELET_ID}" \
+  --kubeconfig=kubelet.kubeconfig
+
+kubectl config use-context default --kubeconfig=kubelet.kubeconfig

roles/kubernetes/files/usr/local/bin/etcdctl.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+if ! [ -r "/etc/etcd/etcdctl.env" ]; then
+	echo "Unable to read the etcdctl environment file '/etc/etcd/etcdctl.env'. The file must exist, and this wrapper must be run as root."
+	exit 1
+fi
+. "/etc/etcd/etcdctl.env"
+"/opt/bin/etcdctl" "$@"

roles/kubernetes/tasks/Amazon.yml

@@ -0,0 +1,6 @@
+---
+- name: Copying systemd cilium BPFFS mount
+  copy:
+    src: "etc/systemd/system/sys-fs-bpf.mount"
+    dest: "/etc/systemd/system/sys-fs-bpf.mount"
+    mode: 0755

roles/kubernetes/tasks/main.yml

@@ -0,0 +1,100 @@
+---
+# Use specific install tasks for each OS family
+- include: "{{ ansible_distribution }}.yml"
+
+- name: install utilities
+  yum:
+    name:
+      - bridge-utils
+      - btrfs-progs-devel
+      - lvm2-devel
+      - nfs-utils
+      - conntrack-tools
+      - ipset
+    state: latest
+    enablerepo: epel
+
+- name: download kubernetes dependencies
+  get_url:
+    url: "{{ item.url }}"
+    dest: "{{ item.dest }}"
+    mode: 0755
+  loop:
+    - { url: "{{ etcd_url }}", dest: "{{ etcd_dest }}" }
+    - { url: "{{ cfssl_url }}", dest: "{{ cfssl_dest }}" }
+    - { url: "{{ cfssljson_url }}", dest: "{{ cfssljson_dest }}" }
+    - { url: "{{ kubernetes_api_url }}", dest: "{{ kubernetes_api_dest }}" }
+    - { url: "{{ kubernetes_controller_manager_url }}", dest: "{{ kubernetes_controller_manager_dest }}" }
+    - { url: "{{ kubernetes_scheduler_url }}", dest: "{{ kubernetes_scheduler_dest }}" }
+    - { url: "{{ kubernetes_proxy_url }}", dest: "{{ kubernetes_proxy_dest }}" }
+    - { url: "{{ kubelet_url }}", dest: "{{ kubelet_dest }}" }
+    - { url: "{{ kubectl_url }}", dest: "{{ kubectl_dest }}" }
+    - { url: "{{ containerd_url }}", dest: "{{ containerd_dest }}" }
+    - { url: "{{ cni_plugins_url }}", dest: "{{ cni_plugins_dest }}" }
+
+- name: create kubernetes, etcd, cni and containerd directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+    recurse: yes
+  loop:
+    - /etc/etcd
+    - /etc/cni/net.d
+    - /etc/kubernetes/config
+    - /opt/cni/bin
+    - /var/lib/etcd
+    - /var/lib/kubelet
+    - /var/lib/kube-proxy
+    - /var/lib/kubernetes
+    - /var/run/kubernetes
+    - /opt/ivy/kubernetes
+
+- name: extract files and place it on /tmp
+  unarchive:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}/"
+    remote_src: yes
+  loop:
+    - { src: "{{ etcd_dest }}", dest: "{{ tmp_dir }}" }
+    - { src: "{{ containerd_dest }}", dest: "{{ tmp_dir }}" }
+    - { src: "{{ cni_plugins_dest }}", dest: "/opt/cni/bin/" }
+
+- name: copy etcd and runc binaries to PATH
+  copy:
+    src: "{{ item }}"
+    dest: "{{ bin_dir }}"
+    mode: 0755
+    remote_src: yes
+  loop:
+    - "{{ tmp_dir }}/etcd-{{ etcd_version }}-linux-amd64/etcd"
+    - "{{ tmp_dir }}/etcd-{{ etcd_version }}-linux-amd64/etcdctl"
+    - "{{ tmp_dir }}/usr/local/sbin/runc"
+
+- name: copy containerd binaries to PATH
+  copy:
+    src: "{{ item }}"
+    dest: "{{ bin_dir }}"
+    mode: 0755
+    remote_src: yes
+  with_fileglob:
+    - "{{ tmp_dir }}/usr/local/bin/*"
+
+- name: copy shell scripts
+  copy:
+    src: "{{ item }}"
+    dest: "/{{ item }}"
+    mode: 0755
+  loop:
+    - usr/local/bin/etcdctl.sh
+    - etc/etcd/etcdctl.env
+
+- name: copy kubernetes ivy scripts
+  copy:
+    src: "{{ item }}"
+    dest: "/{{ item }}"
+    mode: 0755
+  with_fileglob:
+    - opt/ivy/kubernetes/*

roles/system-base/files/opt/ivy/bash_lib/aws.sh

@@ -318,3 +318,26 @@ function get_secret() {
     local VALUE=$(aws secretsmanager --region "${REGION}" get-secret-value --secret-id "${SECRET_ID}" | jq --raw-output .SecretString)
     echo ${VALUE}
 }
+
+function trust_sysenv_ca() {
+    local DISTRO="$(grep '^NAME=' /etc/os-release | cut -d '"' -f2)"
+    local SSM_CA_CERTIFICATE="/$(get_ivy_tag)/$(get_environment)/CA/ca.pem"
+    local REGION="${1:-$(get_region)}"
+    case "${DISTRO}" in
+      Amazon Linux)
+      local CA_TRUST_DIR='/etc/pki/ca-trust/source/anchors/'
+      local UPDATE_CA_COMMAND='update-ca-trust extract'
+      ;;
+      Ubuntu)
+      local CA_TRUST_DIR='/usr/local/share/ca-certificates/'
+      local UPDATE_CA_COMMAND='update-ca-certificates'
+      ;;
+      *)
+      echo "Only Amazon Linux and Ubuntu are supported at the moment" >&2
+      return 1
+      ;;
+    esac
+    local CA_CRT="${CA_TRUST_DIR}/ivy.pem"
+    get_ssm_param "${SSM_CA_CERTIFICATE}" "${REGION}" > "${CA_CRT}"
+    sudo ${UPDATE_CA_COMMAND}
+}