Skip to content

Commit

Permalink
[WIP] k8s
Browse files Browse the repository at this point in the history
  • Loading branch information
missingcharacter committed Dec 14, 2020
1 parent 45d478d commit 2a8ddf0
Show file tree
Hide file tree
Showing 14 changed files with 373 additions and 0 deletions.
8 changes: 8 additions & 0 deletions providers/amazon/images/ivy-kubernetes/provision.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
---
- name: Provision a machine
hosts: 127.0.0.1
user: ec2-user
connection: local
roles:
- kubernetes
- vault
39 changes: 39 additions & 0 deletions roles/kubernetes/defaults/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
---
# Versions
etcd_version: "v3.4.9"
cfssl_version: "1.4.1"
kubernetes_version: "v1.18.5"
containerd_version: "1.3.4"
cni_plugins_version: "v0.8.6"
# Download dirs
tmp_dir: "/tmp"
bin_dir: "/usr/local/bin"
# ETCD
etcd_url: "https://github.com/etcd-io/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
etcd_dest: "{{ tmp_dir }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
# cfssl
cfssl_url: "https://github.com/cloudflare/cfssl/releases/download/v{{ cfssl_version }}/cfssl_{{ cfssl_version }}_linux_amd64"
cfssl_dest: "{{ bin_dir }}/cfssl"
cfssljson_url: "https://github.com/cloudflare/cfssl/releases/download/v{{ cfssl_version }}/cfssljson_{{ cfssl_version }}_linux_amd64"
cfssljson_dest: "{{ bin_dir }}/cfssljson"
# Kubernetes URL
kubernetes_base_url: "https://storage.googleapis.com/kubernetes-release/release"
# kubernetes master
kubernetes_api_url: "{{ kubernetes_base_url }}/{{ kubernetes_version }}/bin/linux/amd64/kube-apiserver"
kubernetes_api_dest: "{{ bin_dir }}/kube-apiserver"
kubernetes_controller_manager_url: "{{ kubernetes_base_url }}/{{ kubernetes_version }}/bin/linux/amd64/kube-controller-manager"
kubernetes_controller_manager_dest: "{{ bin_dir }}/kube-controller-manager"
kubernetes_scheduler_url: "{{ kubernetes_base_url }}/{{ kubernetes_version }}/bin/linux/amd64/kube-scheduler"
kubernetes_scheduler_dest: "{{ bin_dir }}/kube-scheduler"
# kubernetes worker
kubernetes_proxy_url: "{{ kubernetes_base_url }}/{{ kubernetes_version }}/bin/linux/amd64/kube-proxy"
kubernetes_proxy_dest: "{{ bin_dir }}/kube-proxy"
kubelet_url: "{{ kubernetes_base_url }}/{{ kubernetes_version }}/bin/linux/amd64/kubelet"
kubelet_dest: "{{ bin_dir }}/kubelet"
# all kubernetes hosts
kubectl_url: "{{ kubernetes_base_url }}/{{ kubernetes_version }}/bin/linux/amd64/kubectl"
kubectl_dest: "{{ bin_dir }}/kubectl"
containerd_url: "https://storage.googleapis.com/cri-containerd-release/cri-containerd-{{ containerd_version }}.linux-amd64.tar.gz"
containerd_dest: "{{ tmp_dir }}/cri-containerd-{{ containerd_version }}.linux-amd64.tar.gz"
cni_plugins_url: "https://github.com/containernetworking/plugins/releases/download/{{ cni_plugins_version }}/cni-plugins-linux-amd64-{{ cni_plugins_version }}.tgz"
cni_plugins_dest: "{{ tmp_dir }}/cni-plugins-linux-amd64-{{ cni_plugins_version }}.tgz"
7 changes: 7 additions & 0 deletions roles/kubernetes/files/etc/etcd/etcdctl.env
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
export ETCDCTL_API=3

export ETCDCTL_CACERT=/etc/etcd/pki/ca.crt
export ETCDCTL_CERT=/etc/etcd/pki/etcdctl-etcd-client.crt
export ETCDCTL_KEY=/etc/etcd/pki/etcdctl-etcd-client.key

export ETCDCTL_DIAL_TIMEOUT=3s
15 changes: 15 additions & 0 deletions roles/kubernetes/files/etc/systemd/system/sys-fs-bpf.mount
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
[Unit]
Description=Cilium BPF mounts
Documentation=https://docs.cilium.io/
DefaultDependencies=no
Before=local-fs.target umount.target
After=swap.target

[Mount]
What=bpffs
Where=/sys/fs/bpf
Type=bpf
Options=rw,nosuid,nodev,noexec,relatime,mode=700

[Install]
WantedBy=multi-user.target
29 changes: 29 additions & 0 deletions roles/kubernetes/files/opt/ivy/kubernetes/generate-admin-config.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'

CLUSTER_NAME="${1:-ivynetes}"
CERTIFICATE_AUTHORITY_LOCATION="${2:-/var/lib/kubernetes/ca.pem}"
EMBED_CERTS="${3:-true}"
API_SERVER_ENDPOINT="${4:-https://127.0.0.1:6443}"
ADMIN_CERTIFICATE_LOCATION="${5:-/var/lib/kubernetes/admin.pem}"
ADMIN_KEY_LOCATION="${6:-/var/lib/kubernetes/admin-key.pem}"

kubectl config set-cluster "${CLUSTER_NAME}" \
--certificate-authority="${CERTIFICATE_AUTHORITY_LOCATION}" \
--embed-certs="${EMBED_CERTS}" \
--server="${API_SERVER_ENDPOINT}" \
--kubeconfig=admin.kubeconfig

kubectl config set-credentials admin \
--client-certificate="${ADMIN_CERTIFICATE_LOCATION}" \
--client-key="${ADMIN_KEY_LOCATION}" \
--embed-certs="${EMBED_CERTS}" \
--kubeconfig=admin.kubeconfig

kubectl config set-context default \
--cluster="${CLUSTER_NAME}" \
--user=admin \
--kubeconfig=admin.kubeconfig

kubectl config use-context default --kubeconfig=admin.kubeconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'

ENCRYPTION_KEY=$(head -c 32 /dev/urandom | base64)
ENCRYPTION_LOCATION="${1:-encryption-config.yaml}"

cat > "${ENCRYPTION_LOCATION}" <<EOF
kind: EncryptionConfig
apiVersion: v1
resources:
- resources:
- secrets
providers:
- aescbc:
keys:
- name: key1
secret: ${ENCRYPTION_KEY}
- identity: {}
EOF
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'

CLUSTER_NAME="${1:-ivynetes}"
CERTIFICATE_AUTHORITY_LOCATION="${2:-/var/lib/kubernetes/ca.pem}"
EMBED_CERTS="${3:-true}"
API_SERVER_ENDPOINT="${4:-https://127.0.0.1:6443}"
KUBE_CONTROLLER_MANAGER_CERTIFICATE_LOCATION="${5:-/var/lib/kubernetes/kube-controller-manager.pem}"
KUBE_CONTROLLER_MANAGER_KEY_LOCATION="${6:-/var/lib/kubernetes/kube-controller-manager-key.pem}"

kubectl config set-cluster "${CLUSTER_NAME}" \
--certificate-authority="${CERTIFICATE_AUTHORITY_LOCATION}" \
--embed-certs="${EMBED_CERTS}" \
--server="${API_SERVER_ENDPOINT}" \
--kubeconfig=kube-controller-manager.kubeconfig

kubectl config set-credentials system:kube-controller-manager \
--client-certificate="${KUBE_CONTROLLER_MANAGER_CERTIFICATE_LOCATION}" \
--client-key="${KUBE_CONTROLLER_MANAGER_KEY_LOCATION}" \
--embed-certs="${EMBED_CERTS}" \
--kubeconfig=kube-controller-manager.kubeconfig

kubectl config set-context default \
--cluster="${CLUSTER_NAME}" \
--user=system:kube-controller-manager \
--kubeconfig=kube-controller-manager.kubeconfig

kubectl config use-context default --kubeconfig=kube-controller-manager.kubeconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'

CLUSTER_NAME="${1:-ivynetes}"
CERTIFICATE_AUTHORITY_LOCATION="${2:-/var/lib/kubernetes/ca.pem}"
EMBED_CERTS="${3:-true}"
API_SERVER_ENDPOINT="${4}" # Load balancer endpoint
KUBE_PROXY_CERTIFICATE_LOCATION="${5:-/var/lib/kubernetes/kube-proxy.pem}"
KUBE_PROXY_KEY_LOCATION="${6:-/var/lib/kubernetes/kube-proxy-key.pem}"

kubectl config set-cluster "${CLUSTER_NAME}" \
--certificate-authority="${CERTIFICATE_AUTHORITY_LOCATION}" \
--embed-certs="${EMBED_CERTS}" \
--server="${API_SERVER_ENDPOINT}" \
--kubeconfig=kube-proxy.kubeconfig

kubectl config set-credentials system:kube-proxy \
--client-certificate="${KUBE_PROXY_CERTIFICATE_LOCATION}" \
--client-key="${KUBE_PROXY_KEY_LOCATION}" \
--embed-certs="${EMBED_CERTS}" \
--kubeconfig=kube-proxy.kubeconfig

kubectl config set-context default \
--cluster="${CLUSTER_NAME}" \
--user=system:kube-proxy \
--kubeconfig=kube-proxy.kubeconfig

kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'

CLUSTER_NAME="${1:-ivynetes}"
CERTIFICATE_AUTHORITY_LOCATION="${2:-/var/lib/kubernetes/ca.pem}"
EMBED_CERTS="${3:-true}"
API_SERVER_ENDPOINT="${4:-https://127.0.0.1:6443}"
KUBE_SCHEDULER_CERTIFICATE_LOCATION="${5:-/var/lib/kubernetes/kube-scheduler.pem}"
KUBE_SCHEDULER_KEY_LOCATION="${6:-/var/lib/kubernetes/kube-scheduler-key.pem}"

kubectl config set-cluster "${CLUSTER_NAME}" \
--certificate-authority="${CERTIFICATE_AUTHORITY_LOCATION}" \
--embed-certs="${EMBED_CERTS}" \
--server="${API_SERVER_ENDPOINT}" \
--kubeconfig=kube-scheduler.kubeconfig

kubectl config set-credentials system:kube-scheduler \
--client-certificate="${KUBE_SCHEDULER_CERTIFICATE_LOCATION}" \
--client-key="${KUBE_SCHEDULER_KEY_LOCATION}" \
--embed-certs="${EMBED_CERTS}" \
--kubeconfig=kube-scheduler.kubeconfig

kubectl config set-context default \
--cluster="${CLUSTER_NAME}" \
--user=system:kube-scheduler \
--kubeconfig=kube-scheduler.kubeconfig

kubectl config use-context default --kubeconfig=kube-scheduler.kubeconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash
set -euo pipefail
IFS=$'\n\t'

# Note: It may be a better idea to create 1 key and certificate for all kubelets

CLUSTER_NAME="${1:-ivynetes}"
CERTIFICATE_AUTHORITY_LOCATION="${2:-/var/lib/kubernetes/ca.pem}"
EMBED_CERTS="${3:-true}"
API_SERVER_ENDPOINT="${4}" # Load balancer endpoint
KUBELET_CERTIFICATE_LOCATION="${5:-/var/lib/kubernetes/kubelet.pem}"
KUBELET_KEY_LOCATION="${6:-/var/lib/kubernetes/kubelet-key.pem}"
KUBELET_ID="${7:-kubelet}"

kubectl config set-cluster "${CLUSTER_NAME}" \
--certificate-authority="${CERTIFICATE_AUTHORITY_LOCATION}" \
--embed-certs="${EMBED_CERTS}" \
--server="${API_SERVER_ENDPOINT}" \
--kubeconfig=kubelet.kubeconfig

kubectl config set-credentials "system:node:${KUBELET_ID}" \
--client-certificate="${KUBELET_CERTIFICATE_LOCATION}" \
--client-key="${KUBELET_KEY_LOCATION}" \
--embed-certs="${EMBED_CERTS}" \
--kubeconfig=kubelet.kubeconfig

kubectl config set-context default \
--cluster="${CLUSTER_NAME}" \
--user="system:node:${KUBELET_ID}" \
--kubeconfig=kubelet.kubeconfig

kubectl config use-context default --kubeconfig=kubelet.kubeconfig
7 changes: 7 additions & 0 deletions roles/kubernetes/files/usr/local/bin/etcdctl.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/usr/bin/env bash
if ! [ -r "/etc/etcd/etcdctl.env" ]; then
echo "Unable to read the etcdctl environment file '/etc/etcd/etcdctl.env'. The file must exist, and this wrapper must be run as root."
exit 1
fi
. "/etc/etcd/etcdctl.env"
"/opt/bin/etcdctl" "$@"
6 changes: 6 additions & 0 deletions roles/kubernetes/tasks/Amazon.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
- name: Copying systemd cilium BPFFS mount
copy:
src: "etc/systemd/system/sys-fs-bpf.mount"
dest: "/etc/systemd/system/sys-fs-bpf.mount"
mode: 0755
100 changes: 100 additions & 0 deletions roles/kubernetes/tasks/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
---
# Use specific install tasks for each OS family
- include: "{{ ansible_distribution }}.yml"

- name: install utilities
yum:
name:
- bridge-utils
- btrfs-progs-devel
- lvm2-devel
- nfs-utils
- conntrack-tools
- ipset
state: latest
enablerepo: epel

- name: download kubernetes dependencies
get_url:
url: "{{ item.url }}"
dest: "{{ item.dest }}"
mode: 0755
loop:
- { url: "{{ etcd_url }}", dest: "{{ etcd_dest }}" }
- { url: "{{ cfssl_url }}", dest: "{{ cfssl_dest }}" }
- { url: "{{ cfssljson_url }}", dest: "{{ cfssljson_dest }}" }
- { url: "{{ kubernetes_api_url }}", dest: "{{ kubernetes_api_dest }}" }
- { url: "{{ kubernetes_controller_manager_url }}", dest: "{{ kubernetes_controller_manager_dest }}" }
- { url: "{{ kubernetes_scheduler_url }}", dest: "{{ kubernetes_scheduler_dest }}" }
- { url: "{{ kubernetes_proxy_url }}", dest: "{{ kubernetes_proxy_dest }}" }
- { url: "{{ kubelet_url }}", dest: "{{ kubelet_dest }}" }
- { url: "{{ kubectl_url }}", dest: "{{ kubectl_dest }}" }
- { url: "{{ containerd_url }}", dest: "{{ containerd_dest }}" }
- { url: "{{ cni_plugins_url }}", dest: "{{ cni_plugins_dest }}" }

- name: create kubernetes, etcd, cni and containerd directories
file:
path: "{{ item }}"
state: directory
owner: root
group: root
mode: 0755
recurse: yes
loop:
- /etc/etcd
- /etc/cni/net.d
- /etc/kubernetes/config
- /opt/cni/bin
- /var/lib/etcd
- /var/lib/kubelet
- /var/lib/kube-proxy
- /var/lib/kubernetes
- /var/run/kubernetes
- /opt/ivy/kubernetes

- name: extract files and place it on /tmp
unarchive:
src: "{{ item.src }}"
dest: "{{ item.dest }}/"
remote_src: yes
loop:
- { src: "{{ etcd_dest }}", dest: "{{ tmp_dir }}" }
- { src: "{{ containerd_dest }}", dest: "{{ tmp_dir }}" }
- { src: "{{ cni_plugins_dest }}", dest: "/opt/cni/bin/" }

- name: copy etcd and runc binaries to PATH
copy:
src: "{{ item }}"
dest: "{{ bin_dir }}"
mode: 0755
remote_src: yes
loop:
- "{{ tmp_dir }}/etcd-{{ etcd_version }}-linux-amd64/etcd"
- "{{ tmp_dir }}/etcd-{{ etcd_version }}-linux-amd64/etcdctl"
- "{{ tmp_dir }}/usr/local/sbin/runc"

- name: copy containerd binaries to PATH
copy:
src: "{{ item }}"
dest: "{{ bin_dir }}"
mode: 0755
remote_src: yes
with_fileglob:
- "{{ tmp_dir }}/usr/local/bin/*"

- name: copy shell scripts
copy:
src: "{{ item }}"
dest: "/{{ item }}"
mode: 0755
loop:
- usr/local/bin/etcdctl.sh
- etc/etcd/etcdctl.env

- name: copy kubernetes ivy scripts
copy:
src: "{{ item }}"
dest: "/{{ item }}"
mode: 0755
with_fileglob:
- opt/ivy/kubernetes/*
23 changes: 23 additions & 0 deletions roles/system-base/files/opt/ivy/bash_lib/aws.sh
Original file line number Diff line number Diff line change
Expand Up @@ -318,3 +318,26 @@ function get_secret() {
local VALUE=$(aws secretsmanager --region "${REGION}" get-secret-value --secret-id "${SECRET_ID}" | jq --raw-output .SecretString)
echo ${VALUE}
}

function trust_sysenv_ca() {
local DISTRO="$(grep '^NAME=' /etc/os-release | cut -d '"' -f2)"
local SSM_CA_CERTIFICATE="/$(get_ivy_tag)/$(get_environment)/CA/ca.pem"
local REGION="${1:-$(get_region)}"
case "${DISTRO}" in
Amazon Linux)
local CA_TRUST_DIR='/etc/pki/ca-trust/source/anchors/'
local UPDATE_CA_COMMAND='update-ca-trust extract'
;;
Ubuntu)
local CA_TRUST_DIR='/usr/local/share/ca-certificates/'
local UPDATE_CA_COMMAND='update-ca-certificates'
;;
*)
echo "Only Amazon Linux and Ubuntu are supported at the moment" >&2
return 1
;;
esac
local CA_CRT="${CA_TRUST_DIR}/ivy.pem"
get_ssm_param "${SSM_CA_CERTIFICATE}" "${REGION}" > "${CA_CRT}"
sudo ${UPDATE_CA_COMMAND}
}

0 comments on commit 2a8ddf0

Please sign in to comment.