Support Fedora & Support HA mode with embedded DB

This enables initializing a cluster in HA mode with an embedded DB.
https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/

When multiple masters are specified in the master group, k3s-ansible will add
the necessary flags during the initialization phase.
(i.e. --cluster-init and --server)

For the embedded HA mode to work the k3s version must be >= v1.19.1

This change also adds support for Fedora >= 31

Signed-off-by: Julien DOCHE <[email protected]>

README.md

@@ -9,6 +9,7 @@ Build a Kubernetes cluster using Ansible with k3s. The goal is easily install a
 - [X] Debian
 - [X] Ubuntu
 - [X] CentOS
+- [X] Fedora >= 31
 
 on processor architecture:
 
@@ -26,6 +27,7 @@ Master and nodes must have passwordless SSH access
 First create a new directory based on the `sample` directory within the `inventory` directory:
 
 ```bash
+pip install -r requirements.txt
 cp -R inventory/sample inventory/my-cluster
 ```
 
@@ -43,6 +45,10 @@ master
 node
 ```
 
+If multiple hosts are in the master group, the playbook will automatically setup k3s in HA mode with etcd.
+https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/
+This requires at least k3s version 1.19.1
+
 If needed, you can also edit `inventory/my-cluster/group_vars/all.yml` to match your environment.
 
 Start provisioning of the cluster using the following command:

requirements.txt

@@ -0,0 +1 @@
+jmespath

roles/k3s/master/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+ansible_user: root
+server_init_args: >-
+  {% if groups['master'] | length > 1 %}
+    {% if ansible_host == groups['master'][0] %}
+      --cluster-init
+    {% else %}
+      --server https://{{ groups['master'][0] }}:6443
+    {% endif %}
+  {% endif %}
+  {{ extra_server_args | default('') }}

roles/k3s/master/tasks/main.yml

@@ -1,4 +1,35 @@
 ---
+- name: Clean previous runs of k3s-init
+  systemd:
+    name: k3s-init
+    state: stopped
+  failed_when: false
+
+- name: Clean previous runs of k3s-init
+  command: systemctl reset-failed k3s-init
+  failed_when: false
+  changed_when: false
+
+- name: Init cluster inside the transient k3s-init service
+  command:
+    cmd: "systemd-run -p RestartSec=2 -p Restart=on-failure -E K3S_TOKEN={{ lookup('password') }} -u k3s-init k3s server {{ server_init_args }}"
+    creates: /var/lib/rancher/k3s/server
+
+- name: Verification
+  block:
+    - name: Verify that all nodes actually joined
+      command: k3s kubectl get --raw /api/v1/nodes/
+      register: nodes
+      until: nodes.rc == 0 and
+        ((nodes.stdout | from_json)['items'] | json_query('[*].metadata.labels."node-role.kubernetes.io/master"') | count) == (groups['master'] | length)
+      retries: 20
+      delay: 10
+  always:
+    - name: Kill the temporary service used for initialization
+      systemd:
+        name: k3s-init
+        state: stopped
+      failed_when: false
 
 - name: Copy K3s service file
   register: k3s_service

roles/prereq/handlers/main.yml

@@ -0,0 +1,8 @@
+---
+- name: reboot machine
+  reboot:
+
+- name: restart firewalld
+  service:
+    name: firewalld.service
+    state: restarted

roles/prereq/tasks/main.yml

@@ -1,8 +1,49 @@
 ---
+
+- name: permit traffic in default zone for kube-apiserver service
+  firewalld:
+    service: "{{ item }}"
+    permanent: yes
+    state: enabled
+  with_items:
+    - etcd-server
+    - etcd-client
+    - kube-apiserver
+
 - name: Set SELinux to disabled state
   selinux:
     state: disabled
-  when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux']
+  when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux', 'Fedora']
+  notify: reboot machine
+
+- name: Change the default firewalld backend
+  lineinfile:
+    path: /etc/firewalld/firewalld.conf
+    line: FirewallBackend=iptables
+    regexp: FirewallBackend=
+  when: ansible_distribution == "Fedora" and ansible_distribution_major_version >= '31'
+  notify: restart firewalld
+
+- name: Install grubby
+  dnf:
+    name: grubby
+    state: present
+  when: ansible_distribution == "Fedora" and ansible_distribution_major_version >= '31'
+
+- name: Check if systemd.unified_cgroup_hierarchy=0 is set in kernelopts
+  command: grep -Eq '^kernelopts=.* systemd\.unified_cgroup_hierarchy=0' /boot/grub2/grubenv
+  register: grubenv_has_cgroup_exception
+  check_mode: no
+  ignore_errors: yes
+  changed_when: no
+  when: ansible_distribution == "Fedora" and ansible_distribution_major_version >= '31'
+
+- name: Configure systemd.unified_cgroup_hierarchy=0 in kernelopts
+  command: grubby --update-kernel=ALL --args="systemd.unified_cgroup_hierarchy=0"
+  when:
+    - ansible_distribution == "Fedora" and ansible_distribution_major_version >= '31'
+    - grubenv_has_cgroup_exception.rc != 0
+  notify: reboot machine
 
 - name: Enable IPv4 forwarding
   sysctl:
@@ -23,21 +64,21 @@
     content: "br_netfilter"
     dest: /etc/modules-load.d/br_netfilter.conf
     mode: "u=rw,g=,o="
-  when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux']
+  when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux', 'Fedora']
 
 - name: Load br_netfilter
   modprobe:
     name: br_netfilter
     state: present
-  when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux']
+  when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux', 'Fedora']
 
 - name: Set bridge-nf-call-iptables (just to be sure)
   sysctl:
     name: "{{ item }}"
     value: "1"
     state: present
     reload: yes
-  when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux']
+  when: ansible_distribution in ['CentOS', 'Red Hat Enterprise Linux', 'Fedora']
   loop:
     - net.bridge.bridge-nf-call-iptables
     - net.bridge.bridge-nf-call-ip6tables

roles/reset/tasks/main.yml

@@ -8,6 +8,7 @@
   with_items:
     - k3s
     - k3s-node
+    - k3s-init
 
 - name: pkill -9 -f "k3s/data/[^/]+/bin/containerd-shim-runc"
   register: pkill_containerd_shim_runc
@@ -30,12 +31,12 @@
     name: "{{ item }}"
     state: absent
   with_items:
+    - /usr/local/bin/k3s
     - "{{ systemd_dir }}/k3s.service"
     - "{{ systemd_dir }}/k3s-node.service"
     - /etc/rancher/k3s
     - /var/lib/rancher/k3s
     - /var/lib/kubelet
-    - /usr/local/bin/k3s
 
 - name: daemon_reload
   systemd: