Merge pull request #126 from lablabs/feat/node_taints

Feat/node taints
lablabs · Dec 29, 2022 · ff23dfd · ff23dfd
2 parents dd6eeed + 3a46637
commit ff23dfd
Show file tree

Hide file tree

Showing 20 changed files with 177 additions and 142 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,91 @@
+name: molecule test
+
+on:
+    pull_request:
+        branches:
+            - main
+        types: [opened, synchronize, reopened]
+        paths:
+          - 'defaults/**'
+          - 'handlers/**'
+          - 'molecule/**'
+          - 'tasks/**'
+          - 'templates/**'
+          - 'vars/**'
+    schedule:
+        - cron: '0 1 1 * *'
+    workflow_dispatch:
+
+jobs:
+  standalone:
+    name: Single node setup
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        config:
+          - image: "rockylinux8"
+            tag: "latest"
+            cgroup_mode: "private"
+          # - image: "ubuntu2004"
+          #   tag: "latest"
+          #   cgroup_mode: "private"
+          - image: "ubuntu2204"
+            tag: "latest"
+            cgroup_mode: "private"
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          path: 'lablabs.rke2'
+
+      - name: Set up Python 3
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install test dependencies
+        run: pip3 install ansible ansible-lint docker molecule[docker] yamllint
+
+      - name: Run Molecule tests
+        run: molecule test
+        working-directory: ./lablabs.rke2
+        env:
+          PY_COLORS: '1'
+          ANSIBLE_FORCE_COLOR: '1'
+          MOLECULE_IMAGE: ${{ matrix.config.image }}
+          MOLECULE_TAG: ${{ matrix.config.tag }}
+          MOLECULE_DOCKER_CGROUPS_MODE: ${{ matrix.config.cgroup_mode }}
+
+  cluster:
+    name: Cluster setup
+    runs-on: ubuntu-20.04
+    needs: standalone
+    strategy:
+      matrix:
+        config:
+          - image: "ubuntu2204"
+            tag: "latest"
+            cgroup_mode: "private"
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          path: 'lablabs.rke2'
+
+      - name: Set up Python 3
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install test dependencies
+        run: pip3 install ansible ansible-lint docker molecule[docker] yamllint
+
+      - name: Run Molecule tests
+        run: molecule test
+        working-directory: ./lablabs.rke2
+        env:
+          PY_COLORS: '1'
+          ANSIBLE_FORCE_COLOR: '1'
+          MOLECULE_IMAGE: ${{ matrix.config.image }}
+          MOLECULE_TAG: ${{ matrix.config.tag }}
+          MOLECULE_DOCKER_CGROUPS_MODE: ${{ matrix.config.cgroup_mode }}
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -12,7 +12,7 @@ jobs:
       fail-fast: true
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           path: "${{ github.repository }}"
       - name: Molecule for Ansible - lint

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml
@@ -9,7 +9,7 @@ jobs:
   pre-commit:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - uses: actions/setup-python@v2
 
     - name: Run pre-commit

diff --git a/README.md b/README.md
@@ -39,9 +39,6 @@ This is a copy of `defaults/main.yml`
 
 ```yaml
 ---
-# Default nodetaints
-node_taints: []
-
 # The node type - server or agent
 rke2_type: server
 
@@ -82,8 +79,12 @@ rke2_additional_sans: []
 # API Server destination port
 rke2_apiserver_dest_port: 6443
 
-# If false, server node(s) will be schedulable and thus your workloads can get launched on them
-rke2_server_taint: false
+# Server nodes taints
+rke2_server_node_taints: []
+  # - 'CriticalAddonsOnly=true:NoExecute'
+
+# Agent nodes taints
+rke2_agent_node_taints: []
 
 # Pre-shared secret token that other server or agent nodes will register with when connecting to the cluster
 rke2_token: defaultSecret12345
@@ -309,17 +310,18 @@ This playbook will deploy RKE2 to a cluster with one server(master) and several
 
 ```
 
-This playbook will deploy RKE2 to a cluster with HA server(master) control-plane and several agent(worker) nodes. The server(master) nodes will be tainted so the workload will be distributed only on worker/agent nodes. The role will install also keepalived on the control-plane nodes and setup VIP address where the Kubernetes API will be reachable. it will also download the Kubernetes config file to the local machine.
+This playbook will deploy RKE2 to a cluster with HA server(master) control-plane and several agent(worker) nodes. The server(master) nodes will be tainted so the workload will be distributed only on worker(agent) nodes. The role will install also keepalived on the control-plane nodes and setup VIP address where the Kubernetes API will be reachable. it will also download the Kubernetes config file to the local machine.
 
 ```yaml
 - name: Deploy RKE2
   hosts: all
   become: yes
   vars:
     rke2_ha_mode: true
-    rke2_server_taint: true
     rke2_api_ip : 192.168.123.100
     rke2_download_kubeconf: true
+    rke2_server_node_taints:
+      - 'CriticalAddonsOnly=true:NoExecute'
   roles:
      - role: lablabs.rke2
 
@@ -333,7 +335,6 @@ If the playbook starts to hang at the `Start RKE2 service on the rest of the nod
 
 Please check the required *Inbound Rules for RKE2 Server Nodes* at the following link: <https://docs.rke2.io/install/requirements/#networking>.
 
-
 ## License
 
 MIT

diff --git a/defaults/main.yml b/defaults/main.yml
@@ -1,7 +1,4 @@
 ---
-# Default nodetaints
-node_taints: []
-
 # The node type - server or agent
 rke2_type: server
 
@@ -42,8 +39,12 @@ rke2_additional_sans: []
 # API Server destination port
 rke2_apiserver_dest_port: 6443
 
-# If false, server node(s) will be schedulable and thus your workloads can get launched on them
-rke2_server_taint: false
+# Server nodes taints
+rke2_server_node_taints: []
+  # - 'CriticalAddonsOnly=true:NoExecute'
+
+# Agent nodes taints
+rke2_agent_node_taints: []
 
 # Pre-shared secret token that other server or agent nodes will register with when connecting to the cluster
 rke2_token: defaultSecret12345

diff --git a/handlers/main.yml b/handlers/main.yml
@@ -1,11 +1,11 @@
 ---
 
-- name: restart keepalived
+- name: Restart keepalived
   ansible.builtin.service:
     name: keepalived
     state: restarted
 
-- name: restart systemd-sysctl
+- name: Restart systemd-sysctl
   ansible.builtin.service:
     state: restarted
     name: systemd-sysctl

diff --git a/molecule/cluster/converge.yml b/molecule/cluster/converge.yml
@@ -3,7 +3,9 @@
   hosts: all
   become: yes
   vars:
-    rke2_version: v1.20.7+rke2r2
+    rke2_version: v1.20.15+rke2r1
     rke2_snapshooter: native
+    rke2_server_node_taints:
+      - 'CriticalAddonsOnly=true:NoExecute'
   roles:
-     - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
+    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
diff --git a/molecule/cluster/molecule.yml b/molecule/cluster/molecule.yml
@@ -9,24 +9,26 @@ lint: |
   ansible-lint --exclude molecule/
 platforms:
   - name: node1
-    image: "geerlingguy/docker-${image:-ubuntu2004}-ansible:${tag:-latest}"
+    image: "geerlingguy/docker-${MOLECULE_IMAGE:-ubuntu2204}-ansible:${MOLECULE_TAG:-latest}"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+      - "/sys/fs/cgroup:/sys/fs/cgroup:${MOLECULE_DOCKER_VOLUMES:-ro}" # Use "ro" for cgroup v1 and "rw" for cgroup v2
+    cgroups_mode: ${MOLECULE_DOCKER_CGROUPS_MODE:-"private"} # Use "private" for cgroup v1 and "host" for cgroup v2
     privileged: true
-    pre_build_image: ${MOLECULE_PREBUILT:-true}
+    pre_build_image: true
     networks:
       - name: rke2net
     groups:
       - masters
       - k8s_cluster
   - name: node2
-    image: "geerlingguy/docker-${image:-ubuntu2004}-ansible:${tag:-latest}"
+    image: "geerlingguy/docker-${MOLECULE_IMAGE:-ubuntu2204}-ansible:${MOLECULE_TAG:-latest}"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+      - "/sys/fs/cgroup:/sys/fs/cgroup:${MOLECULE_DOCKER_VOLUMES:-ro}" # Use "ro" for cgroup v1 and "rw" for cgroup v2
+    cgroups_mode: ${MOLECULE_DOCKER_CGROUPS_MODE:-"private"} # Use "private" for cgroup v1 and "host" for cgroup v2
     privileged: true
-    pre_build_image: ${MOLECULE_PREBUILT:-true}
+    pre_build_image: true
     networks:
       - name: rke2net
     groups:

diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml
@@ -3,7 +3,7 @@
   hosts: all
   become: yes
   vars:
-    rke2_version: v1.20.7+rke2r2
+    rke2_version: v1.20.15+rke2r1
     rke2_snapshooter: native
   roles:
-     - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
+    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml
@@ -9,12 +9,13 @@ lint: |
   ansible-lint --exclude molecule/
 platforms:
   - name: node1
-    image: "geerlingguy/docker-${image:-rockylinux8}-ansible:${tag:-latest}"
+    image: "geerlingguy/docker-${MOLECULE_IMAGE:-ubuntu2204}-ansible:${MOLECULE_TAG:-latest}"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+      - "/sys/fs/cgroup:/sys/fs/cgroup:${MOLECULE_DOCKER_VOLUMES:-ro}" # Use "ro" for cgroup v1 and "rw" for cgroup v2
+    cgroups_mode: ${MOLECULE_DOCKER_CGROUPS_MODE:-"private"} # Use "private" for cgroup v1 and "host" for cgroup v2
     privileged: true
-    pre_build_image: ${MOLECULE_PREBUILT:-true}
+    pre_build_image: true
     networks:
       - name: rke2net
     groups:

diff --git a/molecule/ha_cluster/converge.yml b/molecule/ha_cluster/converge.yml
@@ -6,8 +6,9 @@
     rke2_version: v1.25.3+rke2r1
     rke2_cis_profile: cis-1.23
     rke2_ha_mode: true
-    rke2_server_taint: true
     rke2_api_ip: 192.168.123.100
     rke2_snapshooter: native
+    rke2_server_node_taints:
+      - 'CriticalAddonsOnly=true:NoExecute'
   roles:
-     - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
+    - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"