enable qa-ing of microk8s

.gitignore

@@ -21,6 +21,8 @@ local/vault/vault.env
 *.csr
 /jimmctl
 /jimmsrv
-qa-controller
+qa-lxd
 
 /cloudinit.temp.yaml
+
+local/traefik/certs/ca.srl

docker-compose.yaml

@@ -23,8 +23,6 @@ services:
   jimm:
     image: cosmtrek/air:latest
     profiles: ["dev"]
-    # extra_hosts:
-    # - "host.docker.internal:host-gateway"
     # working_dir value has to be the same of mapped volume
     hostname: jimm.localhost
     working_dir: /jimm

local/README.md

@@ -56,10 +56,10 @@ Note that you can export an environment variable `CONTROLLER_NAME` and re-run st
 controllers that will be controlled by JIMM.
 
 1. `juju unregister jimm-dev`                     - Unregister any other local JIMM you have.
-2. `juju login jimm.localhost -c jimm-dev`        - Login to local JIMM with `Username: jimm-test, Password: password`. (If you name the controller jimm-dev, the script will pick it up!)
+2. `juju login jimm.localhost -c jimm-dev`        - Login to local JIMM with username "jimm-test" password "password"
 3. `./local/jimm/setup-controller.sh`             - Performs controller setup.
 4. `./local/jimm/add-controller.sh`               - A local script to do many of the manual steps for us. See script for more details.
-5. `juju add-model test`                          - Adds a model to qa-controller via JIMM.
+5. `juju add-model test`                          - Adds a model to qa-lxd via JIMM.
 
 # Helpful tidbits!
 > Note: For any secure step to work, ensure you've run the local traefik certs script!

local/jimm/add-controller.sh

@@ -10,10 +10,8 @@
 #
 # Requirements to run this script:
 # - yq (snap)
-set -eux
-
 JIMM_CONTROLLER_NAME="${JIMM_CONTROLLER_NAME:-jimm-dev}"
-CONTROLLER_NAME="${CONTROLLER_NAME:-qa-controller}"
+CONTROLLER_NAME="${CONTROLLER_NAME:-qa-lxd}"
 CONTROLLER_YAML_PATH="${CONTROLLER_NAME}".yaml
 CLIENT_CREDENTIAL_NAME="${CLIENT_CREDENTIAL_NAME:-localhost}"
 

local/jimm/add-microk8s-controller.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+
+# Explanation:
+# JIMM needs to contact the controller and cannot do so from the docker compose to microk8s easily.
+# As such, we turn the controllers default service into a node port service.
+# This allows the service to be access on the hosts network at 30040.
+
+# Next, we have TLS issues as the controller only has limited SANs, one of them being "juju-apiserver"
+# As such, we update jimm's container to map juju-apiserver to "172.17.0.1". This IP address is dockers
+# host network interface address, enabling access to the localhost of the host.
+
+# Finally, we update jimmctls info output attempt to contact the controller on "juju-apiserver"
+# and due to the SAN matching, having a nodeport available and using dockers host network interface,
+# we can contact.
+
+# For routing explanation:
+# JIMM -> jujuapi-server -> 172.17.0.1 -> localhost (of the host) -> localhost:30040 -> NodePort -> Cluster -> Controller
+
+go build ./cmd/jimmctl
+
+# Patch the controller such that it is reachable on the host at 30040
+microk8s.kubectl patch -n controller-qa-microk8s svc/controller-service --type='json' -p '[{"op":"replace","path":"/spec/type","value":"NodePort"},{"op":"replace","path":"/spec/ports/0/nodePort","value":30040}]'
+
+# 172.17.0.1 is dockers host interface, enabling access the host machines host network
+# despite being in a strictly confined docker compose network.
+docker compose exec jimm bash -c "echo '172.17.0.1 juju-apiserver' >> /etc/hosts"
+
+./jimmctl controller-info --local qa-microk8s ./qa-microk8s-controller.yaml
+
+# Update api & public addresses to match /etc/hosts of jimm container
+yq e -i '.api-addresses = ["juju-apiserver:30040"]' ./qa-microk8s-controller.yaml
+yq e -i '.public-address = "juju-apiserver:30040"' ./qa-microk8s-controller.yaml
+
+# Finally add the controller to jimm and add the microk8s credential
+juju switch jimm-dev
+./jimmctl add-controller ./qa-microk8s-controller.yaml
+
+juju update-credentials microk8s --controller jimm-dev
+

local/jimm/setup-controller.sh

@@ -4,15 +4,13 @@
 # It will bootstrap a Juju controller and configure the necessary config to enable the controller
 # to communicate with the docker compose
 
-set -ux
-
 CLOUDINIT_FILE="cloudinit.temp.yaml"
 function finish {
     rm "$CLOUDINIT_FILE"
 }
 trap finish EXIT
 
-CONTROLLER_NAME="${CONTROLLER_NAME:-qa-controller}"
+CONTROLLER_NAME="${CONTROLLER_NAME:-qa-lxd}"
 CLOUDINIT_TEMPLATE=$'cloudinit-userdata: |
   preruncmd:
     - echo "%s    jimm.localhost" >> /etc/hosts
@@ -23,4 +21,4 @@ CLOUDINIT_TEMPLATE=$'cloudinit-userdata: |
 printf "$CLOUDINIT_TEMPLATE" "$(lxc network get lxdbr0 ipv4.address | cut -f1 -d/)" "$(cat local/traefik/certs/ca.crt | sed -e 's/^/        /')" > "${CLOUDINIT_FILE}"
 
 echo "Bootstrapping controller"
-juju bootstrap lxd "${CONTROLLER_NAME}" --config "${CLOUDINIT_FILE}" --config login-token-refresh-url=https://jimm.localhost/.well-known/jwks.json --debug 
+juju bootstrap lxd "${CONTROLLER_NAME}" --config "${CLOUDINIT_FILE}" --config login-token-refresh-url=https://jimm.localhost/.well-known/jwks.json

local/jimm/setup-microk8s-controller.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Host-access has some issues, TLDR to fix it:
+# 1. enable host-access
+# 2. ifconfig 172.16.12.223 (get private address)
+# 3. append line: 
+#   --node-ip=172.16.12.223
+#   to /var/snap/microk8s/current/args/kubelet
+# 4. sudo snap restart microk8s
+juju bootstrap microk8s "qa-microk8s" --config login-token-refresh-url=http://10.0.1.1:17070/.well-known/jwks.json 
+
+
+

qa-lxd.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# QA-lxd
+# This script spins up JIMM (from compose) and sets up a LXD controller and a test model
+# to QA against.
+#
+# It handles the removal of all older resources to ensure a fresh QA env.
+
+
+cleanup() {
+    echo "Destroying qa-lxd controller if exists..."
+    destroy_qa_output=$(juju destroy-controller qa-lxd --force --no-prompt --destroy-all-models 2>&1) || true
+    if [ $? -ne 0 ]; then
+        echo "$destroy_qa_output"
+    fi
+
+    echo "Unregistering jimm-dev controller if exists..."
+    unregister_jimm_output=$(juju unregister jimm-dev --no-prompt 2>&1) || true
+    if [ $? -ne 0 ]; then
+        echo "$unregister_jimm_output"
+    fi
+
+    echo "Tearing down compose..."
+    compose_teardown_output=$(docker compose --profile dev down -v 2>&1) || true
+    if [ $? -ne 0 ]; then
+        echo "$compose_teardown_output"
+    fi
+}
+
+cleanup
+
+echo "*** Starting QA environment setup ***"
+
+docker compose --profile dev up -d
+
+juju login jimm.localhost -c jimm-dev
+
+./local/jimm/setup-controller.sh
+./local/jimm/add-controller.sh
+
+juju add-model test-lxd
+
+# Add a test charm (this is a basic hello-juju, that requires postgres to become healthy)
+# Essentially, a perfect test bed for performing relations etc against.
+juju deploy hello-juju

qa-microk8s.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# QA-microk8s
+# This script spins up JIMM (from compose) and sets up a K8S controller and a test model
+# to QA against.
+#
+# It handles the removal of all older resources to ensure a fresh QA env.
+
+cleanup() {
+    echo "Destroying qa-microk8s controller if exists..."
+    destroy_qa_output=$(juju destroy-controller qa-microk8s --force --no-prompt --destroy-all-models 2>&1) || true
+    if [ $? -ne 0 ]; then
+        echo "$destroy_qa_output"
+    fi
+
+    echo "Unregistering jimm-dev controller if exists..."
+    unregister_jimm_output=$(juju unregister jimm-dev --no-prompt 2>&1) || true
+    if [ $? -ne 0 ]; then
+        echo "$unregister_jimm_output"
+    fi
+
+    echo "Tearing down compose..."
+    compose_teardown_output=$(docker compose --profile dev down -v 2>&1) || true
+    if [ $? -ne 0 ]; then
+        echo "$compose_teardown_output"
+    fi
+}
+
+cleanup
+
+docker compose --profile dev up -d
+
+juju login jimm.localhost -c jimm-dev
+
+./local/jimm/setup-microk8s-controller.sh
+./local/jimm/add-microk8s-controller.sh
+
+# Add a test model
+juju add-model test microk8s
+