diff --git a/.gitignore b/.gitignore index 8e4046690..18d156d4a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ local/vault/vault.env *.csr /jimmctl /jimmsrv -qa-controller +qa-lxd /cloudinit.temp.yaml + +local/traefik/certs/ca.srl diff --git a/Makefile b/Makefile index 4e8e5c91e..c81e42d4f 100644 --- a/Makefile +++ b/Makefile @@ -115,7 +115,9 @@ sysdeps: ifeq ($(APT_BASED),0) @$(call check_dep,go,Missing Go - install from https://go.dev/doc/install or 'sudo snap install go') @$(call check_dep,git,Missing Git - install with 'sudo apt install git') - @$(call check_dep,gcc,Missing gcc - install with 'sudo apt install build-essential') + @$(call check_dep,gcc,Missing gcc - install with 'sudo apt install build-essentials') + @$(call check_dep,yq,Missing yq - install with 'sudo snap install yq') + @$(call check_dep,gcc,Missing microk8s - install latest none-classic from snapstore') @$(call check_dep,docker,Missing Docker - install from https://docs.docker.com/engine/install/') @$(call check_dep,docker-compose,Missing Docker Compose - install from https://docs.docker.com/engine/install/') @$(call check_dep,juju-db.mongo,Missing juju-db - install with 'sudo snap install juju-db --channel=4.4/stable') diff --git a/docker-compose.yaml b/docker-compose.yaml index d9fa3bdda..e04bcd0f2 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -23,8 +23,6 @@ services: jimm: image: cosmtrek/air:latest profiles: ["dev"] - # extra_hosts: - # - "host.docker.internal:host-gateway" # working_dir value has to be the same of mapped volume hostname: jimm.localhost working_dir: /jimm diff --git a/local/README.md b/local/README.md index c75ad608e..c97ffca6e 100644 --- a/local/README.md +++ b/local/README.md @@ -50,16 +50,26 @@ The `request name` represents the literal WS endpoint, i.e., `API = /api`. 2. Install Juju: `sudo snap install juju --channel=3.5/stable` (minimum Juju version is `3.5`). 3. Install JQ: `sudo snap install jq`. -## Controller set up +## All-In-One scripts +We have two all-in-one scripts, namely: +- qa-lxd.sh +- qa-microk8s.sh +These scripts respectively spin up jimm in compose, setup controllers in the targeted environment +and handle connectivity. Finally, adding a test model to Q/A against. + +Please ensure you've run "make dev-env-setup" first though! + +## Manual +### Controller set up Note that you can export an environment variable `CONTROLLER_NAME` and re-run steps 3. and 4. below to create multiple Juju controllers that will be controlled by JIMM. 1. `juju unregister jimm-dev` - Unregister any other local JIMM you have. -2. `juju login jimm.localhost -c jimm-dev` - Login to local JIMM with `Username: jimm-test, Password: password`. (If you name the controller jimm-dev, the script will pick it up!) +2. `juju login jimm.localhost -c jimm-dev` - Login to local JIMM with username "jimm-test" password "password" 3. `./local/jimm/setup-controller.sh` - Performs controller setup. 4. `./local/jimm/add-controller.sh` - A local script to do many of the manual steps for us. See script for more details. -5. `juju add-model test` - Adds a model to qa-controller via JIMM. +5. `juju add-model test` - Adds a model to qa-lxd via JIMM. # Helpful tidbits! > Note: For any secure step to work, ensure you've run the local traefik certs script! diff --git a/local/jimm/add-controller.sh b/local/jimm/add-controller.sh index a93be2651..152d4a169 100755 --- a/local/jimm/add-controller.sh +++ b/local/jimm/add-controller.sh @@ -10,10 +10,8 @@ # # Requirements to run this script: # - yq (snap) -set -eux - JIMM_CONTROLLER_NAME="${JIMM_CONTROLLER_NAME:-jimm-dev}" -CONTROLLER_NAME="${CONTROLLER_NAME:-qa-controller}" +CONTROLLER_NAME="${CONTROLLER_NAME:-qa-lxd}" CONTROLLER_YAML_PATH="${CONTROLLER_NAME}".yaml CLIENT_CREDENTIAL_NAME="${CLIENT_CREDENTIAL_NAME:-localhost}" diff --git a/local/jimm/add-microk8s-controller.sh b/local/jimm/add-microk8s-controller.sh new file mode 100755 index 000000000..4f3ff98a8 --- /dev/null +++ b/local/jimm/add-microk8s-controller.sh @@ -0,0 +1,40 @@ +#!/bin/bash + + +# Explanation: +# JIMM needs to contact the controller and cannot do so from the docker compose to microk8s easily. +# As such, we turn the controllers default service into a node port service. +# This allows the service to be access on the hosts network at 30040. + +# Next, we have TLS issues as the controller only has limited SANs, one of them being "juju-apiserver" +# As such, we update jimm's container to map juju-apiserver to "172.17.0.1". This IP address is dockers +# host network interface address, enabling access to the localhost of the host. + +# Finally, we update jimmctls info output attempt to contact the controller on "juju-apiserver" +# and due to the SAN matching, having a nodeport available and using dockers host network interface, +# we can contact. + +# For routing explanation: +# JIMM -> jujuapi-server -> 172.17.0.1 -> localhost (of the host) -> localhost:30040 -> NodePort -> Cluster -> Controller + +go build ./cmd/jimmctl + +# Patch the controller such that it is reachable on the host at 30040 +microk8s.kubectl patch -n controller-qa-microk8s svc/controller-service --type='json' -p '[{"op":"replace","path":"/spec/type","value":"NodePort"},{"op":"replace","path":"/spec/ports/0/nodePort","value":30040}]' + +# 172.17.0.1 is dockers host interface, enabling access the host machines host network +# despite being in a strictly confined docker compose network. +docker compose exec jimm bash -c "echo '172.17.0.1 juju-apiserver' >> /etc/hosts" + +./jimmctl controller-info --local qa-microk8s ./qa-microk8s-controller.yaml + +# Update api & public addresses to match /etc/hosts of jimm container +yq e -i '.api-addresses = ["juju-apiserver:30040"]' ./qa-microk8s-controller.yaml +yq e -i '.public-address = "juju-apiserver:30040"' ./qa-microk8s-controller.yaml + +# Finally add the controller to jimm and add the microk8s credential +juju switch jimm-dev +./jimmctl add-controller ./qa-microk8s-controller.yaml + +juju update-credentials microk8s --controller jimm-dev + diff --git a/local/jimm/qa-lxd.sh b/local/jimm/qa-lxd.sh new file mode 100755 index 000000000..61e4cd2f0 --- /dev/null +++ b/local/jimm/qa-lxd.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# QA-lxd +# This script spins up JIMM (from compose) and sets up a LXD controller and a test model +# to QA against. +# +# It handles the removal of all older resources to ensure a fresh QA env. +# +# Please make sure you've run make "make dev-env-setup" for this script to work. + + +cleanup() { + echo "Destroying qa-lxd controller if exists..." + destroy_qa_output=$(juju destroy-controller qa-lxd --force --no-prompt --destroy-all-models 2>&1) || true + if [ $? -ne 0 ]; then + echo "$destroy_qa_output" + fi + + echo "Unregistering jimm-dev controller if exists..." + unregister_jimm_output=$(juju unregister jimm-dev --no-prompt 2>&1) || true + if [ $? -ne 0 ]; then + echo "$unregister_jimm_output" + fi + + echo "Tearing down compose..." + compose_teardown_output=$(docker compose --project-directory ../../ --file ../../docker-compose.yaml --profile dev down -v 2>&1) || true + if [ $? -ne 0 ]; then + echo "$compose_teardown_output" + fi +} + +cleanup + +echo "*** Starting QA environment setup ***" + +docker compose --project-directory ../../ --file ../../docker-compose.yaml --profile dev up -d + +juju login jimm.localhost -c jimm-dev + +./local/jimm/setup-controller.sh +./local/jimm/add-controller.sh + +juju add-model test-lxd + +# Add a test charm (this is a basic hello-juju, that requires postgres to become healthy) +# Essentially, a perfect test bed for performing relations etc against. +juju deploy hello-juju diff --git a/local/jimm/qa-microk8s.sh b/local/jimm/qa-microk8s.sh new file mode 100755 index 000000000..4a47afd3e --- /dev/null +++ b/local/jimm/qa-microk8s.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# QA-microk8s +# This script spins up JIMM (from compose) and sets up a K8S controller and a test model +# to QA against. +# +# It handles the removal of all older resources to ensure a fresh QA env. +# +# Please make sure you've run make "make dev-env-setup" for this script to work. + +cleanup() { + echo "Destroying qa-microk8s controller if exists..." + destroy_qa_output=$(juju destroy-controller qa-microk8s --force --no-prompt --destroy-all-models 2>&1) || true + if [ $? -ne 0 ]; then + echo "$destroy_qa_output" + fi + + echo "Unregistering jimm-dev controller if exists..." + unregister_jimm_output=$(juju unregister jimm-dev --no-prompt 2>&1) || true + if [ $? -ne 0 ]; then + echo "$unregister_jimm_output" + fi + + echo "Tearing down compose..." + compose_teardown_output=$(docker compose --project-directory ../../ --file ../../docker-compose.yaml --profile dev down -v 2>&1) || true + if [ $? -ne 0 ]; then + echo "$compose_teardown_output" + fi +} + +cleanup + +docker compose --project-directory ../../ --file ../../docker-compose.yaml --profile dev up -d + +juju login jimm.localhost -c jimm-dev + +./setup-microk8s-controller.sh +./add-microk8s-controller.sh + +# Add a test model +juju add-model test microk8s + diff --git a/local/jimm/setup-controller.sh b/local/jimm/setup-controller.sh index 8bb982336..4c1a7a181 100755 --- a/local/jimm/setup-controller.sh +++ b/local/jimm/setup-controller.sh @@ -4,15 +4,13 @@ # It will bootstrap a Juju controller and configure the necessary config to enable the controller # to communicate with the docker compose -set -ux - CLOUDINIT_FILE="cloudinit.temp.yaml" function finish { rm "$CLOUDINIT_FILE" } trap finish EXIT -CONTROLLER_NAME="${CONTROLLER_NAME:-qa-controller}" +CONTROLLER_NAME="${CONTROLLER_NAME:-qa-lxd}" CLOUDINIT_TEMPLATE=$'cloudinit-userdata: | preruncmd: - echo "%s jimm.localhost" >> /etc/hosts @@ -23,4 +21,4 @@ CLOUDINIT_TEMPLATE=$'cloudinit-userdata: | printf "$CLOUDINIT_TEMPLATE" "$(lxc network get lxdbr0 ipv4.address | cut -f1 -d/)" "$(cat local/traefik/certs/ca.crt | sed -e 's/^/ /')" > "${CLOUDINIT_FILE}" echo "Bootstrapping controller" -juju bootstrap lxd "${CONTROLLER_NAME}" --config "${CLOUDINIT_FILE}" --config login-token-refresh-url=https://jimm.localhost/.well-known/jwks.json --debug +juju bootstrap lxd "${CONTROLLER_NAME}" --config "${CLOUDINIT_FILE}" --config login-token-refresh-url=https://jimm.localhost/.well-known/jwks.json diff --git a/local/jimm/setup-microk8s-controller.sh b/local/jimm/setup-microk8s-controller.sh new file mode 100755 index 000000000..d742ad0a1 --- /dev/null +++ b/local/jimm/setup-microk8s-controller.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Host-access has some issues, TLDR to fix it: +# 1. enable host-access +# 2. ifconfig 172.16.12.223 (get private address) +# 3. append line: +# --node-ip=172.16.12.223 +# to /var/snap/microk8s/current/args/kubelet +# 4. sudo snap restart microk8s +juju bootstrap microk8s "qa-microk8s" --config login-token-refresh-url=http://10.0.1.1:17070/.well-known/jwks.json +