✨(project) add k3d utils for tray development

Ralph should be self-contained, and helpers are required for Kubernetes-related developments. We've added such helpers to: 1. run a local k3d cluster 2. run arnold commands
bmtcril · Oct 18, 2022 · 1cc1ccc · 1cc1ccc
1 parent 40f386f
commit 1cc1ccc
Show file tree

Hide file tree

Showing 5 changed files with 325 additions and 2 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -48,3 +48,4 @@ data
 .team
 group_vars
 src/tray
+k3d-storage
diff --git a/.gitignore b/.gitignore
@@ -54,9 +54,12 @@ data/
 site/
 
 # Arnold environment
+bin/arnold
+bin/init-cluster
 bin/vault_password
 .team
 group_vars/
+k3d-storage/
 
 # hypothesis
 .hypothesis/
diff --git a/.k3d-cluster.env.sh b/.k3d-cluster.env.sh
@@ -0,0 +1,18 @@
+# k3d cluster environment for development purpose
+
+echo -n "Loading k3d cluster environment... "
+
+export ARNOLD_DEFAULT_VAULT_PASSWORD=arnold
+export ANSIBLE_VAULT_PASSWORD="${ARNOLD_DEFAULT_VAULT_PASSWORD}"
+export ARNOLD_IMAGE_TAG=master
+export K3D_BIND_HOST_PORT_HTTP=80
+export K3D_BIND_HOST_PORT_HTTPS=443
+export K3D_CLUSTER_NAME=ralph
+export K3D_ENABLE_REGISTRY=1
+export K3D_REGISTRY_HOST=registry.127.0.0.1.nip.io
+export K3D_REGISTRY_NAME=k3d-registry.127.0.0.1.nip.io
+export K3D_REGISTRY_PORT=5000
+export K8S_DOMAIN=$(hostname -I | awk '{print $1}')
+export MINIMUM_AVAILABLE_RWX_VOLUME=3
+
+echo "done."
diff --git a/Makefile b/Makefile
@@ -1,3 +1,6 @@
+# -- General
+SHELL := /bin/bash
+
 # -- Docker
 # Get the current user ID to use for docker run and docker exec commands
 DOCKER_UID           = $(shell id -u)
@@ -16,22 +19,85 @@ ES_PORT     = 9200
 ES_INDEX    = statements
 ES_URL      = $(ES_PROTOCOL)://$(ES_HOST):$(ES_PORT)
 
+# -- Arnold
+ARNOLD             = ARNOLD_IMAGE_TAG=6.13.0 bin/arnold
+ARNOLD_CUSTOMER    = ralph
+ARNOLD_ENVIRONMENT = development
+ARNOLD_APP         = ralph
+ARNOLD_APP_VARS    = group_vars/customer/$(ARNOLD_CUSTOMER)/$(ARNOLD_ENVIRONMENT)/main.yml
+
+# -- RALPH
+RALPH_IMAGE_NAME ?= ralph
+RALPH_IMAGE_TAG  ?= development
+
+# -- K3D
+K3D_CLUSTER_NAME              ?= ralph
+K3D_REGISTRY_HOST             ?= registry.127.0.0.1.nip.io
+K3D_REGISTRY_NAME             ?= k3d-registry.127.0.0.1.nip.io
+K3D_REGISTRY_PORT             ?= 5000
+K3D_REGISTRY_RALPH_IMAGE_NAME  = $(K3D_REGISTRY_NAME):$(K3D_REGISTRY_PORT)/$(ARNOLD_ENVIRONMENT)-$(ARNOLD_APP)/$(RALPH_IMAGE_NAME)
+
 # ==============================================================================
 # RULES
 
 default: help
 
+bin/arnold:
+	curl -Lo "bin/arnold" "https://raw.githubusercontent.com/openfun/arnold/master/bin/arnold"
+	chmod +x bin/arnold
+
+bin/init-cluster:
+	curl -Lo "bin/init-cluster" "https://raw.githubusercontent.com/openfun/arnold/master/bin/init-cluster"
+	chmod +x bin/init-cluster
+
 .env:
 	cp .env.dist .env
 
 # -- Docker/compose
+arnold-bootstrap: ## bootstrap arnold's project
+arnold-bootstrap: \
+	bin/arnold
+	$(ARNOLD) -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) setup
+	$(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -a $(ARNOLD_APP) create_app_vaults
+	$(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -a elasticsearch create_app_vaults
+	$(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -- vault -a $(ARNOLD_APP) decrypt
+	sed -i 's/^# RALPH_BACKENDS__DATABASE__ES/RALPH_BACKENDS__DATABASE__ES/g' group_vars/customer/$(ARNOLD_CUSTOMER)/$(ARNOLD_ENVIRONMENT)/secrets/$(ARNOLD_APP).vault.yml
+	$(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -- vault -a $(ARNOLD_APP) encrypt
+	echo "skip_verification: True" > $(ARNOLD_APP_VARS)
+	echo "apps:" >> $(ARNOLD_APP_VARS)
+	echo "  - name: elasticsearch" >> $(ARNOLD_APP_VARS)
+	echo "  - name: $(ARNOLD_APP)" >> $(ARNOLD_APP_VARS)
+	echo "ralph_image_name: $(K3D_REGISTRY_RALPH_IMAGE_NAME)" >> $(ARNOLD_APP_VARS)
+	echo "ralph_image_tag: $(RALPH_IMAGE_TAG)" >> $(ARNOLD_APP_VARS)
+	echo "ralph_app_replicas: 1" >> $(ARNOLD_APP_VARS)
+	echo "ralph_cronjobs:" >> $(ARNOLD_APP_VARS)
+	echo "  - name: $(ARNOLD_ENVIRONMENT)-test" >> $(ARNOLD_APP_VARS)
+	echo "    schedule: '* * * * *'" >> $(ARNOLD_APP_VARS)
+	echo "    command: ['date']" >> $(ARNOLD_APP_VARS)
+.PHONY: arnold-bootstrap
+
+arnold-deploy: ## deploy Ralph to k3d using Arnold
+	source .k3d-cluster.env.sh && \
+		$(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -a $(ARNOLD_APP) deploy && \
+		$(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -a $(ARNOLD_APP) switch
+.PHONY: arnold-deploy
+
+arnold-init: ## initialize Ralph k3d project using Arnold
+arnold-init:
+	source .k3d-cluster.env.sh && \
+		$(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -a elasticsearch,ralph init && \
+		$(ARNOLD) -d -c $(ARNOLD_CUSTOMER) -e $(ARNOLD_ENVIRONMENT) -a elasticsearch deploy && \
+		kubectl exec svc/elasticsearch -- curl -s -X PUT "localhost:9200/statements?pretty"
+.PHONY: arnold-deploy
+
 bootstrap: ## bootstrap the project for development
 bootstrap: \
   .env \
   build \
   dev \
   es-index
 .PHONY: bootstrap
+
 build: ## build the app container
 	@$(COMPOSE) build app
 .PHONY: build
@@ -61,12 +127,34 @@ down: ## stop and remove backend containers
 	@$(COMPOSE) down
 .PHONY: down
 
-es-index:  ## create elasticsearch index and sample documents
+es-index: ## create elasticsearch index and sample documents
 es-index: run-es
 	@echo "Creating $(ES_INDEX) index"
 	bin/es index $(ES_INDEX)
 .PHONY: es-index
 
+k3d-cluster: ## boot a k3d cluster for k8s-related development
+k3d-cluster: \
+	bin/init-cluster
+	source .k3d-cluster.env.sh && \
+		bin/init-cluster "$(K3D_CLUSTER_NAME)"
+.PHONY: k3d-cluster
+
+k3d-push: ## push build image to local k3d docker registry
+k3d-push: build
+	source .k3d-cluster.env.sh && \
+		docker tag \
+			$(RALPH_IMAGE_NAME):$(RALPH_IMAGE_TAG) \
+			"$(K3D_REGISTRY_RALPH_IMAGE_NAME):$(RALPH_IMAGE_TAG)" && \
+		docker push \
+			"$(K3D_REGISTRY_RALPH_IMAGE_NAME):$(RALPH_IMAGE_TAG)"
+.PHONY: k3d-push
+
+k3d-stop: ## stop local k8s cluster
+	source .k3d-cluster.env.sh && \
+		k3d cluster stop "$(K3D_CLUSTER_NAME)"
+.PHONY: k3d-stop
+
 # Nota bene: Black should come after isort just in case they don't agree...
 lint: ## lint back-end python sources
 lint: \

diff --git a/docs/contribute.md b/docs/contribute.md
@@ -69,6 +69,16 @@ To configure those backends, we provide default parameters in the `.env.dist`
 template, you can copy/paste them in your `.env` file (and uncomment them so
 that they are properly injected in running containers).
 
+> In order to run the Elasticsearch backend locally on GNU/Linux operating
+> systems, ensure that your virtual memory limits are not too low and increase
+> them (temporally) if needed by typing this command from your terminal (as
+> `root` or using `sudo`):
+>
+> `sysctl -w vm.max_map_count=262144`
+>
+> Reference:
+> https://www.elastic.co/guide/en/elasticsearch/reference/master/vm-max-map-count.html
+
 Once configured, start available backends using:
 
 ```bash
@@ -112,4 +122,207 @@ $ bin/ralph fetch -b es
 
 Ralph is distributed along with its tray (a deployable package for Kubernetes
 clusters using [Arnold](https://github.com/openfun/arnold)). If you intend to
-work on this tray, please refer to Arnold's documentation.
+work on this tray, please refer to Arnold's documentation first.
+
+Dependencies:
+
+- [Kubectl](https://kubernetes.io/docs/tasks/tools/) (>`v.1.23.5`):
+  This CLI is used to communicate with the running Kubernetes instance you
+  will use.
+- [k3d](https://k3d.io/) (>`v.5.0.0`): This tool is used to setup
+  and run a lightweight Kubernetes cluster, in order to have a local
+  environment (it is required to complete below's quickstart instructions to
+  avoid depending on an existing Kubernetes cluster).
+- [curl](https://curl.se/) is required by Arnold's CLI.
+- [gnupg](https://gnupg.org/) to encrypt Ansible vaults passwords and
+  collaborate with your team.
+
+### Create a local `k3d` cluster
+
+To create (or run) a local kubernetes cluster, we use `k3d`. The cluster's
+bootstrapping should be run _via_:
+
+```bash
+$ make k3d-cluster
+```
+
+> Running a k3d-cluster locally supposes that the 80 and 443 ports of your
+> machine are available, so that the ingresses created for your project
+> responds properly. If one or both ports are already used by another service
+> running on your machine, the `make k3d-cluster` command may fail.
+
+You can check that your cluster is running using the `k3d cluster` command:
+
+```bash
+$ k3d cluster list
+NAME     SERVERS   AGENTS   LOADBALANCER
+ralph    1/1       0/0      true
+```
+
+As you can see, we are running a single node cluster called `ralph`.
+
+### Bootstrap an Arnold project
+
+Once your Kubernetes cluster is running, you need to create a standard Arnold
+project describing applications and environments you need to deploy:
+
+```bash
+$ make arnold-bootstrap
+```
+
+Once bootstrapped, Arnold should have created a `group_vars` directory
+containing the following files:
+
+```bash
+$ tree group_vars
+group_vars
+├── common
+└── customer
+    └── ralph
+        ├── development
+        │   ├── main.yml
+        │   └── secrets
+        │       ├── databases.vault.yml
+        │       ├── elasticsearch.vault.yml
+        │       └── ralph.vault.yml
+        └── main.yml
+
+5 directories, 5 files
+```
+
+To create the LRS credentials file, you need to provide a list of accounts
+allowed to request the LRS in Ralph's vault:
+
+```bash
+# Setup your kubernetes environment
+$ source .k3d-cluster.env.sh
+
+# Decrypt the vault
+$ bin/arnold -d -c ralph -e development -- vault -a ralph decrypt
+```
+
+Edit the vault file to add a new account for the `foo` user with the `bar`
+password and a relevant scope:
+
+```yaml
+# group_vars/customer/ralph/development/secrets/ralph.vault.yml
+#
+# [...]
+#
+# LRS
+LRS_AUTH:
+  - username: "foo"
+    hash: "$2b$12$lCggI749U6TrzK7Qyr7xGe1KVSAXdPjtkMew.BD6lzIk//T5YSb72"
+    scopes:
+      - "foo_scope"
+```
+
+The password hash has been generated using `bcrypt` as explained in the [API
+user guide](./api/#creating_a_credentials_file).
+
+And finally (re-)encrypt Ralph's vault:
+
+```bash
+# Encrypt the vault
+$ bin/arnold -d -c ralph -e development -- vault -a ralph encrypt
+```
+
+You are now ready to create the related Kubernetes Secret while initializing
+Arnold project in the next step.
+
+### Prepare working namespace
+
+You are now ready to create required Kubernetes objects to start working on
+Ralph's deployment:
+
+```bash
+$ make arnold-init
+```
+
+At this point an Elasticsearch cluster should be running on your Kubernetes
+cluster:
+
+```bash
+$ kubectl -n development-ralph get -l app=elasticsearch pod
+NAME                                         READY   STATUS      RESTARTS   AGE
+elasticsearch-node-0                         1/1     Running     0          69s
+elasticsearch-node-1                         1/1     Running     0          69s
+elasticsearch-node-2                         1/1     Running     0          69s
+es-index-template-j-221010-09h25m24s-nx5qz   0/1     Completed   0          49s
+```
+
+We are now ready to deploy Ralph to Kubernetes!
+
+### Deploy, code, repeat
+
+To test your local docker image, you need to build it and publish it to the
+local kubernetes cluster docker registry using the `k3d-push` Makefile rule:
+
+```bash
+$ make k3d-push
+```
+
+> Note that each time you modify Ralph's application or its Docker image, you
+> will need to make this update.
+
+Now that your Docker image is published, it's time to deploy it!
+
+```bash
+$ make arnold-deploy
+```
+
+To test this deployment, let's try to make an authenticated request to the LRS:
+
+```bash
+$ curl -sLk \
+    --user foo:bar \
+    "https://$(\
+        kubectl -n development-ralph \
+        get \
+        ingress/ralph-app-current \
+        -o jsonpath='{.spec.rules[0].host}')/whoami"
+```
+
+And why not send test statements from Potsie's repository:
+
+```bash
+$ curl -sL \
+    https://github.com/openfun/potsie/raw/main/fixtures/elasticsearch/lrs.json.gz | \
+  gunzip | \
+  head -n 100 | \
+  jq -s . | \
+  sed "s/@timestamp/timestamp/g" | \
+  curl -sLk \
+    --user foo:bar \
+    -X POST \
+    -H "Content-Type: application/json" \
+    "https://$(\
+        kubectl -n development-ralph \
+        get \
+        ingress/ralph-app-current \
+        -o jsonpath='{.spec.rules[0].host}')/xAPI/statements/" \
+    -d @-
+```
+
+> This example command requires [`jq`](https://stedolan.github.io/jq/) to
+> serialize the request payload (xAPI statements). When dealing with JSON data,
+> we strongly recommend to install it to manipulate them from the command line.
+
+### Perform Arnold's operations
+
+If you want to run the `bin/arnold` script to run specific Arnold commands, you
+must ensure that your environment is properly set and that Arnold runs in
+development mode (_i.e._ using the `-d` flag):
+
+```bash
+$ source .k3d-cluster.env.sh
+$ bin/arnold -d -c ralph -e development -- vault -a ralph view
+```
+
+### Stop `k3d` cluster
+
+When finished to work on the Tray, you can stop the `k3d` cluster using the `k3d-stop` helper:
+
+```bash
+$ make k3d-stop
+```