Add composite action for starting a JIMM environment

.air.toml

@@ -4,7 +4,6 @@ tmp_dir = "tmp"
 
 [build]
   args_bin = []
-  bin = "env $(cat /vault/vault.env | xargs) ./tmp/jimm"
   cmd = "go build -gcflags='all=-N -l' -buildvcs=false -o ./tmp/jimm ./cmd/jimmsrv"
   delay = 1000
   exclude_dir = [".vscode", "assets", "tmp", "vendor", "testdata"]

.github/actions/test-server/README.md

@@ -0,0 +1,28 @@
+# test-server
+An action to create a JIMM server with real dependencies for integration test purposes.
+
+This action requires Docker to be installed to start JIMM and its related services.
+
+The action performs the following steps:
+- Starts JIMM's docker compose test environment.
+- Uses https://github.com/charmed-kubernetes/actions-operator action to start a Juju controller and connects it to JIMM.
+- Ensures the local Juju CLI is setup to communicate with JIMM authenticating as a test user.
+
+Use the action by adding the following to a Github workflow:
+
+```yaml
+  integration-test:
+    runs-on: ubuntu-latest
+    name: Integration testing with JIMM
+    steps:
+      - name: Setup JIMM environment
+        uses: canonical/[email protected]
+        with:
+          jimm-version: "v3.1.7"
+          juju-channel: "3/stable"
+          ghcr-pat: ${{ secrets.GHCR_PAT }}
+```
+
+Note that it's recommended to pin the action version to the same version as `jimm-version` to ensure the action works as expected for that specific version of JIMM.
+
+For full details on the inputs see `action.yaml`.

.github/actions/test-server/action.yaml

@@ -0,0 +1,99 @@
+name: JIMM Server Setup
+description: "Create a JIMM environment"
+
+inputs:
+  jimm-version:
+    description: >
+      JIMM version tag to use. This will decide the version of JIMM to start e.g. v3.1.7
+      A special tag of "dev" can be provided to use the current development version of JIMM.
+    required: true
+  juju-channel:
+    description: 'Juju snap channel to pass to charmed-kubernetes/actions-operator'
+    required: false
+  ghcr-pat:
+    description: >
+      PAT Token that has package:read access to canonical/JIMM
+      The PAT token can be left empty when building the development version of JIMM.
+    required: true
+
+output:
+  url:
+    description: 'URL where JIMM can be reached.'
+    value: "https://jimm.localhost"
+  client-id:
+    description: 'Test client ID to login to JIMM with a service account.'
+    value: "test-client-id"
+  client-secret:
+    description: 'Test client Secret to login to JIMM with a service account.'
+    value: "2M2blFbO4GX4zfggQpivQSxwWX1XGgNf"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Login to GitHub Container Registry
+      if: ${{ inputs.jimm-version != 'dev' }}
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ inputs.ghcr-pat }}
+
+    - name: Start server based on released version
+      if: ${{ inputs.jimm-version != 'dev' }}
+      run: make integration-test-env
+      shell: bash
+      env:
+        JIMM_VERSION: ${{ inputs.jimm-version }}
+
+    - name: Start server based on development version
+      if: ${{ inputs.jimm-version == 'dev' }}
+      run: make dev-env
+      shell: bash
+
+    - name: Initialise LXD
+      run: |
+        sudo lxd waitready && \
+        sudo lxd init --auto && \
+        sudo chmod a+wr /var/snap/lxd/common/lxd/unix.socket && \
+        lxc network set lxdbr0 ipv6.address none && \
+        sudo usermod -a -G lxd $USER
+      shell: bash
+
+    - name: Setup cloud-init script for bootstraping Juju controllers
+      run: ./local/jimm/setup-controller.sh
+      shell: bash
+      env:
+        SKIP_BOOTSTRAP: true
+        CLOUDINIT_FILE: "cloudinit.temp.yaml"
+
+    - name: Setup Juju Controller
+      uses: charmed-kubernetes/actions-operator@main
+      with:
+        provider: "lxd"
+        channel: "5.19/stable"
+        juju-channel: ${{ inputs.juju-channel }}
+        bootstrap-options: "--config cloudinit.temp.yaml --config login-token-refresh-url=https://jimm.localhost/.well-known/jwks.json"
+
+    # As described in https://github.com/charmed-kubernetes/actions-operator grab the newly setup controller name
+    - name: Save LXD controller name
+      id: lxd-controller
+      run: echo "name=$CONTROLLER_NAME" >> $GITHUB_OUTPUT
+      shell: bash
+
+    - name: Install jimmctl and yq
+      run: sudo snap install jimmctl --channel=3/stable && sudo snap install yq
+      shell: bash
+
+    - name: Authenticate Juju CLI
+      run: chmod -R 666 ~/.local/share/juju/*.yaml && ./local/jimm/setup-cli-auth.sh
+      shell: bash
+      # Below is a hardcoded JWT using the same test-secret used in JIMM's docker compose and allows the CLI to authenticate as the [email protected] user.
+      env:
+        JWT: ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKSVV6STFOaUo5LmV5SnBjM01pT2lKUGJteHBibVVnU2xkVUlFSjFhV3hrWlhJaUxDSnBZWFFpT2pFM01qUXlNamcyTmpBc0ltVjRjQ0k2TXprMk5EYzFNelEyTUN3aVlYVmtJam9pYW1sdGJTSXNJbk4xWWlJNkltcHBiVzB0ZEdWemRFQmpZVzV2Ym1sallXd3VZMjl0SW4wLkpTWVhXcGF6T0FnX1VFZ2hkbjlOZkVQdWxhWWlJQVdaX3BuSmRDbnJvWEk=
+
+    - name: Add LXD Juju controller to JIMM
+      run: ./local/jimm/add-controller.sh
+      shell: bash
+      env:
+        JIMM_CONTROLLER_NAME: "jimm"
+        CONTROLLER_NAME: ${{ steps.lxd-controller.outputs.name }}

.github/workflows/ci.yaml

@@ -54,21 +54,3 @@ jobs:
           PGSSLMODE: disable
           PGUSER: jimm
           PGPORT: 5432
-
-  smoke_test:
-    name: Smoke Test
-    runs-on: ubuntu-22.04
-    # The docker compose has a healthcheck on the JIMM container.
-    # So if the compose returns with exit code 0 then the JIMM server successfully started.
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Add volume files
-        run: |
-          touch ./local/vault/approle.json
-          touch ./local/vault/roleid.txt
-          touch ./local/vault/vault.env
-
-      - name: Run Smoke Test
-        run: docker compose --profile dev up -d --wait --timestamps

.github/workflows/integration-test.yaml

@@ -0,0 +1,47 @@
+name: Integration Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      jimm-version:
+        description: >
+          JIMM version tag to use. This will decide the version of JIMM to start e.g. v3.1.7.
+          View all available versions at https://github.com/canonical/jimm/pkgs/container/jimm.
+        required: true
+  pull_request:
+
+jobs:
+  startjimm:
+    name: Test JIMM with Juju controller
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout JIMM repo
+        uses: actions/checkout@v4
+      - name: Setup Go
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: 'go.mod'
+      - name: Go vendor to speed up docker build
+        if: ${{ github.event_name == 'pull_request' }}
+        run: go mod vendor
+      - name: Start JIMM (pull request)
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: ./.github/actions/test-server
+        with:
+          jimm-version: dev
+          juju-channel: "3/stable"
+          ghcr-pat: ${{ secrets.GITHUB_TOKEN }}
+      - name: Start JIMM (manual run)
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        uses: ./.github/actions/test-server
+        with:
+          jimm-version: ${{ inputs.jimm-version }}
+          juju-channel: "3/stable"
+          ghcr-pat: ${{ secrets.GITHUB_TOKEN }}
+      - name: Create a model, deploy an application and run juju status
+        run: |
+          juju add-model foo && \
+          juju deploy haproxy && \
+          sleep 5 && \
+          juju status

Makefile

@@ -47,11 +47,14 @@ dev-env-setup: sys-deps certs
 	@make version/commit.txt && make version/version.txt
 
 dev-env: dev-env-setup
-	@docker compose --profile dev up --force-recreate
+	@docker compose --profile dev up -d --force-recreate --wait
 
 dev-env-cleanup:
 	@docker compose --profile dev down -v --remove-orphans
 
+integration-test-env: dev-env-setup
+	@JIMM_VERSION=$(JIMM_VERSION) docker compose --profile test up -d --force-recreate --wait
+
 # Reformat all source files.
 format:
 	gofmt -w -l .
@@ -116,17 +119,25 @@ define check_dep
 	fi
 endef
 
-# Install packages required to develop JIMM and run tests.
+# Install packages required to develop JIMM and/or run tests.
 APT_BASED := $(shell command -v apt-get >/dev/null; echo $$?)
 sys-deps:
 ifeq ($(APT_BASED),0)
+# golangci-lint is necessary for linting.
 	@$(call check_dep,golangci-lint,Missing Golangci-lint - install from https://golangci-lint.run/welcome/install/)
+# Go acts as the test runner.
 	@$(call check_dep,go,Missing Go - install from https://go.dev/doc/install or 'sudo snap install go --classic')
+# Git is useful to have.
 	@$(call check_dep,git,Missing Git - install with 'sudo apt install git')
+# GCC is required for the compilation process.
 	@$(call check_dep,gcc,Missing gcc - install with 'sudo apt install build-essential')
+# yq is necessary for some scripts that process controller-info yaml files.
 	@$(call check_dep,yq,Missing yq - install with 'sudo snap install yq')
-	@$(call check_dep,gcc,Missing microk8s - install latest none-classic from snapstore')
+# Microk8s is required if you want to start a Juju controller on Microk8s.
+	@$(call check_dep,microk8s,Missing microk8s - install with 'sudo snap install microk8s')
+# Docker is required to start the test dependencies in containers.
 	@$(call check_dep,docker,Missing Docker - install from https://docs.docker.com/engine/install/)
+# juju-db is required for tests that use Juju's test fixture, requiring MongoDB.
 	@$(call check_dep,juju-db.mongo,Missing juju-db - install with 'sudo snap install juju-db --channel=4.4/stable')
 else
 	@echo sys-deps runs only on systems with apt-get

compose-common.yaml

@@ -0,0 +1,62 @@
+# This file contains a collection of common configurations used in JIMM's Docker compose file.
+
+services:
+  jimm-base:
+    environment:
+      JIMM_LOG_LEVEL: "debug"
+      JIMM_UUID: "3217dbc9-8ea9-4381-9e97-01eab0b3f6bb"
+      JIMM_DSN: "postgresql://jimm:jimm@db/jimm"
+      # Not needed for local test (yet).
+      # BAKERY_AGENT_FILE: ""
+      JIMM_ADMINS: "[email protected]"
+      # Note: You can comment out the Vault ENV vars below and instead use INSECURE_SECRET_STORAGE to place secrets in Postgres.
+      VAULT_ADDR: "http://vault:8200"
+      VAULT_PATH: "/jimm-kv/"
+      # Note: By default we should use Vault as that is the primary means of secret storage.
+      # INSECURE_SECRET_STORAGE: "enabled"
+      # JIMM_DASHBOARD_LOCATION: ""
+      JIMM_DNS_NAME: "jimm.localhost"
+      JIMM_LISTEN_ADDR: "0.0.0.0:80"
+      JIMM_TEST_PGXDSN: "postgresql://jimm:jimm@db/jimm"
+      JIMM_JWT_EXPIRY: 30s
+      JIMM_AUDIT_LOG_RETENTION_PERIOD_IN_DAYS: "1"
+      TEST_LOGGING_CONFIG: ""
+      BAKERY_PUBLIC_KEY: "izcYsQy3TePp6bLjqOo3IRPFvkQd2IKtyODGqC6SdFk="
+      BAKERY_PRIVATE_KEY: "ly/dzsI9Nt/4JxUILQeAX79qZ4mygDiuYGqc2ZEiDEc="
+      OPENFGA_SCHEME: "http"
+      OPENFGA_HOST: "openfga"
+      OPENFGA_PORT: 8080
+      OPENFGA_STORE: "01GP1254CHWJC1MNGVB0WDG1T0"
+      OPENFGA_AUTH_MODEL: "01GP1EC038KHGB6JJ2XXXXCXKB"
+      OPENFGA_TOKEN: "jimm"
+      JIMM_IS_LEADER: true
+      JIMM_OAUTH_ISSUER_URL: "http://keycloak.localhost:8082/realms/jimm" # Scheme required
+      JIMM_OAUTH_CLIENT_ID: "jimm-device"
+      JIMM_OAUTH_CLIENT_SECRET: "SwjDofnbDzJDm9iyfUhEp67FfUFMY8L4"
+      JIMM_OAUTH_SCOPES: "openid profile email" # Space separated list of scopes
+      JIMM_DASHBOARD_FINAL_REDIRECT_URL: "https://jaas.ai" # Example URL
+      JIMM_ACCESS_TOKEN_EXPIRY_DURATION: 1h
+      JIMM_SECURE_SESSION_COOKIES: false
+      JIMM_SESSION_COOKIE_MAX_AGE: 86400
+      JIMM_SESSION_SECRET_KEY: Xz2RkR9g87M75xfoumhEs5OmGziIX8D88Rk5YW8FSvkBPSgeK9t5AS9IvPDJ3NnB
+    healthcheck:
+      test: [ "CMD", "curl", "http://jimm.localhost:80" ]
+      interval: 5s
+      timeout: 5s
+      retries: 50 # Should fail after approximately (interval*retry) seconds
+    depends_on:
+      db:
+        condition: service_healthy
+      openfga:
+        condition: service_healthy
+      traefik:
+        condition: service_healthy
+      insert-hardcoded-auth-model:
+        condition: service_completed_successfully
+      keycloak:
+        condition: service_healthy
+    labels:
+      traefik.enable: true
+      traefik.http.routers.jimm.rule: Host(`jimm.localhost`)
+      traefik.http.routers.jimm.entrypoints: websecure
+      traefik.http.routers.jimm.tls: true