Merge branch 'v3' into merge-v3-feature-rebac-admin

.air.toml

@@ -4,15 +4,14 @@ tmp_dir = "tmp"
 
 [build]
   args_bin = []
-  bin = "env $(cat /vault/vault.env | xargs) ./tmp/jimm"
   cmd = "go build -gcflags='all=-N -l' -buildvcs=false -o ./tmp/jimm ./cmd/jimmsrv"
   delay = 1000
   exclude_dir = [".vscode", "assets", "tmp", "vendor", "testdata"]
   exclude_file = []
   exclude_regex = ["_test.go"]
   exclude_unchanged = false
   follow_symlink = false
-  full_bin = "env $(cat /vault/vault.env | xargs) dlv exec --accept-multiclient --log --headless --continue --listen :2345 --api-version 2 ./tmp/jimm"
+  full_bin = "dlv exec --accept-multiclient --log --headless --continue --listen :2345 --api-version 2 ./tmp/jimm"
   include_dir = []
   include_ext = ["go", "tpl", "tmpl", "html"]
   kill_delay = "0s"

.github/actions/test-server/README.md

@@ -0,0 +1,28 @@
+# test-server
+An action to create a JIMM server with real dependencies for integration test purposes.
+
+This action requires Docker to be installed to start JIMM and its related services.
+
+The action performs the following steps:
+- Starts JIMM's docker compose test environment.
+- Uses https://github.com/charmed-kubernetes/actions-operator action to start a Juju controller and connects it to JIMM.
+- Ensures the local Juju CLI is setup to communicate with JIMM authenticating as a test user.
+
+Use the action by adding the following to a Github workflow:
+
+```yaml
+  integration-test:
+    runs-on: ubuntu-latest
+    name: Integration testing with JIMM
+    steps:
+      - name: Setup JIMM environment
+        uses: canonical/[email protected]
+        with:
+          jimm-version: "v3.1.7"
+          juju-channel: "3/stable"
+          ghcr-pat: ${{ secrets.GHCR_PAT }}
+```
+
+Note that it's recommended to pin the action version to the same version as `jimm-version` to ensure the action works as expected for that specific version of JIMM.
+
+For full details on the inputs see `action.yaml`.

.github/actions/test-server/action.yaml

@@ -0,0 +1,117 @@
+name: JIMM Server Setup
+description: "Create a JIMM environment"
+
+inputs:
+  jimm-version:
+    description: >
+      JIMM version tag to use. This will decide the version of JIMM to start e.g. v3.1.7
+      A special tag of "dev" can be provided to use the current development version of JIMM.
+    required: true
+  juju-channel:
+    description: 'Juju snap channel to pass to charmed-kubernetes/actions-operator'
+    required: false
+  ghcr-pat:
+    description: >
+      PAT Token that has package:read access to canonical/JIMM
+      The PAT token can be left empty when building the development version of JIMM.
+    required: true
+
+outputs:
+  url:
+    description: 'URL where JIMM can be reached.'
+    value: "https://jimm.localhost"
+  client-id:
+    description: 'Test client ID to login to JIMM with a service account.'
+    value: "test-client-id"
+  client-secret:
+    description: 'Test client Secret to login to JIMM with a service account.'
+    value: "2M2blFbO4GX4zfggQpivQSxwWX1XGgNf"
+  ca-cert:
+    description: 'The CA certificate used to genereate the JIMM server cert.'
+    value: ${{ steps.fetch-cert.outputs.jimm-ca }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Login to GitHub Container Registry
+      if: ${{ inputs.jimm-version != 'dev' }}
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ inputs.ghcr-pat }}
+
+    - name: Start server based on released version
+      if: ${{ inputs.jimm-version != 'dev' }}
+      run: make integration-test-env
+      shell: bash
+      env:
+        JIMM_VERSION: ${{ inputs.jimm-version }}
+
+    - name: Start server based on development version
+      if: ${{ inputs.jimm-version == 'dev' }}
+      run: make dev-env
+      shell: bash
+
+    - name: Retrieve server CA cert.
+      id: fetch-cert
+      run: |
+        echo 'jimm-ca<<EOF' >> $GITHUB_OUTPUT
+        cat ./local/traefik/certs/ca.crt >> $GITHUB_OUTPUT
+        echo 'EOF' >> $GITHUB_OUTPUT
+      shell: bash
+
+    - name: Initialise LXD
+      run: |
+        sudo lxd waitready && \
+        sudo lxd init --auto && \
+        sudo chmod a+wr /var/snap/lxd/common/lxd/unix.socket && \
+        lxc network set lxdbr0 ipv6.address none && \
+        sudo usermod -a -G lxd $USER
+      shell: bash
+
+    - name: Setup cloud-init script for bootstraping Juju controllers
+      run: ./local/jimm/setup-controller.sh
+      shell: bash
+      env:
+        SKIP_BOOTSTRAP: true
+        CLOUDINIT_FILE: "cloudinit.temp.yaml"
+
+    - name: Setup Juju Controller
+      uses: charmed-kubernetes/actions-operator@main
+      with:
+        provider: "lxd"
+        channel: "5.19/stable"
+        juju-channel: ${{ inputs.juju-channel }}
+        bootstrap-options: "--config cloudinit.temp.yaml --config login-token-refresh-url=https://jimm.localhost/.well-known/jwks.json"
+
+    # As described in https://github.com/charmed-kubernetes/actions-operator grab the newly setup controller name
+    - name: Save LXD controller name
+      id: lxd-controller
+      run: echo "name=$CONTROLLER_NAME" >> $GITHUB_OUTPUT
+      shell: bash
+
+    - name: Install jimmctl, jaas plugin and yq
+      run: |
+        sudo snap install jimmctl --channel=3/stable && \
+        sudo snap install jaas --channel=3/stable &&
+        sudo snap install yq
+      shell: bash
+
+    - name: Authenticate Juju CLI
+      run: chmod -R 666 ~/.local/share/juju/*.yaml && ./local/jimm/setup-cli-auth.sh
+      shell: bash
+      # Below is a hardcoded JWT using the same test-secret used in JIMM's docker compose and allows the CLI to authenticate as the [email protected] user.
+      env:
+        JWT: ZXlKMGVYQWlPaUpLVjFRaUxDSmhiR2NpT2lKSVV6STFOaUo5LmV5SnBjM01pT2lKUGJteHBibVVnU2xkVUlFSjFhV3hrWlhJaUxDSnBZWFFpT2pFM01qUXlNamcyTmpBc0ltVjRjQ0k2TXprMk5EYzFNelEyTUN3aVlYVmtJam9pYW1sdGJTSXNJbk4xWWlJNkltcHBiVzB0ZEdWemRFQmpZVzV2Ym1sallXd3VZMjl0SW4wLkpTWVhXcGF6T0FnX1VFZ2hkbjlOZkVQdWxhWWlJQVdaX3BuSmRDbnJvWEk=
+
+    - name: Add LXD Juju controller to JIMM
+      run: ./local/jimm/add-controller.sh
+      shell: bash
+      env:
+        JIMM_CONTROLLER_NAME: "jimm"
+        CONTROLLER_NAME: ${{ steps.lxd-controller.outputs.name }}
+
+    - name: Provide service account with cloud-credentials
+      run: ./local/jimm/setup-service-account.sh
+      shell: bash

.github/workflows/cache.yaml

@@ -0,0 +1,32 @@
+name: Cache on default branch
+on:
+  push:
+    branches:
+      - v3
+      - "feature*"
+
+jobs:
+  go_cache:
+    name: Cache Go Dependencies and Build/Lint Artifacts
+    runs-on: ubuntu-22.04
+    timeout-minutes: 15
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-tags: true
+          fetch-depth: 0
+
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Build
+        run: go build ./...
+
+      - name: Run Golangci-lint
+        uses: golangci/golangci-lint-action@v6
+        with:
+          args: --timeout 30m --verbose
+          version: v1.60

.github/workflows/ci.yaml

@@ -12,9 +12,6 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          fetch-tags: true
-          fetch-depth: 0
 
       - name: Setup Go
         uses: actions/setup-go@v4
@@ -27,15 +24,6 @@ jobs:
       - name: Install juju-db
         run: sudo snap install juju-db --channel 4.4/stable
 
-      - name: Add volume files
-        run: |
-          touch ./local/vault/approle.json
-          touch ./local/vault/roleid.txt
-          touch ./local/vault/vault.env
-
-      - name: Create test certs
-        run: make certs
-
       - name: Start test environment
         run: docker compose up -d --wait
 
@@ -46,29 +34,3 @@ jobs:
 
       - name: Build and Test
         run: go test -mod readonly ./... -timeout 1h -cover
-        env:
-          JIMM_DSN: postgresql://jimm:jimm@localhost:5432/jimm
-          JIMM_TEST_PGXDSN: postgresql://jimm:jimm@localhost:5432/jimm
-          PGHOST: localhost
-          PGPASSWORD: jimm
-          PGSSLMODE: disable
-          PGUSER: jimm
-          PGPORT: 5432
-
-  smoke_test:
-    name: Smoke Test
-    runs-on: ubuntu-22.04
-    # The docker compose has a healthcheck on the JIMM container.
-    # So if the compose returns with exit code 0 then the JIMM server successfully started.
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Add volume files
-        run: |
-          touch ./local/vault/approle.json
-          touch ./local/vault/roleid.txt
-          touch ./local/vault/vault.env
-
-      - name: Run Smoke Test
-        run: docker compose --profile dev up -d --wait --timestamps

.github/workflows/golangci-lint.yaml

@@ -0,0 +1,27 @@
+name: golangci-lint
+on:
+  pull_request:
+
+permissions:
+  contents: read
+  checks: write # Optional: allow write access to checks to allow the action to annotate code in the PR.
+
+jobs:
+  golangci:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+
+      - name: Run Golangci-lint
+        uses: golangci/golangci-lint-action@v6
+        with:
+          args: --timeout 30m --verbose
+          version: v1.60
+

.github/workflows/integration-test.yaml

@@ -0,0 +1,52 @@
+name: Integration Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      jimm-version:
+        description: >
+          JIMM version tag to use. This will decide the version of JIMM to start e.g. v3.1.7.
+          View all available versions at https://github.com/canonical/jimm/pkgs/container/jimm.
+        required: true
+  pull_request:
+
+jobs:
+  startjimm:
+    name: Test JIMM with Juju controller
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout JIMM repo
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Go vendor to speed up docker build
+        if: ${{ github.event_name == 'pull_request' }}
+        run: go mod vendor
+
+      - name: Start JIMM (pull request)
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: ./.github/actions/test-server
+        with:
+          jimm-version: dev
+          juju-channel: "3/stable"
+          ghcr-pat: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Start JIMM (manual run)
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        uses: ./.github/actions/test-server
+        with:
+          jimm-version: ${{ inputs.jimm-version }}
+          juju-channel: "3/stable"
+          ghcr-pat: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create a model, deploy an application and run juju status
+        run: |
+          juju add-model foo && \
+          juju deploy haproxy && \
+          sleep 5 && \
+          juju status

.github/workflows/update-sdk.yaml

@@ -31,7 +31,6 @@ jobs:
         repository: ${{ github.event.inputs.sdk-repo }}
         ref: ${{ github.event.inputs.sdk-version }}
         path: ./sdk
-        token: ${{ secrets.PAT }}
 
     - name: Setup Go
       uses: actions/setup-go@v5
@@ -46,11 +45,12 @@ jobs:
         SDK_VERSION: ${{ github.event.inputs.sdk-version }}
       run: |
         # Remove all in case some files are removed
-        rm -rf .[!.]* *
+        shopt -s nullglob
+        rm -rf .[!.git]* *
         cp -r $PROJECT/pkg/.[^.]* $PROJECT/pkg/* $PROJECT/go.mod .
 
         # Replace module references
-        find . -type f -exec sed -i "s|github.com/canonical/jimm/pkg|github.com/$SDK_REPO/$SDK_VERSION|" {} +
+        find . -type f -exec sed -i "s|github.com/canonical/jimm/v3/pkg|github.com/$SDK_REPO/$SDK_VERSION|" {} +
         sed -i "s|module .*|module github.com/$SDK_REPO/$SDK_VERSION|" go.mod
 
         # Needed to remove unused dependencies
@@ -59,7 +59,7 @@ jobs:
     - name: Create Pull Request
       uses: peter-evans/create-pull-request@v6
       with:
-        token: ${{ secrets.PAT }}
+        token: ${{ secrets.JIMM_GO_SDK_PAT }}
         path: ./sdk
         branch: update-sdk-${{ github.run_number }}
         title: Update SDK ${{ github.event.inputs.sdk-version }}

.github/workflows/vulncheck.yaml → .github/workflows/vulnerability-check.yaml

@@ -1,4 +1,4 @@
-name: Security Check
+name: Vulnerability Check
 
 on:
   schedule:
@@ -16,5 +16,5 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version-file: go.mod
-      - name: Security checks
+      - name: Security scan
         uses: canonical/comsys-build-tools/.github/actions/security-scan@main

.gitignore

@@ -11,10 +11,6 @@
 /version/commit.txt
 /version/version.txt
 /tmp
-/local/vault/approle.json
-local/vault/approle.json
-local/vault/roleid.txt
-local/vault/vault.env
 
 *.crt
 *.key