From bb8f0e4c15595b895319fb00420ad37de1ce3428 Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 09:48:35 +0000 Subject: [PATCH 01/39] :wrench: Add GitHub Workflows --- .github/workflows/build-test.yml | 36 ++++++++++++++++ .github/workflows/codeql-analysis.yml | 41 ++++++++++++++++++ .github/workflows/dependency-review.yml | 5 ++- .github/workflows/release.yml | 54 +++++++++++++++++++++++ .github/workflows/scan-image.yml | 57 +++++++++++++++++++++++++ .github/workflows/super-linter.yml | 42 ++++++++++++++++++ 6 files changed, 233 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/build-test.yml create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/scan-image.yml create mode 100644 .github/workflows/super-linter.yml diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml new file mode 100644 index 0000000..bafeb68 --- /dev/null +++ b/.github/workflows/build-test.yml @@ -0,0 +1,36 @@ +--- +name: Build and Test + +on: + pull_request: + branches: + - main + +permissions: {} + +jobs: + build: + name: Build and Test + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + id: checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Build Image + id: build_image + uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 + with: + push: false + load: true + tags: dashboard + + - name: Run Python Tests + id: run_python_tests + run: | + docker compose --file contrib/docker-compose-test.yml run --rm interfaces + env: + NETWORK: default + IMAGE_TAG: dashboard diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..567180e --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,41 @@ +--- +name: CodeQL Analysis + +on: + pull_request: + branches: + - main + +permissions: {} + +jobs: + codeql-analysis: + name: CodeQL Analysis + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + strategy: + fail-fast: false + matrix: + language: ["javascript", "python"] + steps: + - name: Checkout + id: checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Initialise CodeQL + id: initialise_codeql + uses: github/codeql-action/init@407ffafae6a767df3e0230c3df91b6443ae8df75 # v2.22.8 + with: + languages: ${{ matrix.language }} + + - name: CodeQL Autobuild + id: codeql_autobuild + uses: github/codeql-action/autobuild@407ffafae6a767df3e0230c3df91b6443ae8df75 # v2.22.8 + + - name: CodeQL Analysis + id: codeql_analysis + uses: github/codeql-action/analyze@407ffafae6a767df3e0230c3df91b6443ae8df75 # v2.22.8 + with: + category: "language:${{ matrix.language }}" diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index e2f64bf..f1dcfde 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -20,9 +20,10 @@ jobs: steps: - name: Checkout id: checkout - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Dependency Review - uses: actions/dependency-review-action@72eb03d02c7872a771aacd928f3123ac62ad6d3a # v4.3.3 + id: dependency_review + uses: actions/dependency-review-action@01bc87099ba56df1e897b6874784491ea6309bc4 # v3.1.4 with: fail-on-severity: critical diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..a110715 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,54 @@ +--- +name: Release + +on: + push: + tags: + - '*.*.*' + +permissions: {} + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + packages: write + steps: + - name: Checkout + id: checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Install cosign + id: install-cosign + uses: sigstore/cosign-installer@1fc5bd396d372bee37d608f955b336615edf79c8 # v3.2.0 + + - name: Login to GitHub Container Registry + id: login + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push + id: push + uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 + with: + context: . + push: true + tags: ghcr.io/${{ github.repository_owner }}/analytical-platform-ui:${{ github.ref_name }} + + - name: Sign + id: sign + run: | + cosign sign --yes ghcr.io/${{ github.repository_owner }}/analytical-platform-ui@${{ steps.push.outputs.digest }} + + - name: Verify + id: verify + run: | + cosign verify \ + --certificate-oidc-issuer=https://token.actions.githubusercontent.com \ + --certificate-identity=https://github.com/${{ github.repository_owner }}/analytical-platform-ui/.github/workflows/release.yml@refs/tags/${{ github.ref_name }} \ + ghcr.io/${{ github.repository_owner }}/analytical-platform-ui@${{ steps.push.outputs.digest }} diff --git a/.github/workflows/scan-image.yml b/.github/workflows/scan-image.yml new file mode 100644 index 0000000..de72c32 --- /dev/null +++ b/.github/workflows/scan-image.yml @@ -0,0 +1,57 @@ +--- +name: Scan Image + +on: + pull_request: + branches: + - main + +permissions: {} + +jobs: + scan-image: + name: Scan Image + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + steps: + - name: Checkout + id: checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Build Image + id: build_image + uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 + with: + push: false + load: true + tags: dashboard + + - name: Scan Image + id: scan_image + uses: aquasecurity/trivy-action@062f2592684a31eb3aa050cc61e7ca1451cecd3d # v0.18.0 + with: + image-ref: dashboard + exit-code: 1 + format: sarif + output: trivy-results.sarif + severity: CRITICAL + limit-severities-for-sarif: true + + - name: Scan Image (On SARIF Scan Failure) + if: failure() && steps.scan_image.outcome == 'failure' + id: scan_image_on_failure + uses: aquasecurity/trivy-action@062f2592684a31eb3aa050cc61e7ca1451cecd3d # v0.18.0 + with: + image-ref: dashboard + exit-code: 1 + format: table + severity: CRITICAL + + - name: Upload SARIF + if: always() + id: upload_sarif + uses: github/codeql-action/upload-sarif@cdcdbb579706841c47f7063dda365e292e5cad7a # v2.2.7 + with: + sarif_file: trivy-results.sarif diff --git a/.github/workflows/super-linter.yml b/.github/workflows/super-linter.yml new file mode 100644 index 0000000..393bc8e --- /dev/null +++ b/.github/workflows/super-linter.yml @@ -0,0 +1,42 @@ +--- +name: Super-Linter + +on: + pull_request: + branches: + - main + types: + - edited + - opened + - reopened + - synchronize + +permissions: {} + +jobs: + super-linter: + name: Super-Linter + runs-on: ubuntu-latest + permissions: + contents: read + statuses: write + steps: + - name: Checkout + id: checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + + - name: Super-Linter + id: super_linter + # yamllint disable-line rule:line-length + uses: super-linter/super-linter/slim@ba3315d7e5da8cee94ef552f3baf1c34f6021345 # v5.7.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEFAULT_BRANCH: main + VALIDATE_ALL_CODEBASE: false + LINTER_RULES_PATH: / + PYTHON_BLACK_CONFIG_FILE: pyproject.toml + PYTHON_FLAKE8_CONFIG_FILE: .flake8 + PYTHON_ISORT_CONFIG_FILE: pyproject.toml + PYTHON_MYPY_CONFIG_FILE: mypy.ini From 56d69f7e3ae91d4499096961c3c41981ebe56cda Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 09:50:48 +0000 Subject: [PATCH 02/39] :wrench: Add --- .editorconfig | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..d159d74 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,10 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.sh] +indent_style = space +indent_size = 2 \ No newline at end of file From 328d7961a313363f0968da023f7c8963bbe7fd16 Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 10:50:42 +0000 Subject: [PATCH 03/39] :wrench: Add devcontainer --- .devcontainer/devcontainer.json | 27 +++++++++++++++++ .../src/postgresql/devcontainer-feature.json | 6 ++++ .../features/src/postgresql/install.sh | 14 +++++++++ contrib/docker-compose-postgresql.yml | 18 ++++++++++++ contrib/docker-compose-test.yml | 29 +++++++++++++++++++ 5 files changed, 94 insertions(+) create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/features/src/postgresql/devcontainer-feature.json create mode 100644 .devcontainer/features/src/postgresql/install.sh create mode 100644 contrib/docker-compose-postgresql.yml create mode 100644 contrib/docker-compose-test.yml diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..fa9f44e --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,27 @@ +{ + "name": "analytical-platform-ollamate", + "image": "ghcr.io/ministryofjustice/devcontainer-base:latest", + "features": { + "ghcr.io/devcontainers/features/node:1": { + "version": "20.11.1" + }, + "ghcr.io/devcontainers/features/python:1": { + "version": "3.12" + }, + "ghcr.io/devcontainers/features/docker-in-docker:2": {}, + "./features/src/postgresql": {}, + "ghcr.io/ministryofjustice/devcontainer-feature/aws:0": {} + }, + "postCreateCommand": "bash scripts/devcontainer/post-create.sh", + "postStartCommand": "bash scripts/devcontainer/post-start.sh", + "runArgs": ["--name=analytical-platform-ollamate-devcontainer"], + "customizations": { + "vscode": { + "extensions": [ + "EditorConfig.EditorConfig", + "GitHub.vscode-github-actions", + "GitHub.vscode-codeql" + ] + } + } + } \ No newline at end of file diff --git a/.devcontainer/features/src/postgresql/devcontainer-feature.json b/.devcontainer/features/src/postgresql/devcontainer-feature.json new file mode 100644 index 0000000..d8487e3 --- /dev/null +++ b/.devcontainer/features/src/postgresql/devcontainer-feature.json @@ -0,0 +1,6 @@ +{ + "id": "postgresql", + "version": "1.0.0", + "name": "postgresql", + "description": "PostgreSQL" + } \ No newline at end of file diff --git a/.devcontainer/features/src/postgresql/install.sh b/.devcontainer/features/src/postgresql/install.sh new file mode 100644 index 0000000..7be223b --- /dev/null +++ b/.devcontainer/features/src/postgresql/install.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" >/etc/apt/sources.list.d/pgdg.list + +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + +apt-get update + +apt-get -y install \ + postgresql-common \ + postgresql-client-common \ + postgresql-15 \ + postgresql-client-15 \ + libpq-dev \ No newline at end of file diff --git a/contrib/docker-compose-postgresql.yml b/contrib/docker-compose-postgresql.yml new file mode 100644 index 0000000..09d3ab9 --- /dev/null +++ b/contrib/docker-compose-postgresql.yml @@ -0,0 +1,18 @@ +--- +version: '3.8' + +services: + postgres: + image: public.ecr.aws/docker/library/postgres:15.4 + restart: always + environment: + POSTGRES_USER: ollamate + POSTGRES_PASSWORD: ollamate + POSTGRES_DB: ollamate + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ollamate"] + interval: 5s + timeout: 5s + retries: 5 diff --git a/contrib/docker-compose-test.yml b/contrib/docker-compose-test.yml new file mode 100644 index 0000000..809760a --- /dev/null +++ b/contrib/docker-compose-test.yml @@ -0,0 +1,29 @@ +--- +services: + db: + extends: + file: docker-compose-postgres.yml + service: postgres + + interfaces: + image: ${IMAGE_TAG} + ports: ["8000:8000"] + depends_on: + db: + condition: service_healthy + links: [db] + environment: + ALLOWED_HOSTS: "localhost 127.0.0.1 0.0.0.0" + DB_HOST: "db" + DB_NAME: ollamate + DB_PASSWORD: ollamate + DB_PORT: 5432 + DB_USER: ollamate + DEBUG: "True" + DJANGO_SETTINGS_MODULE: "ollamate.settings.test" + ENV: "dev" + PYTHONUNBUFFERED: "1" + SECRET_KEY: "1234567890" + ENABLE_DB_SSL: "False" + entrypoint: pytest + command: ./tests --color=yes From 26ae35ec0d9d8dbca7965215c5761d5845fa39bc Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 14:53:03 +0000 Subject: [PATCH 04/39] :construction: Add many files --- .devcontainer/devcontainer-lock.json | 42 ++++++ .devcontainer/devcontainer.json | 52 ++++--- .editorconfig | 15 +- .flake8 | 5 + .github/CODEOWNERS | 4 +- .github/dependabot.yml | 18 +-- .github/workflows/chart-lint.yml | 33 ++++ .github/workflows/enforce-version-pinning.yml | 38 +++++ .github/workflows/release.yml | 141 +++++++++++------- .github/workflows/scan-image.yml | 98 ++++++------ .pre-commit-config.yaml | 45 ++++++ Dockerfile | 42 ++++++ Makefile | 21 +++ README.md | 65 +------- chart/.helmignore | 23 +++ chart/Chart.yaml | 11 ++ chart/ci/lint-values.yaml | 0 chart/templates/.helpers.tpl | 62 ++++++++ chart/templates/deployment.yml | 49 ++++++ chart/templates/scaled-job.yaml | 49 ++++++ chart/templates/secret-account.yaml | 12 ++ chart/templates/secret.yaml | 10 ++ chart/values.yaml | 49 ++++++ nginx/nginx.conf | 33 ++++ ollamate/settings.py | 21 ++- package-lock.json | 6 + requirements.dev.txt | 7 + requirements.txt | 4 + scripts/container/entrypoint.sh | 21 +++ scripts/devcontainer/post-create.sh | 20 +++ scripts/devcontainer/post-start.sh | 4 + 31 files changed, 793 insertions(+), 207 deletions(-) create mode 100644 .devcontainer/devcontainer-lock.json create mode 100644 .flake8 create mode 100644 .github/workflows/chart-lint.yml create mode 100644 .github/workflows/enforce-version-pinning.yml create mode 100644 .pre-commit-config.yaml create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 chart/.helmignore create mode 100644 chart/Chart.yaml create mode 100644 chart/ci/lint-values.yaml create mode 100644 chart/templates/.helpers.tpl create mode 100644 chart/templates/deployment.yml create mode 100644 chart/templates/scaled-job.yaml create mode 100644 chart/templates/secret-account.yaml create mode 100644 chart/templates/secret.yaml create mode 100644 chart/values.yaml create mode 100644 nginx/nginx.conf create mode 100644 package-lock.json create mode 100644 requirements.dev.txt create mode 100644 requirements.txt create mode 100644 scripts/container/entrypoint.sh create mode 100644 scripts/devcontainer/post-create.sh create mode 100644 scripts/devcontainer/post-start.sh diff --git a/.devcontainer/devcontainer-lock.json b/.devcontainer/devcontainer-lock.json new file mode 100644 index 0000000..a94c6d8 --- /dev/null +++ b/.devcontainer/devcontainer-lock.json @@ -0,0 +1,42 @@ +{ + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": { + "version": "2.11.0", + "resolved": "ghcr.io/devcontainers/features/docker-in-docker@sha256:503f23cd692325b3cbb8c20a0ecfabb3444b0c786b363e0c82572bd7d71dc099", + "integrity": "sha256:503f23cd692325b3cbb8c20a0ecfabb3444b0c786b363e0c82572bd7d71dc099" + }, + "ghcr.io/devcontainers/features/node:1": { + "version": "1.5.0", + "resolved": "ghcr.io/devcontainers/features/node@sha256:a124954d7ed085eb90e08e6fcecac8cbcbb866317ab16deb2c7797d63cbf35d6", + "integrity": "sha256:a124954d7ed085eb90e08e6fcecac8cbcbb866317ab16deb2c7797d63cbf35d6" + }, + "ghcr.io/devcontainers/features/python:1": { + "version": "1.6.2", + "resolved": "ghcr.io/devcontainers/features/python@sha256:adf861c49eb404ce507280936fa626dcfdc4cffeb7f0a975ef400861a0cb3313", + "integrity": "sha256:adf861c49eb404ce507280936fa626dcfdc4cffeb7f0a975ef400861a0cb3313" + }, + "ghcr.io/ministryofjustice/devcontainer-feature/aws:1": { + "version": "1.0.0", + "resolved": "ghcr.io/ministryofjustice/devcontainer-feature/aws@sha256:bb07a76c8e7a6b630a2056ce959addddee436e3f9936c69b9163eff54f58dbd5", + "integrity": "sha256:bb07a76c8e7a6b630a2056ce959addddee436e3f9936c69b9163eff54f58dbd5" + }, + "ghcr.io/ministryofjustice/devcontainer-feature/container-structure-test:1": { + "version": "1.0.0", + "resolved": "ghcr.io/ministryofjustice/devcontainer-feature/container-structure-test@sha256:19eb30f9eb327b667be2002757d55381de87cdb5a79a6e37d293369fe8ad01ad", + "integrity": "sha256:19eb30f9eb327b667be2002757d55381de87cdb5a79a6e37d293369fe8ad01ad", + "dependsOn": [ + "ghcr.io/devcontainers/features/docker-in-docker:2" + ] + }, + "ghcr.io/ministryofjustice/devcontainer-feature/kubernetes:1": { + "version": "1.0.1", + "resolved": "ghcr.io/ministryofjustice/devcontainer-feature/kubernetes@sha256:0ec758e44468ba2a8b70b87613762ab04e50f7bb5eac8f2aea592cff213dbde5", + "integrity": "sha256:0ec758e44468ba2a8b70b87613762ab04e50f7bb5eac8f2aea592cff213dbde5" + }, + "ghcr.io/ministryofjustice/devcontainer-feature/static-analysis:1": { + "version": "1.0.0", + "resolved": "ghcr.io/ministryofjustice/devcontainer-feature/static-analysis@sha256:e81d52725655c8ffb861605feac7ad155b447d51af65f6c3a03cab32d59f1e16", + "integrity": "sha256:e81d52725655c8ffb861605feac7ad155b447d51af65f6c3a03cab32d59f1e16" + } + } +} \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index fa9f44e..8fbc8d4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,27 +1,31 @@ { - "name": "analytical-platform-ollamate", - "image": "ghcr.io/ministryofjustice/devcontainer-base:latest", - "features": { - "ghcr.io/devcontainers/features/node:1": { - "version": "20.11.1" - }, - "ghcr.io/devcontainers/features/python:1": { - "version": "3.12" - }, - "ghcr.io/devcontainers/features/docker-in-docker:2": {}, - "./features/src/postgresql": {}, - "ghcr.io/ministryofjustice/devcontainer-feature/aws:0": {} + "name": "analytical-platform-ollamate", + "image": "ghcr.io/ministryofjustice/devcontainer-base:latest", + "features": { + "ghcr.io/devcontainers/features/node:1": { + "version": "20.11.1" }, - "postCreateCommand": "bash scripts/devcontainer/post-create.sh", - "postStartCommand": "bash scripts/devcontainer/post-start.sh", - "runArgs": ["--name=analytical-platform-ollamate-devcontainer"], - "customizations": { - "vscode": { - "extensions": [ - "EditorConfig.EditorConfig", - "GitHub.vscode-github-actions", - "GitHub.vscode-codeql" - ] - } + "ghcr.io/devcontainers/features/python:1": { + "version": "3.12" + }, + "ghcr.io/devcontainers/features/docker-in-docker:2": {}, + "./features/src/postgresql": {}, + "ghcr.io/ministryofjustice/devcontainer-feature/aws:1": {}, + "ghcr.io/ministryofjustice/devcontainer-feature/container-structure-test:1": {}, + "ghcr.io/ministryofjustice/devcontainer-feature/kubernetes:1": {}, + "ghcr.io/ministryofjustice/devcontainer-feature/static-analysis:1": {} + }, + "postCreateCommand": "bash scripts/devcontainer/post-create.sh", + "postStartCommand": "bash scripts/devcontainer/post-start.sh", + "runArgs": ["--name=analytical-platform-ollamate-devcontainer"], + "customizations": { + "vscode": { + "extensions": [ + "EditorConfig.EditorConfig", + "GitHub.vscode-github-actions", + "GitHub.vscode-codeql", + "ms-vsliveshare.vsliveshare" + ] } - } \ No newline at end of file + } +} diff --git a/.editorconfig b/.editorconfig index d159d74..fa1752a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -5,6 +5,19 @@ end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true +# This file is autogenerated +[.devcontainer/devcontainer-lock.json] +end_of_line = unset +insert_final_newline = unset + +[*.json] +indent_style = space +indent_size = 2 + [*.sh] indent_style = space -indent_size = 2 \ No newline at end of file +indent_size = 2 + +[{*.yml,*.yaml}] +indent_style = space +indent_size = 2 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..ceeb99e --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 88 +extend-ignore = E203, E704 +exclude = + venv diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ac066e6..c516797 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1 @@ -# Add a team or username to this file -# Example: -# * @ministryofjustice/operations-engineering +@ministryofjustice/analytical-platforms diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 15fe7f0..f5613dd 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,21 +1,11 @@ --- -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file - version: 2 - updates: - - package-ecosystem: "bundler" + - package-ecosystem: "github-actions" directory: "/" schedule: interval: "daily" - - package-ecosystem: "terraform" - directory: "/terraform" - schedule: - interval: "daily" - - package-ecosystem: "github-actions" + - package-ecosystem: "devcontainers" directory: "/" schedule: interval: "daily" @@ -27,10 +17,6 @@ updates: directory: "/" schedule: interval: "daily" - - package-ecosystem: "gomod" - directory: "/" - schedule: - interval: "daily" - package-ecosystem: "docker" directory: "/" schedule: diff --git a/.github/workflows/chart-lint.yml b/.github/workflows/chart-lint.yml new file mode 100644 index 0000000..e7393be --- /dev/null +++ b/.github/workflows/chart-lint.yml @@ -0,0 +1,33 @@ +--- +name: Chart Lint + +on: + pull_request: + branches: + - main + +permissions: {} + +jobs: + chart-lint: + name: Chart Lint + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + id: checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Set Up Helm + id: setup_helm + uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 + + - name: Set Up Helm Chart Testing + id: setup_chart_testing + uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1 + + - name: Lint Chart + id: lint_chart + run: | + make ct diff --git a/.github/workflows/enforce-version-pinning.yml b/.github/workflows/enforce-version-pinning.yml new file mode 100644 index 0000000..5ded51f --- /dev/null +++ b/.github/workflows/enforce-version-pinning.yml @@ -0,0 +1,38 @@ +--- +name: Enforce Version Pinning + +on: + pull_request: + branches: + - main + +permissions: {} + +jobs: + enforce-version-pinning: + name: Enforce Version Pinning + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Check for pinned versions in requirements.txt + run: | + if grep -q -v '==' requirements.txt; then + echo "Unpinned dependencies found in requirements.txt" + echo "❌ Unpinned dependencies found in requirements.txt" + exit 1 + else + echo "✅ All dependencies are correctly pinned." + fi + + - name: Check for pinned versions in package.json + run: | + UNPINNED=$(grep -E '"[^"]+": "\^|~' package.json || true) + if [ -n "$UNPINNED" ]; then + echo "❌ Unpinned dependencies found in package.json:" + echo "$UNPINNED" + exit 1 + else + echo "✅ All dependencies are correctly pinned." + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a110715..feab62f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,54 +1,89 @@ --- -name: Release - -on: - push: - tags: - - '*.*.*' - -permissions: {} - -jobs: - release: - runs-on: ubuntu-latest - permissions: - contents: read - id-token: write - packages: write - steps: - - name: Checkout - id: checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - name: Install cosign - id: install-cosign - uses: sigstore/cosign-installer@1fc5bd396d372bee37d608f955b336615edf79c8 # v3.2.0 - - - name: Login to GitHub Container Registry - id: login - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Push - id: push - uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 - with: - context: . - push: true - tags: ghcr.io/${{ github.repository_owner }}/analytical-platform-ui:${{ github.ref_name }} - - - name: Sign - id: sign - run: | - cosign sign --yes ghcr.io/${{ github.repository_owner }}/analytical-platform-ui@${{ steps.push.outputs.digest }} - - - name: Verify - id: verify - run: | - cosign verify \ - --certificate-oidc-issuer=https://token.actions.githubusercontent.com \ - --certificate-identity=https://github.com/${{ github.repository_owner }}/analytical-platform-ui/.github/workflows/release.yml@refs/tags/${{ github.ref_name }} \ - ghcr.io/${{ github.repository_owner }}/analytical-platform-ui@${{ steps.push.outputs.digest }} + name: Release + + on: + push: + tags: + - "*" + + permissions: {} + + jobs: + release-image: + name: Release Image + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + packages: write + steps: + - name: Checkout + id: checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Install cosign + id: install_cosign + uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 # v3.5.0 + + - name: Log in to GitHub Container Registry + id: login_ghcr + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and Push + id: build_and_push + uses: docker/build-push-action@15560696de535e4014efeff63c48f16952e52dd1 # v6.2.0 + with: + push: true + tags: ghcr.io/${{ github.repository }}:${{ github.ref_name }} + + - name: Sign + id: sign + shell: bash + run: | + cosign sign --yes ghcr.io/${{ github.repository }}@${{ steps.build_and_push.outputs.digest }} + + - name: Verify + id: verify + run: | + cosign verify \ + --certificate-oidc-issuer=https://token.actions.githubusercontent.com \ + --certificate-identity=https://github.com/${{ github.repository }}/.github/workflows/release.yml@refs/tags/${{ github.ref_name }} \ + ghcr.io/${{ github.repository }}@${{ steps.build_and_push.outputs.digest }} + + release-chart: + name: Release Chart + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + packages: write + steps: + - name: Checkout + id: checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Set Up Helm + id: setup_helm + uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 + + - name: Log in to GitHub Container Registry + id: login_ghcr + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Package Chart + id: package_chart + run: | + helm package chart --destination .helm-deploy + + - name: Push Chart + id: push_chart + run: | + helm push .helm-deploy/analytical-platform-ollamate-${{ github.ref_name }}.tgz oci://ghcr.io/ministryofjustice/analytical-platform-charts diff --git a/.github/workflows/scan-image.yml b/.github/workflows/scan-image.yml index de72c32..406193e 100644 --- a/.github/workflows/scan-image.yml +++ b/.github/workflows/scan-image.yml @@ -1,57 +1,57 @@ --- -name: Scan Image + name: Scan Image -on: - pull_request: - branches: - - main + on: + pull_request: + branches: + - main -permissions: {} + permissions: {} -jobs: - scan-image: - name: Scan Image - runs-on: ubuntu-latest - permissions: - contents: read - security-events: write - steps: - - name: Checkout - id: checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + jobs: + scan-image: + name: Scan Image + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + steps: + - name: Checkout + id: checkout + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - name: Build Image - id: build_image - uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 - with: - push: false - load: true - tags: dashboard + - name: Build Image + id: build_image + uses: docker/build-push-action@15560696de535e4014efeff63c48f16952e52dd1 # v6.2.0 + with: + push: false + load: true + tags: ollamate - - name: Scan Image - id: scan_image - uses: aquasecurity/trivy-action@062f2592684a31eb3aa050cc61e7ca1451cecd3d # v0.18.0 - with: - image-ref: dashboard - exit-code: 1 - format: sarif - output: trivy-results.sarif - severity: CRITICAL - limit-severities-for-sarif: true + - name: Scan Image + id: scan_image + uses: aquasecurity/trivy-action@7c2007bcb556501da015201bcba5aa14069b74e2 # v0.23.0 + with: + image-ref: ollamate + exit-code: 1 + format: sarif + output: trivy-results.sarif + severity: CRITICAL + limit-severities-for-sarif: true - - name: Scan Image (On SARIF Scan Failure) - if: failure() && steps.scan_image.outcome == 'failure' - id: scan_image_on_failure - uses: aquasecurity/trivy-action@062f2592684a31eb3aa050cc61e7ca1451cecd3d # v0.18.0 - with: - image-ref: dashboard - exit-code: 1 - format: table - severity: CRITICAL + - name: Scan Image (On SARIF Scan Failure) + if: failure() && steps.scan_image.outcome == 'failure' + id: scan_image_on_failure + uses: aquasecurity/trivy-action@7c2007bcb556501da015201bcba5aa14069b74e2 # v0.23.0 + with: + image-ref: ollamate + exit-code: 1 + format: table + severity: CRITICAL - - name: Upload SARIF - if: always() - id: upload_sarif - uses: github/codeql-action/upload-sarif@cdcdbb579706841c47f7063dda365e292e5cad7a # v2.2.7 - with: - sarif_file: trivy-results.sarif + - name: Upload SARIF + if: always() + id: upload_sarif + uses: github/codeql-action/upload-sarif@b611370bb5703a7efb587f9d136a52ea24c5c38c # v2.2.7 + with: + sarif_file: trivy-results.sarif diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..5577601 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,45 @@ +--- +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - name: End of File Fixer + id: end-of-file-fixer + - name: Trailing Whitespace Fixer + id: trailing-whitespace + - name: Check yaml + id: check-yaml + - name: requirements.txt fixer + id: requirements-txt-fixer + + - repo: https://github.com/psf/black + rev: 23.10.1 + hooks: + - id: black + name: black formatting + + - repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + name: flake8 lint + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.6.1 + hooks: + - id: mypy + name: mypy + additional_dependencies: + - django-stubs + - psycopg2-binary + + - repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + name: isort (python) + + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.32.0 + hooks: + - id: yamllint diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c829c6d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,42 @@ +# FROM public.ecr.aws/docker/library/node:20.11.1 AS build-node + +# WORKDIR / +# COPY package.json package-lock.json ./ +# COPY assets/scss/app.scss ./assets/scss/app.scss + +# RUN npm install \ +# && npm run css + +FROM public.ecr.aws/docker/library/python:3.12-alpine3.19 AS base + +RUN apk add --no-cache --virtual .build-deps \ + libffi-dev=3.4.4-r3 \ + gcc=13.2.1_git20231014-r0 \ + musl-dev=1.2.4_git20230717-r4 \ + && apk add --no-cache libpq-dev=16.3-r0 + +WORKDIR /ollamate + +RUN mkdir --parents static/assets/fonts \ + && mkdir --parents static/assets/images \ + && mkdir --parents static/assets/js + +# COPY --from=build-node static/app.css static/app.css +# COPY --from=build-node node_modules/govuk-frontend/dist/govuk/assets/fonts/. static/assets/fonts +# COPY --from=build-node node_modules/govuk-frontend/dist/govuk/assets/images/. static/assets/images +# COPY --from=build-node node_modules/govuk-frontend/dist/govuk/all.bundle.js static/assets/js/govuk.js +COPY scripts/container/entrypoint.sh /usr/local/bin/entrypoint.sh +COPY requirements.txt manage.py ./ +COPY ollamate ollamate +COPY tests tests + +RUN pip install --no-cache-dir --requirement requirements.txt \ + && chmod +x /usr/local/bin/entrypoint.sh \ + && python manage.py collectstatic --noinput --ignore=*.scss \ + && apk del .build-deps + +EXPOSE 8000 + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] + +CMD ["run"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c3916a7 --- /dev/null +++ b/Makefile @@ -0,0 +1,21 @@ +#!make + +db-migrate: + python manage.py migrate + +db-drop: + python manage.py reset_db + +serve: + python manage.py runserver + +container: + docker build -t ollamate . + +test: container + @echo + @echo "> Running Python Tests (In Docker)..." + IMAGE_TAG=ollamate docker compose --file=contrib/docker-compose-test.yml run --rm interfaces + +ct: + ct lint --charts chart diff --git a/README.md b/README.md index a6fbc77..9eb1f26 100644 --- a/README.md +++ b/README.md @@ -2,65 +2,16 @@ [![repo standards badge](https://img.shields.io/endpoint?labelColor=231f20&color=005ea5&style=for-the-badge&label=MoJ%20Compliant&url=https%3A%2F%2Foperations-engineering-reports.cloud-platform.service.justice.gov.uk%2Fapi%2Fv1%2Fcompliant_public_repositories%2Fendpoint%2Ftemplate-repository&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACgAAAAoCAYAAACM/rhtAAAABmJLR0QA/wD/AP+gvaeTAAAHJElEQVRYhe2YeYyW1RWHnzuMCzCIglBQlhSV2gICKlHiUhVBEAsxGqmVxCUUIV1i61YxadEoal1SWttUaKJNWrQUsRRc6tLGNlCXWGyoUkCJ4uCCSCOiwlTm6R/nfPjyMeDY8lfjSSZz3/fee87vnnPu75z3g8/kM2mfqMPVH6mf35t6G/ZgcJ/836Gdug4FjgO67UFn70+FDmjcw9xZaiegWX29lLLmE3QV4Glg8x7WbFfHlFIebS/ANj2oDgX+CXwA9AMubmPNvuqX1SnqKGAT0BFoVE9UL1RH7nSCUjYAL6rntBdg2Q3AgcAo4HDgXeBAoC+wrZQyWS3AWcDSUsomtSswEtgXaAGWlVI2q32BI0spj9XpPww4EVic88vaC7iq5Hz1BvVf6v3qe+rb6ji1p3pWrmtQG9VD1Jn5br+Knmm70T9MfUh9JaPQZu7uLsR9gEsJb3QF9gOagO7AuUTom1LpCcAkoCcwQj0VmJregzaipA4GphNe7w/MBearB7QLYCmlGdiWSm4CfplTHwBDgPHAFmB+Ah8N9AE6EGkxHLhaHU2kRhXc+cByYCqROs05NQq4oR7Lnm5xE9AL+GYC2gZ0Jmjk8VLKO+pE4HvAyYRnOwOH5N7NhMd/WKf3beApYBWwAdgHuCLn+tatbRtgJv1awhtd838LEeq30/A7wN+AwcBt+bwpD9AdOAkYVkpZXtVdSnlc7QI8BlwOXFmZ3oXkdxfidwmPrQXeA+4GuuT08QSdALxC3OYNhBe/TtzON4EziZBXD36o+q082BxgQuqvyYL6wtBY2TyEyJ2DgAXAzcC1+Xxw3RlGqiuJ6vE6QS9VGZ/7H02DDwAvELTyMDAxbfQBvggMAAYR9LR9J2cluH7AmnzuBowFFhLJ/wi7yiJgGXBLPq8A7idy9kPgvAQPcC9wERHSVcDtCfYj4E7gr8BRqWMjcXmeB+4tpbyG2kG9Sl2tPqF2Uick8B+7szyfvDhR3Z7vvq/2yqpynnqNeoY6v7LvevUU9QN1fZ3OTeppWZmeyzRoVu+rhbaHOledmoQ7LRd3SzBVeUo9Wf1DPs9X90/jX8m/e9Rn1Mnqi7nuXXW5+rK6oU7n64mjszovxyvVh9WeDcTVnl5KmQNcCMwvpbQA1xE8VZXhwDXAz4FWIkfnAlcBAwl6+SjD2wTcmPtagZnAEuA3dTp7qyNKKe8DW9UeBCeuBsbsWKVOUPvn+MRKCLeq16lXqLPVFvXb6r25dlaGdUx6cITaJ8fnpo5WI4Wuzcjcqn5Y8eI/1F+n3XvUA1N3v4ZamIEtpZRX1Y6Z/DUK2g84GrgHuDqTehpBCYend94jbnJ34DDgNGArQT9bict3Y3p1ZCnlSoLQb0sbgwjCXpY2blc7llLW1UAMI3o5CD4bmuOlwHaC6xakgZ4Z+ibgSxnOgcAI4uavI27jEII7909dL5VSrimlPKgeQ6TJCZVQjwaOLaW8BfyWbPEa1SaiTH1VfSENd85NDxHt1plA71LKRvX4BDaAKFlTgLeALtliDUqPrSV6SQCBlypgFlbmIIrCDcAl6nPAawmYhlLKFuB6IrkXAadUNj6TXlhDcCNEB/Jn4FcE0f4UWEl0NyWNvZxGTs89z6ZnatIIrCdqcCtRJmcCPwCeSN3N1Iu6T4VaFhm9n+riypouBnepLsk9p6p35fzwvDSX5eVQvaDOzjnqzTl+1KC53+XzLINHd65O6lD1DnWbepPBhQ3q2jQyW+2oDkkAtdt5udpb7W+Q/OFGA7ol1zxu1tc8zNHqXercfDfQIOZm9fR815Cpt5PnVqsr1F51wI9QnzU63xZ1o/rdPPmt6enV6sXqHPVqdXOCe1rtrg5W7zNI+m712Ir+cer4POiqfHeJSVe1Raemwnm7xD3mD1E/Z3wIjcsTdlZnqO8bFeNB9c30zgVG2euYa69QJ+9G90lG+99bfdIoo5PU4w362xHePxl1slMab6tV72KUxDvzlAMT8G0ZohXq39VX1bNzzxij9K1Qb9lhdGe931B/kR6/zCwY9YvuytCsMlj+gbr5SemhqkyuzE8xau4MP865JvWNuj0b1YuqDkgvH2GkURfakly01Cg7Cw0+qyXxkjojq9Lw+vT2AUY+DlF/otYq1Ixc35re2V7R8aTRg2KUv7+ou3x/14PsUBn3NG51S0XpG0Z9PcOPKWSS0SKNUo9Rv2Mmt/G5WpPF6pHGra7Jv410OVsdaz217AbkAPX3ubkm240belCuudT4Rp5p/DyC2lf9mfq1iq5eFe8/lu+K0YrVp0uret4nAkwlB6vzjI/1PxrlrTp/oNHbzTJI92T1qAT+BfW49MhMg6JUp7ehY5a6Tl2jjmVvitF9fxo5Yq8CaAfAkzLMnySt6uz/1k6bPx59CpCNxGfoSKA30IPoH7cQXdArwCOllFX/i53P5P9a/gNkKpsCMFRuFAAAAABJRU5ErkJggg==)](https://operations-engineering-reports.cloud-platform.service.justice.gov.uk/public-report/template-repository) -This template repository equips you with the default initial files required for a Ministry of Justice GitHub repository. +> [!NOTE] +> This repository is a proof of concept application. -## Included Files +## Analytical Platform Ollamate 🦙🧉 -The repository comes with the following preset files: +This repository includes a Django application which presents a simple frontend to a user which allows them to consume an LLM running from Ollama. -- LICENSE -- .gitignore -- CODEOWNERS -- dependabot.yml -- GitHub Actions example files -- Ministry of Justice Compliance Badge (public repositories only) +The name 'Ollamate' is derived from Ollama and will be changed if/when this application evolves. -## Setup Instructions +## Getting Started -Once you've created your repository using this template, ensure the following steps: - -### Update README - -Edit this README.md file to document your project accurately. Take the time to create a clear, engaging, and informative README.md file. Include information like what your project does, how to install and run it, how to contribute, and any other pertinent details. - -### Update repository description - -After you've created your repository, GitHub provides a brief description field that appears on the top of your repository's main page. This is a summary that gives visitors quick insight into the project. Using this field to provide a succinct overview of your repository is highly recommended. - -This description and your README.md will be one of the first things people see when they visit your repository. It's a good place to make a strong, concise first impression. Remember, this is often visible in search results on GitHub and search engines, so it's also an opportunity to help people discover your project. - -### Grant Team Permissions - -Assign permissions to the appropriate Ministry of Justice teams. Ensure at least one team is granted Admin permissions. Whenever possible, assign permissions to teams rather than individual users. - -### Read about the GitHub repository standards - -Familiarise yourself with the Ministry of Justice GitHub Repository Standards. These standards ensure consistency, maintainability, and best practices across all our repositories. - -You can find the standards [here](https://user-guide.operations-engineering.service.justice.gov.uk/documentation/information/mojrepostandards.html). - -Please read and understand these standards thoroughly and enable them when you feel comfortable. - -### Modify the GitHub Standards Badge - -Once you've ensured that all the [GitHub Repository Standards](https://user-guide.operations-engineering.service.justice.gov.uk/documentation/information/mojrepostandards.html) have been applied to your repository, it's time to update the Ministry of Justice (MoJ) Compliance Badge located in the README file. - -The badge demonstrates that your repository is compliant with MoJ's standards. Please follow these [instructions](https://user-guide.operations-engineering.service.justice.gov.uk/documentation/information/add-repo-badge.html) to modify the badge URL to reflect the status of your repository correctly. - -**Please note** the badge will not function correctly if your repository is internal or private. In this case, you may remove the badge from your README. - -### Manage Outside Collaborators - -To add an Outside Collaborator to the repository, follow the guidelines detailed [here](https://github.com/ministryofjustice/github-collaborators). - -### Update CODEOWNERS - -(Optional) Modify the CODEOWNERS file to specify the teams or users authorized to approve pull requests. - -### Configure Dependabot - -Adapt the dependabot.yml file to match your project's [dependency manager](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem) and to enable [automated pull requests for package updates](https://docs.github.com/en/code-security/supply-chain-security). - -### Dependency Review - -If your repository is private with no GitHub Advanced Security license, remove the `.github/workflows/dependency-review.yml` file. +> [!NOTE] +> 🚧 WIP 🚧 diff --git a/chart/.helmignore b/chart/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/chart/Chart.yaml b/chart/Chart.yaml new file mode 100644 index 0000000..45b5c66 --- /dev/null +++ b/chart/Chart.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v2 +name: analytical-platform-ollamate +description: Analytical Platform Ollamate +type: application +version: 0.0.1 +appVersion: 0.0.1 +icon: https://upload.wikimedia.org/wikipedia/en/thumb/4/4a/Ministry_of_Justice_logo_%28United_Kingdom%29.svg/611px-Ministry_of_Justice_logo_%28United_Kingdom%29.svg.png +maintainers: + - name: moj-data-platform-robot + email: analytical-platform@digital.justice.gov.uk diff --git a/chart/ci/lint-values.yaml b/chart/ci/lint-values.yaml new file mode 100644 index 0000000..e69de29 diff --git a/chart/templates/.helpers.tpl b/chart/templates/.helpers.tpl new file mode 100644 index 0000000..3e33227 --- /dev/null +++ b/chart/templates/.helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "actions-runner.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "actions-runner.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "actions-runner.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "actions-runner.labels" -}} +helm.sh/chart: {{ include "actions-runner.chart" . }} +{{ include "actions-runner.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "actions-runner.selectorLabels" -}} +app.kubernetes.io/name: {{ include "actions-runner.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "actions-runner.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "actions-runner.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/chart/templates/deployment.yml b/chart/templates/deployment.yml new file mode 100644 index 0000000..582d8c6 --- /dev/null +++ b/chart/templates/deployment.yml @@ -0,0 +1,49 @@ +{{- if not .Values.ephemeral.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "actions-runner.fullname" . }} + labels: + {{- include "actions-runner.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "actions-runner.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "actions-runner.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "actions-runner.serviceAccountName" . }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: GITHUB_REPOSITORY + value: {{ .Values.github.organisation }}/{{ .Values.github.repository }} + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-github-token + key: token + - name: RUNNER_LABELS + value: {{ .Values.github.runner.labels | quote }} + resources: + {{- toYaml .Values.resources | nindent 12 }} +{{- end }} diff --git a/chart/templates/scaled-job.yaml b/chart/templates/scaled-job.yaml new file mode 100644 index 0000000..582d8c6 --- /dev/null +++ b/chart/templates/scaled-job.yaml @@ -0,0 +1,49 @@ +{{- if not .Values.ephemeral.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "actions-runner.fullname" . }} + labels: + {{- include "actions-runner.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "actions-runner.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "actions-runner.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "actions-runner.serviceAccountName" . }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: GITHUB_REPOSITORY + value: {{ .Values.github.organisation }}/{{ .Values.github.repository }} + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-github-token + key: token + - name: RUNNER_LABELS + value: {{ .Values.github.runner.labels | quote }} + resources: + {{- toYaml .Values.resources | nindent 12 }} +{{- end }} diff --git a/chart/templates/secret-account.yaml b/chart/templates/secret-account.yaml new file mode 100644 index 0000000..e207736 --- /dev/null +++ b/chart/templates/secret-account.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "actions-runner.serviceAccountName" . }} + labels: + {{- include "actions-runner.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} diff --git a/chart/templates/secret.yaml b/chart/templates/secret.yaml new file mode 100644 index 0000000..352ffd0 --- /dev/null +++ b/chart/templates/secret.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-github-token + labels: + {{- include "actions-runner.labels" . | nindent 4 }} +type: Opaque +data: + token: {{ .Values.github.token | b64enc }} diff --git a/chart/values.yaml b/chart/values.yaml new file mode 100644 index 0000000..033b5eb --- /dev/null +++ b/chart/values.yaml @@ -0,0 +1,49 @@ +--- +nameOverride: "" +fullnameOverride: "" + +replicaCount: 1 + +image: + pullPolicy: IfNotPresent + repository: ghcr.io/ministryofjustice/analytical-platform-ollamate + tag: 0.0.1 + +imagePullSecrets: [] + +serviceAccount: + create: true + automount: true + name: "" + annotations: {} + +podAnnotations: {} +podLabels: {} + +securityContext: + runAsNonRoot: true + runAsUser: 10000 + +resources: + requests: + cpu: 1 + memory: "5Gi" + limits: + cpu: 2 + memory: "7Gi" + +github: + organisation: + repository: + token: + runner: + labels: + +ephemeral: + enabled: false + karpenter: + enabled: false + nodePool: "general-on-demand" + keda: + maxReplicaCount: 5 + pollingInterval: 30 diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..576b950 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,33 @@ +upstream django { + server localhost:8000; +} + +server { + listen 8080; + + location /nginx-health { + access_log off; + return 200 "healthy\n"; + } + + location /static/ { + root /usr/share/nginx/html; + } + + location / { + try_files $uri @proxy_to_django; + } + + location @proxy_to_django { + proxy_pass http://django; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $server_name; + } +} diff --git a/ollamate/settings.py b/ollamate/settings.py index c02d8e7..ea65907 100644 --- a/ollamate/settings.py +++ b/ollamate/settings.py @@ -47,6 +47,7 @@ ALLOWED_HOSTS = [] +PROJECT_NAME = "ollamate" # Application definition @@ -95,13 +96,25 @@ # Database # https://docs.djangoproject.com/en/5.0/ref/settings/#databases -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': BASE_DIR / 'db.sqlite3', +DB_HOST = os.environ.get("DB_HOST", "127.0.0.1") +ENABLE_DB_SSL = ( + str(os.environ.get("ENABLE_DB_SSL", DB_HOST not in ["127.0.0.1", "localhost"])).lower() + == "true" +) +DATABASES: dict = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": os.environ.get("DB_NAME", PROJECT_NAME), + "USER": os.environ.get("DB_USER", ""), + "PASSWORD": os.environ.get("DB_PASSWORD", ""), + "HOST": DB_HOST, + "PORT": os.environ.get("DB_PORT", "5432"), } } +if ENABLE_DB_SSL: + DATABASES["default"]["OPTIONS"] = {"sslmode": "require"} + # Password validation # https://docs.djangoproject.com/en/5.0/ref/settings/#auth-password-validators diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..ace255c --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "analytical-platform-ollamate", + "lockfileVersion": 3, + "requires": true, + "packages": {} +} diff --git a/requirements.dev.txt b/requirements.dev.txt new file mode 100644 index 0000000..c2497dd --- /dev/null +++ b/requirements.dev.txt @@ -0,0 +1,7 @@ +-r ./requirements.txt +black==23.10.1 +django-stubs[compatible-mypy]==4.2.6 +flake8==6.1.0 +isort==5.12.0 +mypy==1.6.1 +pre-commit==3.5.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..509a6ab --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +django-azure-auth==2.0.0 +django-environ==0.11.2 +requests==2.32.3 +psycopg==3.2.1 diff --git a/scripts/container/entrypoint.sh b/scripts/container/entrypoint.sh new file mode 100644 index 0000000..157e723 --- /dev/null +++ b/scripts/container/entrypoint.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env sh + +MODE=${MODE:-"run"} +ADDRESS=${ADDRESS:-"0.0.0.0"} +PORT=${PORT:-"8000"} +WORKERS=${WORKERS:-"4"} + +case "$MODE" in +"run") + echo "Running Django server on ${ADDRESS}:${PORT}" + gunicorn -b "${ADDRESS}":"${PORT}" -k uvicorn.workers.UvicornWorker -w "${WORKERS}" ap.asgi:application + ;; +"migrate") + echo "Running Django migrations" + python manage.py migrate + ;; +*) + echo "Unknown mode: ${MODE}" + exit 1 + ;; +esac diff --git a/scripts/devcontainer/post-create.sh b/scripts/devcontainer/post-create.sh new file mode 100644 index 0000000..f412ff3 --- /dev/null +++ b/scripts/devcontainer/post-create.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Upgrade NPM +npm install --global npm@latest + +# Start Postgres +docker compose --file contrib/docker-compose-postgres.yml up --detach + +# Upgrade Pip +pip install --break-system-package --upgrade pip + +# Install dependencies +pip install --break-system-package --requirement requirements.dev.txt + +# install npm dependencies and static assets +# npm install +# make build-static + +# Run migrations +python manage.py migrate diff --git a/scripts/devcontainer/post-start.sh b/scripts/devcontainer/post-start.sh new file mode 100644 index 0000000..f0289d1 --- /dev/null +++ b/scripts/devcontainer/post-start.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# Run pre-commit +pre-commit install From 5dcff781ed2253353be558f08f1bf8e990b6ee5e Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:06:00 +0000 Subject: [PATCH 05/39] :wrench: Add package.json --- package.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 package.json diff --git a/package.json b/package.json new file mode 100644 index 0000000..4daec64 --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "name": "analytical-platform-ollamate", + "version": "1.0.0", + "description": "Analytical Platform Ollamate" +} From 249b5387afaafbc3fd7576627f1cdc7cc80b6891 Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:06:19 +0000 Subject: [PATCH 06/39] :fire: Comment out tests in Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c829c6d..1210e52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ RUN mkdir --parents static/assets/fonts \ COPY scripts/container/entrypoint.sh /usr/local/bin/entrypoint.sh COPY requirements.txt manage.py ./ COPY ollamate ollamate -COPY tests tests +# COPY tests tests RUN pip install --no-cache-dir --requirement requirements.txt \ && chmod +x /usr/local/bin/entrypoint.sh \ From b1b7e930420fab706aabd09097c673a794b3d492 Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:07:51 +0000 Subject: [PATCH 07/39] :alert: black linting --- ollamate/asgi.py | 2 +- ollamate/settings.py | 103 ++++++++++++++++++++++-------------------- ollamate/urls.py | 11 +++-- ollamate/wsgi.py | 2 +- streamingapp/apps.py | 4 +- streamingapp/urls.py | 2 +- streamingapp/views.py | 32 ++++++------- 7 files changed, 81 insertions(+), 75 deletions(-) diff --git a/ollamate/asgi.py b/ollamate/asgi.py index 0dcb726..1f3288b 100644 --- a/ollamate/asgi.py +++ b/ollamate/asgi.py @@ -11,6 +11,6 @@ from django.core.asgi import get_asgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ollamate.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ollamate.settings") application = get_asgi_application() diff --git a/ollamate/settings.py b/ollamate/settings.py index ea65907..4587455 100644 --- a/ollamate/settings.py +++ b/ollamate/settings.py @@ -22,7 +22,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Read the .env file -env_file = os.path.join(BASE_DIR, '.env') +env_file = os.path.join(BASE_DIR, ".env") if os.path.isfile(env_file): print(f"Loading environment variables from {env_file}") environ.Env.read_env(env_file) @@ -40,7 +40,7 @@ # See https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'django-insecure-(y%c-1p0lnlrxg!$3w)ptcyp=wzer(biav_-%_fehgn1oatx8p' +SECRET_KEY = "django-insecure-(y%c-1p0lnlrxg!$3w)ptcyp=wzer(biav_-%_fehgn1oatx8p" # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True @@ -52,45 +52,45 @@ # Application definition INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'streamingapp', - 'azure_auth' + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "streamingapp", + "azure_auth", ] MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", ] -ROOT_URLCONF = 'ollamate.urls' +ROOT_URLCONF = "ollamate.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", ], }, }, ] -WSGI_APPLICATION = 'ollamate.wsgi.application' +WSGI_APPLICATION = "ollamate.wsgi.application" # Database @@ -98,7 +98,9 @@ DB_HOST = os.environ.get("DB_HOST", "127.0.0.1") ENABLE_DB_SSL = ( - str(os.environ.get("ENABLE_DB_SSL", DB_HOST not in ["127.0.0.1", "localhost"])).lower() + str( + os.environ.get("ENABLE_DB_SSL", DB_HOST not in ["127.0.0.1", "localhost"]) + ).lower() == "true" ) DATABASES: dict = { @@ -121,16 +123,16 @@ AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, ] @@ -138,9 +140,9 @@ # Internationalization # https://docs.djangoproject.com/en/5.0/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" -TIME_ZONE = 'UTC' +TIME_ZONE = "UTC" USE_I18N = True @@ -150,12 +152,12 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/5.0/howto/static-files/ -STATIC_URL = 'static/' +STATIC_URL = "static/" # Default primary key field type # https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field -DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" # Azure authentication settings @@ -164,21 +166,22 @@ "CLIENT_SECRET": env("CLIENT_SECRET"), "REDIRECT_URI": env("REDIRECT_URI"), "SCOPES": ["User.Read"], - "AUTHORITY": "https://login.microsoftonline.com/{}".format(env("AZURE_TENANT_ID")), # Or https://login.microsoftonline.com/common if multi-tenant -# # "LOGOUT_URI": "https:///logout", # Optional -# # "PUBLIC_URLS": ["",], # Optional, public views accessible by non-authenticated users -# # "PUBLIC_PATHS": ['/go/',], # Optional, public paths accessible by non-authenticated users -# # "ROLES": { -# # "95170e67-2bbf-4e3e-a4d7-e7e5829fe7a7": "GroupName1", -# # "3dc6539e-0589-4663-b782-fef100d839aa": "GroupName2" -# # }, # Optional, will add user to django group if user is in EntraID group - "USERNAME_ATTRIBUTE": "mail", # The AAD attribute or ID token claim you want to use as the value for the user model `USERNAME_FIELD` -# # "EXTRA_FIELDS": [], # Optional, extra AAD user profile attributes you want to make available in the user mapping function -# "USER_MAPPING_FN": "azure_auth.tests.misc.user_mapping_fn", # Optional, path to the function used to map the AAD to Django attributes + "AUTHORITY": "https://login.microsoftonline.com/{}".format( + env("AZURE_TENANT_ID") + ), # Or https://login.microsoftonline.com/common if multi-tenant + # # "LOGOUT_URI": "https:///logout", # Optional + # # "PUBLIC_URLS": ["",], # Optional, public views accessible by non-authenticated users + # # "PUBLIC_PATHS": ['/go/',], # Optional, public paths accessible by non-authenticated users + # # "ROLES": { + # # "95170e67-2bbf-4e3e-a4d7-e7e5829fe7a7": "GroupName1", + # # "3dc6539e-0589-4663-b782-fef100d839aa": "GroupName2" + # # }, # Optional, will add user to django group if user is in EntraID group + "USERNAME_ATTRIBUTE": "mail", # The AAD attribute or ID token claim you want to use as the value for the user model `USERNAME_FIELD` + # # "EXTRA_FIELDS": [], # Optional, extra AAD user profile attributes you want to make available in the user mapping function + # "USER_MAPPING_FN": "azure_auth.tests.misc.user_mapping_fn", # Optional, path to the function used to map the AAD to Django attributes } LOGIN_URL = "/azure_auth/login" LOGIN_REDIRECT_URL = "/" -LOGOUT_REDIRECT_URL = '/' +LOGOUT_REDIRECT_URL = "/" AUTHENTICATION_BACKENDS = ("azure_auth.backends.AzureBackend",) - diff --git a/ollamate/urls.py b/ollamate/urls.py index b37c2b0..ed41375 100644 --- a/ollamate/urls.py +++ b/ollamate/urls.py @@ -19,8 +19,11 @@ from streamingapp.views import redirect_to_ollama urlpatterns = [ - path('admin/', admin.site.urls), - path("azure_auth/", include("azure_auth.urls"),), - path('stream/', include('streamingapp.urls')), - path('', redirect_to_ollama) + path("admin/", admin.site.urls), + path( + "azure_auth/", + include("azure_auth.urls"), + ), + path("stream/", include("streamingapp.urls")), + path("", redirect_to_ollama), ] diff --git a/ollamate/wsgi.py b/ollamate/wsgi.py index a07ca1e..ba2312a 100644 --- a/ollamate/wsgi.py +++ b/ollamate/wsgi.py @@ -11,6 +11,6 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ollamate.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ollamate.settings") application = get_wsgi_application() diff --git a/streamingapp/apps.py b/streamingapp/apps.py index 02796ae..31356e5 100644 --- a/streamingapp/apps.py +++ b/streamingapp/apps.py @@ -2,5 +2,5 @@ class StreamingappConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'streamingapp' + default_auto_field = "django.db.models.BigAutoField" + name = "streamingapp" diff --git a/streamingapp/urls.py b/streamingapp/urls.py index 1d03544..4175580 100644 --- a/streamingapp/urls.py +++ b/streamingapp/urls.py @@ -2,5 +2,5 @@ from . import views urlpatterns = [ - path('call-ollama/', views.call_ollama, name='call_ollama'), + path("call-ollama/", views.call_ollama, name="call_ollama"), ] diff --git a/streamingapp/views.py b/streamingapp/views.py index 9d3a2be..662a79a 100644 --- a/streamingapp/views.py +++ b/streamingapp/views.py @@ -8,26 +8,23 @@ # Configure logging logging.basicConfig(level=logging.DEBUG) + @azure_auth_required def call_ollama(request): - if request.method == 'POST': - user_input = request.POST.get('userInput', '') + if request.method == "POST": + user_input = request.POST.get("userInput", "") try: conversation_history = json.loads(user_input) except json.JSONDecodeError as e: logging.error("JSONDecodeError: %s", e) return JsonResponse({"error": "Invalid input format"}, status=400) - url = 'http://localhost:11434/api/chat' - headers = {'Content-Type': 'application/json'} - data = { - "model": "llama3", - "messages": conversation_history, - "stream": False - } - + url = "http://localhost:11434/api/chat" + headers = {"Content-Type": "application/json"} + data = {"model": "llama3", "messages": conversation_history, "stream": False} + logging.debug("Sending data to Ollama API: %s", json.dumps(data, indent=2)) - + try: response = requests.post(url, json=data, headers=headers) response.raise_for_status() @@ -37,14 +34,17 @@ def call_ollama(request): ollama_response = response_data.get("message", {}).get("content", "") if not ollama_response: logging.error("Empty response from Ollama API") - return JsonResponse({"error": "Empty response from Ollama API"}, status=500) - + return JsonResponse( + {"error": "Empty response from Ollama API"}, status=500 + ) + return JsonResponse({"response": ollama_response}) except requests.RequestException as e: logging.error("RequestException: %s", e) return JsonResponse({"error": str(e)}, status=500) else: - return render(request, 'streamingapp/input_form.html') - + return render(request, "streamingapp/input_form.html") + + def redirect_to_ollama(request): - return redirect('/stream/call-ollama') \ No newline at end of file + return redirect("/stream/call-ollama") From eb41f41a1807c890061c01fcda5bcf2841483b53 Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:11:22 +0000 Subject: [PATCH 08/39] :wrench: Add linting config files --- mypy.ini | 2 ++ pyproject.toml | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 mypy.ini create mode 100644 pyproject.toml diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..976ba02 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +ignore_missing_imports = True diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9bcbe97 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[tool.black] +line-length = 100 +target-version = ["py311"] +extend-exclude = ''' +/( + \.git + env + venv + | migrations +)/ +''' + + +[tool.isort] +profile = 'black' +known_django = "django" +sections = "FUTURE,STDLIB,DJANGO,THIRDPARTY,FIRSTPARTY,LOCALFOLDER" +line_length = 100 +multi_line_output = 3 From 8805bfc7698a9d81cf794beb14b9c326ab7e9505 Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:29:23 +0000 Subject: [PATCH 09/39] :alert: linting --- ollamate/settings.py | 9 +++------ ollamate/urls.py | 18 +----------------- streamingapp/urls.py | 1 + streamingapp/views.py | 4 +--- 4 files changed, 6 insertions(+), 26 deletions(-) diff --git a/ollamate/settings.py b/ollamate/settings.py index 4587455..2103b0c 100644 --- a/ollamate/settings.py +++ b/ollamate/settings.py @@ -9,11 +9,10 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/5.0/ref/settings/ """ - -from pathlib import Path - import os import environ +from pathlib import Path + # Initialize environment variables env = environ.Env() @@ -98,9 +97,7 @@ DB_HOST = os.environ.get("DB_HOST", "127.0.0.1") ENABLE_DB_SSL = ( - str( - os.environ.get("ENABLE_DB_SSL", DB_HOST not in ["127.0.0.1", "localhost"]) - ).lower() + str(os.environ.get("ENABLE_DB_SSL", DB_HOST not in ["127.0.0.1", "localhost"])).lower() == "true" ) DATABASES: dict = { diff --git a/ollamate/urls.py b/ollamate/urls.py index ed41375..0508610 100644 --- a/ollamate/urls.py +++ b/ollamate/urls.py @@ -1,21 +1,5 @@ -""" -URL configuration for ollamate project. - -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/5.0/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: path('', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.urls import include, path - 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) -""" from django.contrib import admin -from django.urls import path, include +from django.urls import include, path from streamingapp.views import redirect_to_ollama urlpatterns = [ diff --git a/streamingapp/urls.py b/streamingapp/urls.py index 4175580..01c2651 100644 --- a/streamingapp/urls.py +++ b/streamingapp/urls.py @@ -1,4 +1,5 @@ from django.urls import path + from . import views urlpatterns = [ diff --git a/streamingapp/views.py b/streamingapp/views.py index 662a79a..5ecaa7e 100644 --- a/streamingapp/views.py +++ b/streamingapp/views.py @@ -34,9 +34,7 @@ def call_ollama(request): ollama_response = response_data.get("message", {}).get("content", "") if not ollama_response: logging.error("Empty response from Ollama API") - return JsonResponse( - {"error": "Empty response from Ollama API"}, status=500 - ) + return JsonResponse({"error": "Empty response from Ollama API"}, status=500) return JsonResponse({"response": ollama_response}) except requests.RequestException as e: From 166d1679b9aff2dc306f0ccd1473c7da24734255 Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:32:30 +0000 Subject: [PATCH 10/39] Add new lines --- .devcontainer/features/src/postgresql/devcontainer-feature.json | 2 +- .devcontainer/features/src/postgresql/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/features/src/postgresql/devcontainer-feature.json b/.devcontainer/features/src/postgresql/devcontainer-feature.json index d8487e3..3adef92 100644 --- a/.devcontainer/features/src/postgresql/devcontainer-feature.json +++ b/.devcontainer/features/src/postgresql/devcontainer-feature.json @@ -3,4 +3,4 @@ "version": "1.0.0", "name": "postgresql", "description": "PostgreSQL" - } \ No newline at end of file + } diff --git a/.devcontainer/features/src/postgresql/install.sh b/.devcontainer/features/src/postgresql/install.sh index 7be223b..4bfb6d8 100644 --- a/.devcontainer/features/src/postgresql/install.sh +++ b/.devcontainer/features/src/postgresql/install.sh @@ -11,4 +11,4 @@ apt-get -y install \ postgresql-client-common \ postgresql-15 \ postgresql-client-15 \ - libpq-dev \ No newline at end of file + libpq-dev From 3623040efdc04ef5ad39d7810af870261fc4147b Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:34:13 +0000 Subject: [PATCH 11/39] :alert: Linting --- ollamate/urls.py | 1 + streamingapp/views.py | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ollamate/urls.py b/ollamate/urls.py index 0508610..e0e3e65 100644 --- a/ollamate/urls.py +++ b/ollamate/urls.py @@ -1,5 +1,6 @@ from django.contrib import admin from django.urls import include, path + from streamingapp.views import redirect_to_ollama urlpatterns = [ diff --git a/streamingapp/views.py b/streamingapp/views.py index 5ecaa7e..e1f34e0 100644 --- a/streamingapp/views.py +++ b/streamingapp/views.py @@ -1,10 +1,13 @@ -from django.shortcuts import render, redirect -from django.http import JsonResponse -from azure_auth.decorators import azure_auth_required -import requests import logging import json +from django.http import JsonResponse +from django.shortcuts import render, redirect + +import requests +from azure_auth.decorators import azure_auth_required + + # Configure logging logging.basicConfig(level=logging.DEBUG) From f42007c0dc89f5476e7cf709a64d8d3e45cc9c52 Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:41:39 +0000 Subject: [PATCH 12/39] Linting --- ollamate/settings.py | 5 +++-- streamingapp/views.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ollamate/settings.py b/ollamate/settings.py index 2103b0c..fd6a9b6 100644 --- a/ollamate/settings.py +++ b/ollamate/settings.py @@ -10,9 +10,10 @@ https://docs.djangoproject.com/en/5.0/ref/settings/ """ import os -import environ from pathlib import Path +import environ + # Initialize environment variables env = environ.Env() @@ -44,7 +45,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = [""] PROJECT_NAME = "ollamate" diff --git a/streamingapp/views.py b/streamingapp/views.py index e1f34e0..8b90fc7 100644 --- a/streamingapp/views.py +++ b/streamingapp/views.py @@ -1,13 +1,13 @@ -import logging import json +import logging + from django.http import JsonResponse -from django.shortcuts import render, redirect +from django.shortcuts import redirect, render import requests from azure_auth.decorators import azure_auth_required - # Configure logging logging.basicConfig(level=logging.DEBUG) From 2569fdf8f738e9e0b226ef1e4288c1dbc6f171f4 Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:45:37 +0000 Subject: [PATCH 13/39] :wrench: Update flake8 configuration --- .flake8 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index ceeb99e..c815588 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,7 @@ [flake8] -max-line-length = 88 +max-line-length = 100 extend-ignore = E203, E704 exclude = venv +per-file-ignores = + ap/*/migrations/*:E501,W292 From 39ae1b0bf5d9653232dc8293157bd98c66214f59 Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:50:28 +0000 Subject: [PATCH 14/39] :wrench: Blank-ify charts --- chart/templates/deployment.yaml | 0 chart/templates/deployment.yml | 49 ----------------------------- chart/templates/scaled-job.yaml | 49 ----------------------------- chart/templates/secret-account.yaml | 12 ------- chart/templates/secret.yaml | 10 ------ 5 files changed, 120 deletions(-) create mode 100644 chart/templates/deployment.yaml delete mode 100644 chart/templates/deployment.yml diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml new file mode 100644 index 0000000..e69de29 diff --git a/chart/templates/deployment.yml b/chart/templates/deployment.yml deleted file mode 100644 index 582d8c6..0000000 --- a/chart/templates/deployment.yml +++ /dev/null @@ -1,49 +0,0 @@ -{{- if not .Values.ephemeral.enabled }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "actions-runner.fullname" . }} - labels: - {{- include "actions-runner.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - {{- include "actions-runner.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "actions-runner.labels" . | nindent 8 }} - {{- with .Values.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "actions-runner.serviceAccountName" . }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: GITHUB_REPOSITORY - value: {{ .Values.github.organisation }}/{{ .Values.github.repository }} - - name: GITHUB_TOKEN - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-github-token - key: token - - name: RUNNER_LABELS - value: {{ .Values.github.runner.labels | quote }} - resources: - {{- toYaml .Values.resources | nindent 12 }} -{{- end }} diff --git a/chart/templates/scaled-job.yaml b/chart/templates/scaled-job.yaml index 582d8c6..e69de29 100644 --- a/chart/templates/scaled-job.yaml +++ b/chart/templates/scaled-job.yaml @@ -1,49 +0,0 @@ -{{- if not .Values.ephemeral.enabled }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "actions-runner.fullname" . }} - labels: - {{- include "actions-runner.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - {{- include "actions-runner.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "actions-runner.labels" . | nindent 8 }} - {{- with .Values.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "actions-runner.serviceAccountName" . }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: GITHUB_REPOSITORY - value: {{ .Values.github.organisation }}/{{ .Values.github.repository }} - - name: GITHUB_TOKEN - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-github-token - key: token - - name: RUNNER_LABELS - value: {{ .Values.github.runner.labels | quote }} - resources: - {{- toYaml .Values.resources | nindent 12 }} -{{- end }} diff --git a/chart/templates/secret-account.yaml b/chart/templates/secret-account.yaml index e207736..e69de29 100644 --- a/chart/templates/secret-account.yaml +++ b/chart/templates/secret-account.yaml @@ -1,12 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "actions-runner.serviceAccountName" . }} - labels: - {{- include "actions-runner.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -automountServiceAccountToken: {{ .Values.serviceAccount.automount }} diff --git a/chart/templates/secret.yaml b/chart/templates/secret.yaml index 352ffd0..e69de29 100644 --- a/chart/templates/secret.yaml +++ b/chart/templates/secret.yaml @@ -1,10 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Release.Name }}-github-token - labels: - {{- include "actions-runner.labels" . | nindent 4 }} -type: Opaque -data: - token: {{ .Values.github.token | b64enc }} From 9cd835f734edeb9bb24e004d17e6199fc3845502 Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:57:11 +0000 Subject: [PATCH 15/39] :wrench: Linting + correction --- .github/workflows/super-linter.yml | 1 + ollamate/settings.py | 3 +-- streamingapp/views.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/super-linter.yml b/.github/workflows/super-linter.yml index 393bc8e..7c054f2 100644 --- a/.github/workflows/super-linter.yml +++ b/.github/workflows/super-linter.yml @@ -40,3 +40,4 @@ jobs: PYTHON_FLAKE8_CONFIG_FILE: .flake8 PYTHON_ISORT_CONFIG_FILE: pyproject.toml PYTHON_MYPY_CONFIG_FILE: mypy.ini + VALIDATE_KUBERNETES_KUBECONFORM: false # Super-Linter doesn't support https://github.com/jtyr/kubeconform-helm diff --git a/ollamate/settings.py b/ollamate/settings.py index fd6a9b6..64299ae 100644 --- a/ollamate/settings.py +++ b/ollamate/settings.py @@ -14,7 +14,6 @@ import environ - # Initialize environment variables env = environ.Env() @@ -34,7 +33,7 @@ print("REDIRECT_URI:", env("REDIRECT_URI")) # Build paths inside the project like this: BASE_DIR / 'subdir'. -BASE_DIR = Path(__file__).resolve().parent.parent +# BASE_DIR = Path(__file__).resolve().parent.parent # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/ diff --git a/streamingapp/views.py b/streamingapp/views.py index 8b90fc7..7de7356 100644 --- a/streamingapp/views.py +++ b/streamingapp/views.py @@ -1,7 +1,6 @@ import json import logging - from django.http import JsonResponse from django.shortcuts import redirect, render From aceebc4b888c7401f8cdf4bd581414f9a957c780 Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 16:00:57 +0000 Subject: [PATCH 16/39] :wrench: Alter flake8 configuration --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index c815588..787c4ab 100644 --- a/.flake8 +++ b/.flake8 @@ -4,4 +4,4 @@ extend-ignore = E203, E704 exclude = venv per-file-ignores = - ap/*/migrations/*:E501,W292 + ollamate/*/migrations/*:E501,W292 From 9df60e251e26cc0f3dbc0aebd49cb2e6caf41294 Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 16:04:54 +0000 Subject: [PATCH 17/39] :wrench: flake8 configuration --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 787c4ab..f4cefcf 100644 --- a/.flake8 +++ b/.flake8 @@ -4,4 +4,4 @@ extend-ignore = E203, E704 exclude = venv per-file-ignores = - ollamate/*/migrations/*:E501,W292 + ollamate/*:E501,W292 From 5f3bdf82f507d404902aeb1918c5608e4f5231b6 Mon Sep 17 00:00:00 2001 From: Gary Henderson <26419401+Gary-H9@users.noreply.github.com> Date: Tue, 2 Jul 2024 16:10:22 +0000 Subject: [PATCH 18/39] :fire: Remove unneeded code --- ollamate/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ollamate/settings.py b/ollamate/settings.py index 64299ae..32171ab 100644 --- a/ollamate/settings.py +++ b/ollamate/settings.py @@ -10,7 +10,7 @@ https://docs.djangoproject.com/en/5.0/ref/settings/ """ import os -from pathlib import Path +# from pathlib import Path import environ From b81bb1bccf4acc9f9e130f3b0bf169cb47d185b9 Mon Sep 17 00:00:00 2001 From: Gary H <26419401+Gary-H9@users.noreply.github.com> Date: Wed, 3 Jul 2024 17:10:27 +0000 Subject: [PATCH 19/39] :recycle: Rename --- ...{docker-compose-postgresql.yml => docker-compose-postgres.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename contrib/{docker-compose-postgresql.yml => docker-compose-postgres.yml} (100%) diff --git a/contrib/docker-compose-postgresql.yml b/contrib/docker-compose-postgres.yml similarity index 100% rename from contrib/docker-compose-postgresql.yml rename to contrib/docker-compose-postgres.yml From 2a9df00e5014da32d771c040a6e847d7b3ba9bd6 Mon Sep 17 00:00:00 2001 From: Gary H <26419401+Gary-H9@users.noreply.github.com> Date: Wed, 3 Jul 2024 17:23:07 +0000 Subject: [PATCH 20/39] :wrench: Add ollama in devcontainer --- contrib/docker-compose-ollama.yml | 29 +++++++++++++++++++++++++++++ ollamate/settings.py | 6 ++++-- requirements.dev.txt | 1 + scripts/devcontainer/post-create.sh | 6 ++++++ streamingapp/views.py | 7 ++++++- 5 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 contrib/docker-compose-ollama.yml diff --git a/contrib/docker-compose-ollama.yml b/contrib/docker-compose-ollama.yml new file mode 100644 index 0000000..8c576bc --- /dev/null +++ b/contrib/docker-compose-ollama.yml @@ -0,0 +1,29 @@ +--- +version: '3.8' + +services: + ollama: + image: ollama/ollama + ports: + - 11434:11434 + # command: > + # echo "Starting Ollama server..." + # ollama serve & + # ollama run llama3 + + + # echo "Waiting for Ollama server to be active..." + # while [ "$(ollama list | grep 'NAME')" == "" ]; do + # sleep 1 + # done + + # command: "ollama pull llama3" + # volumes: + # - ./data:/app/data + # environment: + # - ENV_VARIABLE=value + # healthcheck: + # test: ["CMD", "curl", "-f", "http://localhost:11434/health"] + # interval: 30s + # timeout: 10s + # retries: 3 diff --git a/ollamate/settings.py b/ollamate/settings.py index 32171ab..44d0b04 100644 --- a/ollamate/settings.py +++ b/ollamate/settings.py @@ -10,10 +10,12 @@ https://docs.djangoproject.com/en/5.0/ref/settings/ """ import os -# from pathlib import Path import environ +# from pathlib import Path + + # Initialize environment variables env = environ.Env() @@ -44,7 +46,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [""] +ALLOWED_HOSTS: list = ["127.0.0.1"] PROJECT_NAME = "ollamate" diff --git a/requirements.dev.txt b/requirements.dev.txt index c2497dd..05a49f7 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -5,3 +5,4 @@ flake8==6.1.0 isort==5.12.0 mypy==1.6.1 pre-commit==3.5.0 +types-requests==2.31.0 diff --git a/scripts/devcontainer/post-create.sh b/scripts/devcontainer/post-create.sh index f412ff3..83631b5 100644 --- a/scripts/devcontainer/post-create.sh +++ b/scripts/devcontainer/post-create.sh @@ -6,6 +6,12 @@ npm install --global npm@latest # Start Postgres docker compose --file contrib/docker-compose-postgres.yml up --detach +# Start Ollama Container +docker compose --file contrib/docker-compose-ollama.yml up --detach + +# Start Ollama Model +docker exec -it contrib-ollama-1 ollama run orca-mini + # Upgrade Pip pip install --break-system-package --upgrade pip diff --git a/streamingapp/views.py b/streamingapp/views.py index 7de7356..400901a 100644 --- a/streamingapp/views.py +++ b/streamingapp/views.py @@ -23,12 +23,17 @@ def call_ollama(request): url = "http://localhost:11434/api/chat" headers = {"Content-Type": "application/json"} - data = {"model": "llama3", "messages": conversation_history, "stream": False} + data = { + "model": "orca-mini", + "messages": conversation_history, + "stream": False, + } # The model is hardcoded here, this will be parameterised in future logging.debug("Sending data to Ollama API: %s", json.dumps(data, indent=2)) try: response = requests.post(url, json=data, headers=headers) + logging.debug("HTTP status code: %s", response.status_code) response.raise_for_status() response_data = response.json() logging.debug("Response data: %s", json.dumps(response_data, indent=2)) From f2c4d6aaa95600fc3809f34e137447938f055f61 Mon Sep 17 00:00:00 2001 From: Gary H <26419401+Gary-H9@users.noreply.github.com> Date: Wed, 3 Jul 2024 17:37:31 +0000 Subject: [PATCH 21/39] =?UTF-8?q?=E2=9E=95=20Makefile=20extension?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .devcontainer/devcontainer.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8fbc8d4..406346a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -24,7 +24,8 @@ "EditorConfig.EditorConfig", "GitHub.vscode-github-actions", "GitHub.vscode-codeql", - "ms-vsliveshare.vsliveshare" + "ms-vsliveshare.vsliveshare", + "ms-vscode.makefile-tools" ] } } From 1bfc2f269e9430d6a038cb9d9067fdac4008a5bb Mon Sep 17 00:00:00 2001 From: Gary H <26419401+Gary-H9@users.noreply.github.com> Date: Wed, 3 Jul 2024 17:39:44 +0000 Subject: [PATCH 22/39] :wrench: Comment out prints --- Dockerfile | 2 +- ollamate/settings.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1210e52..7d1e075 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,7 @@ COPY ollamate ollamate RUN pip install --no-cache-dir --requirement requirements.txt \ && chmod +x /usr/local/bin/entrypoint.sh \ - && python manage.py collectstatic --noinput --ignore=*.scss \ + # && python manage.py collectstatic --noinput --ignore=*.scss \ && apk del .build-deps EXPOSE 8000 diff --git a/ollamate/settings.py b/ollamate/settings.py index 44d0b04..aa669f0 100644 --- a/ollamate/settings.py +++ b/ollamate/settings.py @@ -28,11 +28,11 @@ print(f"Loading environment variables from {env_file}") environ.Env.read_env(env_file) else: - print(f"{env_file} not found") - print("CLIENT_ID:", env("CLIENT_ID")) - print("CLIENT_SECRET:", env("CLIENT_SECRET")) - print("TENANT_ID:", env("AZURE_TENANT_ID")) - print("REDIRECT_URI:", env("REDIRECT_URI")) + # print(f"{env_file} not found") + # print("CLIENT_ID:", env("CLIENT_ID")) + # print("CLIENT_SECRET:", env("CLIENT_SECRET")) + # print("TENANT_ID:", env("AZURE_TENANT_ID")) + # print("REDIRECT_URI:", env("REDIRECT_URI")) # Build paths inside the project like this: BASE_DIR / 'subdir'. # BASE_DIR = Path(__file__).resolve().parent.parent From 24f829203943d24a849727ae1cd970fe874927c6 Mon Sep 17 00:00:00 2001 From: Gary H <26419401+Gary-H9@users.noreply.github.com> Date: Wed, 3 Jul 2024 17:46:33 +0000 Subject: [PATCH 23/39] Add pytest + pytest-django to requirements.txt --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 509a6ab..5a1c302 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ django-azure-auth==2.0.0 django-environ==0.11.2 -requests==2.32.3 psycopg==3.2.1 +pytest==8.0.0 +pytest-django==4.8.0 +requests==2.32.3 From f6374e6bdae552a457109e691878cd74aecc6243 Mon Sep 17 00:00:00 2001 From: Gary H <26419401+Gary-H9@users.noreply.github.com> Date: Thu, 4 Jul 2024 08:04:39 +0000 Subject: [PATCH 24/39] :wrench: Remove Testing for now --- .github/workflows/build-test.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index bafeb68..c45e259 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -27,10 +27,10 @@ jobs: load: true tags: dashboard - - name: Run Python Tests - id: run_python_tests - run: | - docker compose --file contrib/docker-compose-test.yml run --rm interfaces - env: - NETWORK: default - IMAGE_TAG: dashboard + # - name: Run Python Tests + # id: run_python_tests + # run: | + # docker compose --file contrib/docker-compose-test.yml run --rm interfaces + # env: + # NETWORK: default + # IMAGE_TAG: dashboard From 1cccef596b070aea856cd78d6c5d3ed06c602e45 Mon Sep 17 00:00:00 2001 From: Gary H <26419401+Gary-H9@users.noreply.github.com> Date: Thu, 4 Jul 2024 08:04:58 +0000 Subject: [PATCH 25/39] :wrench: Add .yamllint --- .yamllint | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .yamllint diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..0db05bb --- /dev/null +++ b/.yamllint @@ -0,0 +1,9 @@ +--- +extends: default + +rules: + comments: disable + line-length: + level: warning + allow-non-breakable-inline-mappings: true + truthy: disable From ca5726c0a363cbc5e05c254b026edfc3f896ef2e Mon Sep 17 00:00:00 2001 From: Gary H <26419401+Gary-H9@users.noreply.github.com> Date: Thu, 4 Jul 2024 08:18:50 +0000 Subject: [PATCH 26/39] Linting --- .env.example | 8 +++++++- ollamate/settings.py | 3 ++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 3bb634c..060353c 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,13 @@ +DB_NAME=ollamate +DB_USER=ollamate +DB_PASSWORD=ollamate + +SECRET_KEY= + CLIENT_ID= CLIENT_SECRET= AZURE_TENANT_ID= REDIRECT_URI= # when running locally, you can use -REDIRECT_URI="https://127.0.0.1:8000/azure_auth/callback" \ No newline at end of file +REDIRECT_URI="https://127.0.0.1:8000/azure_auth/callback" diff --git a/ollamate/settings.py b/ollamate/settings.py index aa669f0..c12f2f4 100644 --- a/ollamate/settings.py +++ b/ollamate/settings.py @@ -28,6 +28,7 @@ print(f"Loading environment variables from {env_file}") environ.Env.read_env(env_file) else: + print("Fix environment variables.") # print(f"{env_file} not found") # print("CLIENT_ID:", env("CLIENT_ID")) # print("CLIENT_SECRET:", env("CLIENT_SECRET")) @@ -41,7 +42,7 @@ # See https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = "django-insecure-(y%c-1p0lnlrxg!$3w)ptcyp=wzer(biav_-%_fehgn1oatx8p" +SECRET_KEY = env("SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True From 4df15830f71e14b8a1caa9b95927e8f86ec7e8fc Mon Sep 17 00:00:00 2001 From: Gary H <26419401+Gary-H9@users.noreply.github.com> Date: Mon, 8 Jul 2024 11:39:14 +0000 Subject: [PATCH 27/39] Final touches --- README.md | 18 ++++++++++++++++-- ollamate/settings.py | 2 +- ollamate/urls.py | 12 ++++++------ scripts/devcontainer/post-create.sh | 2 +- .../templates/streamingapp/input_form.html | 11 ++--------- streamingapp/views.py | 2 +- 6 files changed, 27 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 9eb1f26..d5ecdc9 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,21 @@ This repository includes a Django application which presents a simple frontend t The name 'Ollamate' is derived from Ollama and will be changed if/when this application evolves. -## Getting Started - > [!NOTE] > 🚧 WIP 🚧 + + +## Running Locally + +Ollamate is run in a Development Container via Docker. The Development Container VSCode [extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) is recommended. Additionally ensure [Docker Desktop](https://www.docker.com/products/docker-desktop/) is running. + +For more information on Dev Containers, see the [Analytical Platform docs.](https://technical-documentation.data-platform.service.justice.gov.uk/documentation/platform/infrastructure/developing.html#developing-the-data-platform) + + +### Building the Development Container +Open this repository in Visual Studio Code. Open the Command Palette `Command + Shift + p`, search for and select ```> Dev Containers: Reopen in container```. + +The Development Container utilises Docker in Docker to create two containers - `ollama` and `postgresql`. This allows for easy setup when working on Ollamate locally. + +### Environment Variables +There is an example `.env.example` file. Paste the contents into a new file called ```.env``` in the root of the project. diff --git a/ollamate/settings.py b/ollamate/settings.py index c12f2f4..3c8d461 100644 --- a/ollamate/settings.py +++ b/ollamate/settings.py @@ -61,7 +61,7 @@ "django.contrib.messages", "django.contrib.staticfiles", "streamingapp", - "azure_auth", + # "azure_auth" ] MIDDLEWARE = [ diff --git a/ollamate/urls.py b/ollamate/urls.py index e0e3e65..935ac14 100644 --- a/ollamate/urls.py +++ b/ollamate/urls.py @@ -1,14 +1,14 @@ from django.contrib import admin from django.urls import include, path -from streamingapp.views import redirect_to_ollama +# from streamingapp.views import redirect_to_ollama urlpatterns = [ path("admin/", admin.site.urls), - path( - "azure_auth/", - include("azure_auth.urls"), - ), + # path( + # "azure_auth/", + # include("azure_auth.urls"), + # ), path("stream/", include("streamingapp.urls")), - path("", redirect_to_ollama), + # path("", redirect_to_ollama), ] diff --git a/scripts/devcontainer/post-create.sh b/scripts/devcontainer/post-create.sh index 83631b5..4f6a06f 100644 --- a/scripts/devcontainer/post-create.sh +++ b/scripts/devcontainer/post-create.sh @@ -10,7 +10,7 @@ docker compose --file contrib/docker-compose-postgres.yml up --detach docker compose --file contrib/docker-compose-ollama.yml up --detach # Start Ollama Model -docker exec -it contrib-ollama-1 ollama run orca-mini +docker exec -it contrib-ollama-1 ollama run orca-mini & # Upgrade Pip pip install --break-system-package --upgrade pip diff --git a/streamingapp/templates/streamingapp/input_form.html b/streamingapp/templates/streamingapp/input_form.html index 76409ca..19f93ad 100644 --- a/streamingapp/templates/streamingapp/input_form.html +++ b/streamingapp/templates/streamingapp/input_form.html @@ -46,13 +46,6 @@

Ollamate 🦙🍵

A Simple Conversational AI.

Logged in as: {{ user.username }}

-
- {% if user.is_authenticated %} - Logout - {% else %} - Login - {% endif %} -

Enter text below to interact with Ollamate:

@@ -101,7 +94,7 @@

A Simple Conversational AI.

var ollamaResponse = response.response; var formattedResponse = formatResponse(ollamaResponse); - + conversationHistory.push({"role": "assistant", "content": formattedResponse}); $('#conversation-log').append('
Ollama: ' + formattedResponse + '
'); @@ -133,4 +126,4 @@

A Simple Conversational AI.

}); - \ No newline at end of file + diff --git a/streamingapp/views.py b/streamingapp/views.py index 400901a..edf8c54 100644 --- a/streamingapp/views.py +++ b/streamingapp/views.py @@ -11,7 +11,7 @@ logging.basicConfig(level=logging.DEBUG) -@azure_auth_required +# @azure_auth_required def call_ollama(request): if request.method == "POST": user_input = request.POST.get("userInput", "") From 21d0f68410baec89efee77787ab1d4fdad1f1ded Mon Sep 17 00:00:00 2001 From: Gary H <26419401+Gary-H9@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:28:00 +0000 Subject: [PATCH 28/39] :wrench: Linting --- requirements.txt | 1 + scripts/devcontainer/post-create.sh | 2 +- .../templates/streamingapp/input_form.html | 19 ++++++++++--------- streamingapp/views.py | 5 +++-- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5a1c302..6f362cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ psycopg==3.2.1 pytest==8.0.0 pytest-django==4.8.0 requests==2.32.3 +types-requests diff --git a/scripts/devcontainer/post-create.sh b/scripts/devcontainer/post-create.sh index 4f6a06f..ebb2717 100644 --- a/scripts/devcontainer/post-create.sh +++ b/scripts/devcontainer/post-create.sh @@ -10,7 +10,7 @@ docker compose --file contrib/docker-compose-postgres.yml up --detach docker compose --file contrib/docker-compose-ollama.yml up --detach # Start Ollama Model -docker exec -it contrib-ollama-1 ollama run orca-mini & +docker exec -it contrib-ollama-1 ollama run llama3 & # Upgrade Pip pip install --break-system-package --upgrade pip diff --git a/streamingapp/templates/streamingapp/input_form.html b/streamingapp/templates/streamingapp/input_form.html index 19f93ad..1603702 100644 --- a/streamingapp/templates/streamingapp/input_form.html +++ b/streamingapp/templates/streamingapp/input_form.html @@ -1,10 +1,9 @@ + Ollamate 🦙🍵 - -