From 85bdd88d5bde6430009121ed1118954dec8cdb7a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Sep 2022 15:44:49 +0000 Subject: [PATCH 01/75] Bump django-filter from 2.3.0 to 22.1 Bumps [django-filter](https://github.com/carltongibson/django-filter) from 2.3.0 to 22.1. - [Release notes](https://github.com/carltongibson/django-filter/releases) - [Changelog](https://github.com/carltongibson/django-filter/blob/main/CHANGES.rst) - [Commits](https://github.com/carltongibson/django-filter/compare/2.3.0...22.1) --- updated-dependencies: - dependency-name: django-filter dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 96c0e46a1..6a1127be3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ Django==3.2.15 django-crequest==2018.5.11 django-dotenv==1.4.2 django-extensions==3.1.5 -django-filter==2.4.0 +django-filter==22.1 django-prometheus==2.1.0 django-redis==4.12.1 django-simple-history==3.1.1 From a4174e48f49a857b0f1846decfcd40af9927adcb Mon Sep 17 00:00:00 2001 From: ahbensiali <112475984+ahbensiali@users.noreply.github.com> Date: Fri, 14 Oct 2022 08:53:28 +0100 Subject: [PATCH 02/75] added precommit --- .flake8 | 5 +++++ .pre-commit-config.yaml | 36 ++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- requirements.dev.txt | 6 ++++-- 4 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 .flake8 create mode 100644 .pre-commit-config.yaml diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..d9ad0b409 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +ignore = E203, E266, E501, W503, F403, F401 +max-line-length = 79 +max-complexity = 18 +select = B,C,E,F,W,T4,B9 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..d896fb912 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,36 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: requirements-txt-fixer + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black + rev: 22.8.0 + hooks: + - id: black + args: ['.'] + + - repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + name: isort (python) + args: ['--filter-files', '.'] + entry: isort + + - repo: https://github.com/pycqa/flake8 + rev: 5.0.4 + hooks: + - id: flake8 + args: ['--config=.flake8', '.'] + + - repo: local + hooks: + - id: jira-ticket + name: check for jira ticket + language: pygrep + entry: '\A(?![A-Z]+-[0-9]+)' + args: [--multiline] + stages: [commit-msg] diff --git a/pyproject.toml b/pyproject.toml index b4cf91c49..bd3147e86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ exclude = ''' ''' [tool.isort] +profile = "black" import_heading_firstparty = 'First-party/Local' import_heading_future = 'Future' import_heading_stdlib = 'Standard library' @@ -16,4 +17,3 @@ import_heading_thirdparty = 'Third-party' line_length = 88 multi_line_output = 3 no_lines_before = 'LOCALFOLDER' -profile = 'black' diff --git a/requirements.dev.txt b/requirements.dev.txt index ece35f162..3bd8e9482 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,9 +1,11 @@ django-debug-toolbar==3.2.4 django-debug-toolbar-requests==1.0.5 django-elasticsearch-debug-toolbar==2.0.0 -pylint==2.12.2 +pylint==2.15.4 pylint-django==2.4.4 ipython==7.31.1 ipdb==0.13.9 black==22.8.0 -isort==5.10.1 \ No newline at end of file +isort==5.10.1 +pre-commit==2.20.0 +flake8==5.0.4 From 07590e9faa794a47c1500b3250f64af9b1d98fb5 Mon Sep 17 00:00:00 2001 From: ahbensiali <112475984+ahbensiali@users.noreply.github.com> Date: Fri, 14 Oct 2022 09:21:05 +0100 Subject: [PATCH 03/75] ANPL-862 fixed requirements --- .pre-commit-config.yaml | 2 +- requirements.dev.txt | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d896fb912..d1f2ff780 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: - repo: https://github.com/pycqa/flake8 rev: 5.0.4 hooks: - - id: flake8 + - id: flake8 args: ['--config=.flake8', '.'] - repo: local diff --git a/requirements.dev.txt b/requirements.dev.txt index 3bd8e9482..fabc4f087 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,11 +1,11 @@ +black==22.8.0 django-debug-toolbar==3.2.4 django-debug-toolbar-requests==1.0.5 django-elasticsearch-debug-toolbar==2.0.0 -pylint==2.15.4 -pylint-django==2.4.4 -ipython==7.31.1 +flake8==5.0.4 ipdb==0.13.9 -black==22.8.0 +ipython==7.31.1 isort==5.10.1 pre-commit==2.20.0 -flake8==5.0.4 +pylint==2.15.4 +pylint-django==2.4.4 From 5f0470ce392f747e2aff5c7aafad1ef12644f256 Mon Sep 17 00:00:00 2001 From: ahbensiali <112475984+ahbensiali@users.noreply.github.com> Date: Fri, 14 Oct 2022 09:37:03 +0100 Subject: [PATCH 04/75] ANPL-862 added instructions --- doc/running.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/doc/running.md b/doc/running.md index 659810496..ff3a848c9 100644 --- a/doc/running.md +++ b/doc/running.md @@ -38,6 +38,8 @@ source venv/bin/activate pip3 install -r requirements.txt pip3 install -r requirements.dev.txt pip3 uninstall python-dotenv # see ANPL-823 +pre-commit install --hook-type commit-msg +pre-commit install ``` In addition, you must have: @@ -83,7 +85,7 @@ and have [cluster admin access to Kubernetes](https://silver-dollop-30c6a355.pag ### AWS Configuration In order to run the app you'll need various permissions set up for you in the -wider infrastructure of the project, mainly for AWS platform. +wider infrastructure of the project, mainly for AWS platform. As the docs for AWS (linked above) mention, you'll need to add yourself an AWS user account linked to your MoJ email address via the @@ -258,14 +260,14 @@ and then ask a colleague for help. ### Local AWS profile setup (on first run only) This app needs to interact with multiple AWS accounts in order to support the users' needs. -The AWS resources like IAM, s3 buckets are under our data account and will be managed by +The AWS resources like IAM, s3 buckets are under our data account and will be managed by app through [boto3](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html). In order to make sure the boto3 can obtain the right profile for local env. The following steps will show how to create it. Assume that the name of profile for our aws data account is ```admin-data``` #### Add the AWS credential into .aws/credentials -it should look like below +it should look like below ``` [admin-data] aws_access_key_id = @@ -318,17 +320,17 @@ If you want to run the control panel app to manage AWS resources under single ro following environment variable to define the profile you want to use - ```AWS_PROFILE```: The profile which will be used for ```boto3``` auth export AWS_PROFILE = "admin-data" -- Make sure there is NO other AWS boto3 environment variables defined. +- Make sure there is NO other AWS boto3 environment variables defined. #### AWS credential setting for multiple AWS roles -If you want to run the app to manage the AWS resources cross different AWS accounts by assuming +If you want to run the app to manage the AWS resources cross different AWS accounts by assuming different roles, then - Check whether following 2 more environment variables have been setup in the env file or not - `AWS_DATA_ACCOUNT_ROLE`: The role_arn of admin-data account - `AWS_DEV_ACCOUNT_ROLE` : The role_arn of admin-dev account - + if you are not sure what the value of role_arn of those two accounts is, you can find them out by - checking the aws config file. + checking the aws config file. More detail about the settings for mult-account is [here](architecture.md) (last section) - Make sure other AWS boto3 settings e.g. ```AWS_PROFILE``` are NOT defined in your env, otherwise the app will @@ -371,7 +373,7 @@ Go to http://localhost:8000/, sign in via Auth0 and marvel at your locally running control panel. NOTES: if you use aws-vault to manage your AWS credentials, during the running process of the app, -you may encounter a popup window for asking you to provide key-chain password from time to time, +you may encounter a popup window for asking you to provide key-chain password from time to time, which is normal. ### Loading tools From 36f8be0ec7580e22d2b1da1b6a148dc10de7c414 Mon Sep 17 00:00:00 2001 From: ahbensiali <112475984+ahbensiali@users.noreply.github.com> Date: Mon, 17 Oct 2022 21:19:55 +0100 Subject: [PATCH 05/75] ANPL-862 remove to tty --- .pre-commit-config.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d1f2ff780..d495d9c8d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,4 @@ +default_stages: [commit] repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.2.0 @@ -34,3 +35,11 @@ repos: entry: '\A(?![A-Z]+-[0-9]+)' args: [--multiline] stages: [commit-msg] + + - id: pytest-check + name: pytest-check + entry: bash -c 'DJANGO_SETTINGS_MODULE=controlpanel.settings.test pytest > /dev/tty' + language: system + pass_filenames: false + always_run: true + # verbose: true From 37b353eb2bd2c42271d6ec18e20bd52414edb324 Mon Sep 17 00:00:00 2001 From: ahbensiali <112475984+ahbensiali@users.noreply.github.com> Date: Mon, 17 Oct 2022 21:26:12 +0100 Subject: [PATCH 06/75] ANPL-862 removed warnings --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d495d9c8d..e8faf22a2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,7 +38,7 @@ repos: - id: pytest-check name: pytest-check - entry: bash -c 'DJANGO_SETTINGS_MODULE=controlpanel.settings.test pytest > /dev/tty' + entry: bash -c 'DJANGO_SETTINGS_MODULE=controlpanel.settings.test pytest -W ignore > /dev/tty' language: system pass_filenames: false always_run: true From d66a3d76ab51c4ec882d2e87699c0ac57456b0e4 Mon Sep 17 00:00:00 2001 From: ahbensiali <112475984+ahbensiali@users.noreply.github.com> Date: Tue, 25 Oct 2022 21:37:08 +0100 Subject: [PATCH 07/75] ANPL-862 removed pytest step --- .pre-commit-config.yaml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e8faf22a2..d3b2809ba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,14 +32,6 @@ repos: - id: jira-ticket name: check for jira ticket language: pygrep - entry: '\A(?![A-Z]+-[0-9]+)' + entry: '\A(?!ANPL+-[0-9]+)' args: [--multiline] stages: [commit-msg] - - - id: pytest-check - name: pytest-check - entry: bash -c 'DJANGO_SETTINGS_MODULE=controlpanel.settings.test pytest -W ignore > /dev/tty' - language: system - pass_filenames: false - always_run: true - # verbose: true From 7a03bcfe111dea17204138bf3b907e23389d9dd4 Mon Sep 17 00:00:00 2001 From: ahbensiali <112475984+ahbensiali@users.noreply.github.com> Date: Mon, 24 Oct 2022 15:26:56 +0100 Subject: [PATCH 08/75] Update .flake8 ANPL-862 changed max-line-length --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index d9ad0b409..2a8129f43 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] ignore = E203, E266, E501, W503, F403, F401 -max-line-length = 79 +max-line-length = 88 max-complexity = 18 select = B,C,E,F,W,T4,B9 From 3280087da8fbd66be672d6618e4a2707de5b1aba Mon Sep 17 00:00:00 2001 From: ahbensiali <112475984+ahbensiali@users.noreply.github.com> Date: Wed, 26 Oct 2022 15:22:00 +0100 Subject: [PATCH 09/75] test only staged --- .pre-commit-config.yaml | 72 +++++++++++++++++++++-------------- controlpanel/settings/test.py | 2 + 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d3b2809ba..0720470ea 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,37 +1,51 @@ default_stages: [commit] repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 - hooks: - - id: requirements-txt-fixer - - id: check-yaml - - id: end-of-file-fixer - - id: trailing-whitespace - repo: https://github.com/psf/black rev: 22.8.0 hooks: - - id: black - args: ['.'] + - id: black + args: [".", "--check"] + files: $(git diff --name-only --cached --diff-filter=ACMR) - - repo: https://github.com/pycqa/isort - rev: 5.10.1 - hooks: - - id: isort - name: isort (python) - args: ['--filter-files', '.'] - entry: isort + # hooks: + # - id: changes-files + # language: bash - - repo: https://github.com/pycqa/flake8 - rev: 5.0.4 - hooks: - - id: flake8 - args: ['--config=.flake8', '.'] + # - repo: https://github.com/pre-commit/pre-commit-hooks + # rev: v3.2.0 + # hooks: + # - id: requirements-txt-fixer + # - id: check-yaml + # - id: end-of-file-fixer + # - id: trailing-whitespace - - repo: local - hooks: - - id: jira-ticket - name: check for jira ticket - language: pygrep - entry: '\A(?!ANPL+-[0-9]+)' - args: [--multiline] - stages: [commit-msg] + # - repo: https://github.com/psf/black + # rev: 22.8.0 + # hooks: + # - id: black + # args: ['.', "--check"] + # # files: + + # - repo: https://github.com/pycqa/isort + # rev: 5.10.1 + # hooks: + # - id: isort + # name: isort (python) + # args: ['--filter-files', '.'] + # entry: isort + + # - repo: https://github.com/pycqa/flake8 + # rev: 5.0.4 + # hooks: + # - id: flake8 + # args: ['--config=.flake8', '.'] + + + # - repo: local + # hooks: + # - id: jira-ticket + # name: check for jira ticket + # language: pygrep + # entry: '\A(?!ANPL+-[0-9]+)' + # args: [--multiline] + # stages: [commit-msg] diff --git a/controlpanel/settings/test.py b/controlpanel/settings/test.py index a315a9ecb..0c4f69071 100644 --- a/controlpanel/settings/test.py +++ b/controlpanel/settings/test.py @@ -32,3 +32,5 @@ "api_token": "test-slack-api-token", "channel": "test-slack-channel", } + +test_var_to_delete=1 \ No newline at end of file From e368b4ff13fcfc11ea6d220c3bccce67cfb16f30 Mon Sep 17 00:00:00 2001 From: ahbensiali <112475984+ahbensiali@users.noreply.github.com> Date: Wed, 26 Oct 2022 15:23:29 +0100 Subject: [PATCH 10/75] testing black changes --- controlpanel/settings/test.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/controlpanel/settings/test.py b/controlpanel/settings/test.py index 0c4f69071..a60878b47 100644 --- a/controlpanel/settings/test.py +++ b/controlpanel/settings/test.py @@ -1,6 +1,5 @@ from controlpanel.settings.common import * - ENV = 'test' AWS_COMPUTE_ACCOUNT_ID = "test_compute_account_id" @@ -31,6 +30,4 @@ SLACK = { "api_token": "test-slack-api-token", "channel": "test-slack-channel", -} - -test_var_to_delete=1 \ No newline at end of file +} \ No newline at end of file From 8a2ab61ffe5aff462567a7ace68c028d71acaf46 Mon Sep 17 00:00:00 2001 From: ahbensiali <112475984+ahbensiali@users.noreply.github.com> Date: Fri, 28 Oct 2022 09:12:29 +0100 Subject: [PATCH 11/75] ANPL-862 conflict fix --- .pre-commit-config.yaml | 72 +++++++++++++++-------------------- controlpanel/settings/test.py | 21 +++++----- 2 files changed, 41 insertions(+), 52 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0720470ea..95d38d83d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,51 +1,39 @@ default_stages: [commit] repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: requirements-txt-fixer + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black rev: 22.8.0 hooks: - id: black - args: [".", "--check"] - files: $(git diff --name-only --cached --diff-filter=ACMR) - - # hooks: - # - id: changes-files - # language: bash - - # - repo: https://github.com/pre-commit/pre-commit-hooks - # rev: v3.2.0 - # hooks: - # - id: requirements-txt-fixer - # - id: check-yaml - # - id: end-of-file-fixer - # - id: trailing-whitespace - - # - repo: https://github.com/psf/black - # rev: 22.8.0 - # hooks: - # - id: black - # args: ['.', "--check"] - # # files: + name: black formatting + entry: bash -c 'black $(git diff --name-only --cached --diff-filter=ACMR | grep .py)' - # - repo: https://github.com/pycqa/isort - # rev: 5.10.1 - # hooks: - # - id: isort - # name: isort (python) - # args: ['--filter-files', '.'] - # entry: isort - - # - repo: https://github.com/pycqa/flake8 - # rev: 5.0.4 - # hooks: - # - id: flake8 - # args: ['--config=.flake8', '.'] + - repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + name: isort (python) + entry: bash -c 'isort $(git diff --name-only --cached --diff-filter=ACMR | grep .py)' + - repo: https://github.com/pycqa/flake8 + rev: 5.0.4 + hooks: + - id: flake8 + name: flake8 format check + entry: bash -c 'flake8 --config=.flake8 $(git diff --name-only --cached --diff-filter=ACMR | grep .py)' - # - repo: local - # hooks: - # - id: jira-ticket - # name: check for jira ticket - # language: pygrep - # entry: '\A(?!ANPL+-[0-9]+)' - # args: [--multiline] - # stages: [commit-msg] + - repo: local + hooks: + - id: jira-ticket + name: check for jira ticket + language: pygrep + entry: '\A(?!ANPL+-[0-9]+)' + args: [--multiline] + stages: [commit-msg] diff --git a/controlpanel/settings/test.py b/controlpanel/settings/test.py index a60878b47..94063ad68 100644 --- a/controlpanel/settings/test.py +++ b/controlpanel/settings/test.py @@ -1,28 +1,29 @@ +# First-party/Local from controlpanel.settings.common import * -ENV = 'test' +ENV = "test" AWS_COMPUTE_ACCOUNT_ID = "test_compute_account_id" AWS_DATA_ACCOUNT_ID = "123456789012" # XXX DO NOT CHANGE - it will break moto tests K8S_WORKER_ROLE_NAME = "nodes.example.com" SAML_PROVIDER = "test-saml" -LOGGING["loggers"]["django_structlog"]["level"] = "WARNING" -LOGGING["loggers"]["controlpanel"]["level"] = "WARNING" +LOGGING["loggers"]["django_structlog"]["level"] = "WARNING" # noqa: F405 +LOGGING["loggers"]["controlpanel"]["level"] = "WARNING" # noqa: F405 AUTHENTICATION_BACKENDS = [ - 'rules.permissions.ObjectPermissionBackend', - 'django.contrib.auth.backends.ModelBackend', + "rules.permissions.ObjectPermissionBackend", + "django.contrib.auth.backends.ModelBackend", ] -MIDDLEWARE.remove('mozilla_django_oidc.middleware.SessionRefresh') -REST_FRAMEWORK['DEFAULT_AUTHENTICATION_CLASSES'].remove( - 'mozilla_django_oidc.contrib.drf.OIDCAuthentication', +MIDDLEWARE.remove("mozilla_django_oidc.middleware.SessionRefresh") # noqa: F405 +REST_FRAMEWORK["DEFAULT_AUTHENTICATION_CLASSES"].remove( # noqa: F405 + "mozilla_django_oidc.contrib.drf.OIDCAuthentication", ) OIDC_OP_JWKS_ENDPOINT = "https://example.com/.well-known/jwks.json" OIDC_ALLOW_UNSECURED_JWT = True OIDC_DOMAIN = "oidc.idp.example.com" -TOOLS_DOMAIN = 'example.com' +TOOLS_DOMAIN = "example.com" CSRF_COOKIE_SECURE = False SESSION_COOKIE_SECURE = False @@ -30,4 +31,4 @@ SLACK = { "api_token": "test-slack-api-token", "channel": "test-slack-channel", -} \ No newline at end of file +} From e01a1cc20f68fdeab37eac5a2e184d7dc88daa9b Mon Sep 17 00:00:00 2001 From: ymao2 Date: Fri, 4 Nov 2022 17:26:21 +0000 Subject: [PATCH 12/75] ANPL-1257 Added nomis login option for dashboard app --- controlpanel/api/auth0.py | 165 ++++++++++++++++-- .../api/auth0_conns/auth0_nomis/config.yaml | 15 ++ .../auth0_nomis/fetchUserProfile.js | 37 ++++ controlpanel/api/models/app.py | 7 +- controlpanel/api/validators.py | 14 ++ controlpanel/frontend/forms.py | 94 ++++++++-- .../includes/auth0-connections-form.html | 101 +++++++++++ .../webapp-auth0-connections-update.html | 43 +++++ .../frontend/jinja2/webapp-create.html | 43 +++-- .../frontend/jinja2/webapp-detail.html | 11 +- controlpanel/frontend/urls.py | 5 + controlpanel/frontend/views/__init__.py | 1 + controlpanel/frontend/views/app.py | 38 +++- settings.yaml | 5 + 14 files changed, 528 insertions(+), 51 deletions(-) create mode 100644 controlpanel/api/auth0_conns/auth0_nomis/config.yaml create mode 100644 controlpanel/api/auth0_conns/auth0_nomis/fetchUserProfile.js create mode 100644 controlpanel/frontend/jinja2/includes/auth0-connections-form.html create mode 100644 controlpanel/frontend/jinja2/webapp-auth0-connections-update.html diff --git a/controlpanel/api/auth0.py b/controlpanel/api/auth0.py index 3eb14b5d6..012d29f77 100644 --- a/controlpanel/api/auth0.py +++ b/controlpanel/api/auth0.py @@ -1,5 +1,8 @@ import structlog - +import base64 +import yaml +from pathlib import Path +from collections import defaultdict from rest_framework.exceptions import APIException from auth0.v3 import authentication, exceptions @@ -9,6 +12,7 @@ from auth0.v3.management.device_credentials import DeviceCredentials from auth0.v3.management.users import Users from auth0.v3.management.rest import RestClient +from jinja2 import Environment from django.conf import settings @@ -91,18 +95,52 @@ def _access_token(self, audience): return token["access_token"] - def _disable_all_connections(self, client_id, ignores=[]): - ignore_ids = [connection["id"] for connection in ignores] + def _enable_connections_for_new_client(self, client_id, chosen_connections): + """ + When an auth0 client is created, by default all the available connections are enabled for this client except + those which is not used by any app. This way is quite annoying, it means we have to go through all those + unchosen connections to diable the client from it, then enable nomis login if nomis login has been chosen + """ + # auth0_connections = [ + # self.connections.search_first_match(dict(name=connection)) for connection in chosen_connections + # ] + # ignore_ids = [connection["id"] for connection in auth0_connections if connection] + + # disable the client from unchosen connections connections = self.connections.get_all() for connection in connections: - if connection["id"] in ignore_ids: - continue - if client_id in connection["enabled_clients"]: + if connection["name"] in chosen_connections: + self.connections.enable_client(connection, client_id) + else: self.connections.disable_client(connection, client_id) + + def _create_custom_connection(self, app_name, connections): + new_connections = [] + for connection_name, user_inputs in connections.items(): + if connection_name in self.connections.custom_connections: + if "name" not in user_inputs: + user_inputs["name"] = app_name + new_connection_name = self.connections.create_custom_connection( + connection_name=connection_name, + input_values=user_inputs) + new_connections.append(new_connection_name) + else: + new_connections.append(connection_name) + return new_connections + def setup_auth0_client(self, app_name, connections=None): + """ + parameters: + connections: + { + : { 0 + def get_client_enabled_connections(self, app_name): + """ + There is no Auth0 API to get the list of enabled connection for a client, so we have to get all social + connections, then check whether the client (client_id) is in the list of enabled_clients + """ + client = self.clients.search_first_match(dict(name=app_name)) + if not client: + return [] + else: + connections = self.connections.get_all() + enabled_connections = [] + for connection in connections: + if client["client_id"] in connection["enabled_clients"]: + enabled_connections.append(connection["name"]) + return enabled_connections + + def update_client_auth_connections(self, app_name: str, new_conns: dict, existing_conns: list): + """ + There is no Auth0 API to get the list of enabled connection for a client, so we have to get all social + connections, then check whether the client (client_id) is in the list of enabled_clients + """ + client = self.clients.search_first_match(dict(name=app_name)) + if client: + connections = new_conns or {"email": {}} + new_connections = self._create_custom_connection(app_name, connections) + + # Get the list of removed connections based on the existing connections + removed_connections = list(set(existing_conns) - set(new_connections)) + real_new_connections = list(set(new_connections) - set(existing_conns)) + + # Remove the old connections + auth0_connections = [ + self.connections.search_first_match(dict(name=connection)) for connection in removed_connections + ] + for connection in auth0_connections: + self.connections.disable_client(connection, client["client_id"]) + + # Enable the new connections + auth0_connections = [ + self.connections.search_first_match(dict(name=connection)) for connection in real_new_connections + ] + for connection in auth0_connections: + self.connections.enable_client(connection, client["client_id"]) + class Auth0API(object): @@ -319,6 +397,7 @@ class ExtendedClients(ExtendedAPIMethods, Clients): endpoint = 'clients' + class ExtendedDeviceCredentials(ExtendedAPIMethods, DeviceCredentials): endpoint = 'device-credentials' @@ -326,11 +405,67 @@ class ExtendedDeviceCredentials(ExtendedAPIMethods, DeviceCredentials): class ExtendedConnections(ExtendedAPIMethods, Connections): endpoint = 'connections' + def __init__(self, domain, token, telemetry=True, timeout=5.0): + super(ExtendedConnections, self).__init__(domain=domain, token=token, telemetry=telemetry, timeout=timeout) + self.custom_connections = self._get_custom_connections_from_setting() + def disable_client(self, connection, client_id): if client_id in connection["enabled_clients"]: connection["enabled_clients"].remove(client_id) self.update(connection["id"], body={"enabled_clients": connection["enabled_clients"]}) + def enable_client(self, connection, client_id): + if client_id not in connection["enabled_clients"]: + connection["enabled_clients"].append(client_id) + self.update(connection["id"], body={"enabled_clients": connection["enabled_clients"]}) + + def _get_template_path_for_custom_connection(self, connection_name: str): + return Path(__file__).parents[0] / Path("auth0_conns") / Path(connection_name) + + def _get_custom_connections_from_setting(self): + return (settings.CUSTOM_AUTH_CONNECTIONS or "").split() + + def get_all_connection_names(self): + connections = super(ExtendedConnections, self).get_all() + connection_names = [connection["name"] for connection in connections] + connection_names.extend(self.custom_connections) + return connection_names + + def _get_default_settings_for_custom_connection(self, connection_name, input_values): + input_values["gateway_url"] = "" + if hasattr(settings, "{}_gateway_url".format(connection_name).upper()): + input_values["gateway_url"] = getattr(settings, "{}_gateway_url".format(connection_name).upper()) + + def create_custom_connection(self, connection_name: str, input_values: dict()): + """ This method is only used to create custom connections which has configuration file within this repo""" + jinja_env = Environment() + jinja_env.filters["base64enc"] = lambda x: base64.urlsafe_b64encode( + x.encode("utf8") + ).decode() + + # render the scripts + template_path = self._get_template_path_for_custom_connection(connection_name) + scripts = template_path.glob("*.js") + script_templates = { + x.stem: jinja_env.from_string(x.open(encoding="utf8").read()) + for x in scripts + } + scripts_rendered = {} + self._get_default_settings_for_custom_connection(connection_name, input_values) + for name, script_template in script_templates.items(): + scripts_rendered[name] = script_template.render(**input_values) + + # render the main connection template + with (template_path / Path("config.yaml")).open("r") as config_yaml_file: + yaml_rendered = jinja_env.from_string(config_yaml_file.read()).render( + **input_values + ) + body = yaml.safe_load(yaml_rendered) or defaultdict(dict) + body["options"]["scripts"] = scripts_rendered + + self.create(body) + return input_values["name"] + class ExtendedUsers(ExtendedAPIMethods, Users): endpoint = 'users' diff --git a/controlpanel/api/auth0_conns/auth0_nomis/config.yaml b/controlpanel/api/auth0_conns/auth0_nomis/config.yaml new file mode 100644 index 000000000..808eb07a0 --- /dev/null +++ b/controlpanel/api/auth0_conns/auth0_nomis/config.yaml @@ -0,0 +1,15 @@ +--- +options: + scripts: {} + client_id: {{client_id}} + client_secret: {{client_secret}} + authorizationURL: {{gateway_url}}/auth/oauth/authorize + tokenURL: {{gateway_url}}/auth/oauth/token + scope: '' + customHeaders: + Authorization: Basic {{ (client_id + ':' + client_secret)|base64enc }} + Content-Type: application/json +strategy: oauth2 +name: {{name}} +is_domain_connection: false +enabled_clients: [] diff --git a/controlpanel/api/auth0_conns/auth0_nomis/fetchUserProfile.js b/controlpanel/api/auth0_conns/auth0_nomis/fetchUserProfile.js new file mode 100644 index 000000000..de52f6002 --- /dev/null +++ b/controlpanel/api/auth0_conns/auth0_nomis/fetchUserProfile.js @@ -0,0 +1,37 @@ +function(accessToken, ctx, cb) { + var base_url = "{{gateway_url}}"; + var user_endpoint = "/auth/api/user/me"; + var user_profile_url = base_url + user_endpoint; + + // call oauth2 API with the accesstoken and create the profile + request.get( + user_profile_url, + { + headers: { + Authorization: "Bearer " + accessToken + } + }, + function(err, resp, body) { + if (err) { + cb(err); + return; + } + if (!/^2/.test("" + resp.statusCode)) { + cb(body); + return; + } + var parsedBody = JSON.parse(body); + var profile = { + user_id: parsedBody.staffId, + nickname: parsedBody.name, + name: parsedBody.name, + email: parsedBody.username + "+" + parsedBody.activeCaseLoadId + "@nomis", + username: parsedBody.username, + blocked: !parsedBody.active, + activeCaseLoadId: parsedBody.activeCaseLoadId, + _nomisAccessToken: accessToken + }; + cb(null, profile); + } + ); +} diff --git a/controlpanel/api/models/app.py b/controlpanel/api/models/app.py index d85eaa72b..d22fa6d51 100644 --- a/controlpanel/api/models/app.py +++ b/controlpanel/api/models/app.py @@ -48,6 +48,10 @@ def customers(self): auth0.ExtendedAuth0().groups.get_group_members(group_name=self.slug) or [] ) + @property + def auth0_connections(self): + return auth0.ExtendedAuth0().get_client_enabled_connections(self.slug) + @property def app_aws_secret_name(self): return f"{settings.ENV}/apps/{self.slug}/auth" @@ -62,10 +66,11 @@ def get_secret_key(self, name): return self.app_aws_secret_name def construct_secret_data(self, client): + """ The assumption is per app per callback url""" return { "client_id": client["client_id"], "client_secret": client["client_secret"], - "callbacks": client["callbacks"], + "callbacks": client["callbacks"][0] if len(client["callbacks"])>=1 else "" } def add_customers(self, emails): diff --git a/controlpanel/api/validators.py b/controlpanel/api/validators.py index 0e2b90772..8054d66e9 100644 --- a/controlpanel/api/validators.py +++ b/controlpanel/api/validators.py @@ -32,6 +32,20 @@ def validate_env_prefix(value): ) +validate_auth0_conn_name = RegexValidator( + regex='^([a-z][a-z0-9-]*[a-z0-9])([a-z][a-z0-9-]*[a-z0-9])*$', + message="is invalid, check Auth0 connection name restrictions (for example, " + "can only start and end with alphanumeric alphanumeric, only contain alphanumeric and hyphens)", +) + + +validate_auth0_client_id = RegexValidator( + regex='^([a-z][a-z0-9-]*[a-z0-9])(_[a-z][a-z0-9-]*[a-z0-9])*$', + message="is invalid, check Auth0 connection name restrictions (for example, " + "can only contain alphanumeric, underscores and hyphens)", +) + + def validate_github_repository_url(value): github_base_url = "https://github.com/" diff --git a/controlpanel/frontend/forms.py b/controlpanel/frontend/forms.py index 3cb483c20..f203f8776 100644 --- a/controlpanel/frontend/forms.py +++ b/controlpanel/frontend/forms.py @@ -14,6 +14,8 @@ from controlpanel.api.models.access_to_s3bucket import S3BUCKET_PATH_REGEX from controlpanel.api.models.iam_managed_policy import POLICY_NAME_REGEX from controlpanel.api.models.ip_allowlist import IPAllowlist +from controlpanel.api import auth0 + APP_CUSTOMERS_DELIMITERS = re.compile(r"[,; ]+") @@ -27,7 +29,71 @@ def label_from_instance(self, instance): return instance.name -class CreateAppForm(forms.Form): +class AppAuth0Form(forms.Form): + + def __init__(self, *args, **kwargs): + self.request = kwargs.pop("request", None) + self.auth0_connections = kwargs.pop("auth0_connections", ["email"]) + super(AppAuth0Form, self).__init__(*args, **kwargs) + + self.custom_connections = auth0.ExtendedAuth0().connections.custom_connections + self._create_inputs_for_custom_connections() + + def _create_inputs_for_custom_connections(self): + connections = auth0.ExtendedAuth0().connections.get_all_connection_names() + self.fields["connections"] = forms.MultipleChoiceField( + required=False, + initial=self.auth0_connections, + choices=list(zip(connections, connections)) + ) + for connection in self.custom_connections: + self.fields["{}_auth0_client_id".format(connection)] = forms.CharField( + max_length=128, + required=False, + validators=[validators.validate_auth0_client_id]) + self.fields["{}_auth0_client_secret".format(connection)] = forms.CharField( + widget=forms.PasswordInput, + required=False) + self.fields["{}_auth0_conn_name".format(connection)] = forms.CharField( + max_length=128, + required=False, + validators=[validators.validate_auth0_conn_name]) + + def _chosen_custom_connections(self, connections): + return list(set(self.custom_connections) & set(connections)) + + def _check_inputs_for_custom_connection(self, cleaned_data): + auth0_connections = cleaned_data.get('connections') + auth0_conn_data = {} + chosen_custom_connections = self._chosen_custom_connections(auth0_connections) + for connection in auth0_connections: + auth0_conn_data[connection] = {} + if connection not in chosen_custom_connections: + continue + + if cleaned_data.get("{}_auth0_client_id".format(connection), '') == '': + self.add_error("{}_auth0_client_id".format(connection), "This field is required.") + + if cleaned_data.get("{}_auth0_client_secret".format(connection), '') == '': + self.add_error("{}_auth0_client_secret".format(connection), "This field is required.") + + conn_name = cleaned_data.get("{}_auth0_conn_name".format(connection), '') + if conn_name == '': + self.add_error("{}_auth0_conn_name".format(connection), "This field is required.") + else: + if (conn_name, conn_name) in self.fields["connections"].choices: + self.add_error("{}_auth0_conn_name".format(connection), + "This name has been existed in the connections.") + + auth0_conn_data[connection] = { + "client_id": cleaned_data.get("{}_auth0_client_id".format(connection)), + "client_secret": cleaned_data.get("{}_auth0_client_secret".format(connection)), + "name": cleaned_data.get("{}_auth0_conn_name".format(connection)), + } + return auth0_conn_data + + +class CreateAppForm(AppAuth0Form): repo_url = forms.CharField( max_length=512, validators=[ @@ -58,25 +124,15 @@ class CreateAppForm(forms.Form): required=False, ) - connections = forms.MultipleChoiceField( - required=True, - initial="email", - choices=[("email", "email")], - ) - disable_authentication = forms.BooleanField(required=False) - def __init__(self, *args, **kwargs): - self.request = kwargs.pop("request", None) - super().__init__(*args, **kwargs) - def clean(self): cleaned_data = super().clean() - connect = cleaned_data["connect_bucket"] + connect_data_source = cleaned_data["connect_bucket"] new_datasource = cleaned_data.get("new_datasource_name") existing_datasource = cleaned_data.get("existing_datasource_id") - if connect == "new": + if connect_data_source == "new": if new_datasource: try: S3Bucket.objects.get(name=new_datasource) @@ -90,9 +146,11 @@ def clean(self): else: self.add_error("new_datasource_name", "This field is required.") - if connect == "existing" and not existing_datasource: + if connect_data_source == "existing" and not existing_datasource: self.add_error("existing_datasource_id", "This field is required.") + cleaned_data["auth0_connections"] = self._check_inputs_for_custom_connection(cleaned_data) + return cleaned_data def clean_repo_url(self): @@ -110,6 +168,14 @@ def clean_repo_url(self): return value +class UpdateAppAuth0ConnectionsForm(AppAuth0Form): + + def clean(self): + cleaned_data = super(UpdateAppAuth0ConnectionsForm, self).clean() + cleaned_data["auth0_connections"] = self._check_inputs_for_custom_connection(cleaned_data) + return cleaned_data + + class CreateDatasourceForm(forms.Form): name = forms.CharField( max_length=63, diff --git a/controlpanel/frontend/jinja2/includes/auth0-connections-form.html b/controlpanel/frontend/jinja2/includes/auth0-connections-form.html new file mode 100644 index 000000000..c2221311a --- /dev/null +++ b/controlpanel/frontend/jinja2/includes/auth0-connections-form.html @@ -0,0 +1,101 @@ +{% from "input/macro.html" import govukInput %} +{% from "label/macro.html" import govukLabel %} +{% from "fieldset/macro.html" import govukFieldset %} + + +{% macro auth0_connections_form(params) -%} + +{% set innerHtml %} + {% set main_field_error_message = params.errors.get("connections") %} + {% if main_field_error_message %} + {% set errorId = 'connections-error' %} + {{ govukErrorMessage({ + "id": errorId, + "html": main_field_error_message|join(". "), + }) }} + {% endif %} + +
+ {% for item in params.field.choices %} + {% set id = "connections-" + loop.index|string %} + {% set name = "connections" %} + {% set itemHintId = id + '-item-hint' %} +
+ + {{ govukLabel({ + "text": item[0], + "classes": 'govuk-checkboxes__label' + (' ' + (item.label|default({})).classes|default("")), + "attributes": (item.label|default({})).attributes|default(""), + "for": id + }) }} + {% set client_id_field_name = item[1] + '_auth0_client_id' %} + {% set client_secret_field_name = item[1] + '_auth0_client_secret' %} + {% set conn_name_field_name = item[1] + '_auth0_conn_name' %} + {% set has_linked_client_fields = (form.fields.get(client_id_field_name, None)) %} + + {% if has_linked_client_fields %} + {% set client_id_field = form.fields.get(client_id_field_name) %} + {% set client_secret_field = form.fields.get(client_secret_field_name) %} + {% set conn_name_field = form.fields.get(conn_name_field_name) %} + (Please provide the client credential from the provider) +
+
+ {{ govukInput({ + "name": conn_name_field_name, + "classes": "govuk-!-width-one-half", + "label": { + "text": "connection name", + "classes": "govuk-label--s", + }, + "errorMessage": {"text": params.errors[conn_name_field_name]|join(". ")} if params.errors.get(conn_name_field_name) else {}, + "value": form.auth0_nomis_auth0_conn_name.value() + }) }} + + {{ govukInput({ + "name": client_id_field_name, + "classes": "govuk-!-width-one-half", + "label": { + "text": "client id", + "classes": "govuk-label--s", + }, + "errorMessage": {"text": params.errors[client_id_field_name]|join(". ")} if params.errors.get(client_id_field_name) else {}, + "value": form.auth0_nomis_auth0_client_id.value() + }) }} + + {{ + govukInput({ + "type":"password", + "label": { + "text": "Client secret", + "classes": "govuk-label--s", + }, + "classes": "govuk-!-width-two-thirds", + "name": client_secret_field_name, + "value": "", + "errorMessage": {"text": params.errors[client_secret_field_name]|join(". ")} if params.errors.get(client_secret_field_name) else {}, + }) + }} +
+ {% endif %} +
+ {% endfor %} +
+{% endset -%} + +
+{% if params.fieldset %} + {% call govukFieldset({ + "describedBy": describedBy, + "classes": params.fieldset.classes, + "attributes": params.fieldset.attributes, + "legend": params.fieldset.legend + }) %} + {{ innerHtml }} + {% endcall %} +{% else %} + {{ innerHtml }} +{% endif %} +
+ +{%- endmacro %} diff --git a/controlpanel/frontend/jinja2/webapp-auth0-connections-update.html b/controlpanel/frontend/jinja2/webapp-auth0-connections-update.html new file mode 100644 index 000000000..37c3c7fc7 --- /dev/null +++ b/controlpanel/frontend/jinja2/webapp-auth0-connections-update.html @@ -0,0 +1,43 @@ +{% from "input/macro.html" import govukInput %} +{% from "fieldset/macro.html" import govukFieldset %} +{% from "error-message/macro.html" import govukErrorMessage %} +{% from "includes/auth0-connections-form.html" import auth0_connections_form with context %} + +{% extends "base.html" %} + +{% set legend -%} + Update auth0 connections - {{ app.name }} +{%- endset %} + +{% set page_name = "Auth0 connections" %} + +{% block content %} +

{{ legend }}

+ +
+ {% if form.errors %} + {{ govukErrorMessage({"text": form.errors}) }} + {% endif %} +
+ +
+ {{ csrf_input }} + + {{ auth0_connections_form({ + "fieldset": { + "legend": { + "text": "Oauth0 client - connections", + "classes": "govuk-fieldset__legend--m", + }, + }, + "field": form.fields['connections'], + "errors": form.errors, + "selected_values": form.connections.value() + } ) + }} + + +
+ +{% endblock %} + \ No newline at end of file diff --git a/controlpanel/frontend/jinja2/webapp-create.html b/controlpanel/frontend/jinja2/webapp-create.html index 80c690efa..33be399fb 100644 --- a/controlpanel/frontend/jinja2/webapp-create.html +++ b/controlpanel/frontend/jinja2/webapp-create.html @@ -3,6 +3,8 @@ {% from "label/macro.html" import govukLabel %} {% from "radios/macro.html" import govukRadios %} {% from "checkboxes/macro.html" import govukCheckboxes %} +{% from "includes/auth0-connections-form.html" import auth0_connections_form with context %} + {% extends "base.html" %} @@ -54,6 +56,7 @@

{{ page_title }}