Merge branch 'main' into python-311-test

Dockerfile

@@ -11,7 +11,7 @@ RUN /node_modules/.bin/jest
 
 FROM public.ecr.aws/docker/library/python:3.11-alpine3.18 AS base
 
-ARG HELM_VERSION=3.5.4
+ARG HELM_VERSION=3.14.1
 ARG HELM_TARBALL=helm-v${HELM_VERSION}-linux-amd64.tar.gz
 ARG HELM_BASEURL=https://get.helm.sh
 

controlpanel/api/cluster.py

@@ -389,7 +389,6 @@ class App(EntityResource):
     AUTHENTICATION_REQUIRED = "AUTHENTICATION_REQUIRED"
     AUTH0_PASSWORDLESS = "AUTH0_PASSWORDLESS"
     APP_ROLE_ARN = "APP_ROLE_ARN"
-    DATA_ACCOUNT_ID = 'DATA_ACCOUNT_ID'
 
     def __init__(self, app, github_api_token=None, auth0_instance=None):
         super(App, self).__init__()
@@ -415,7 +414,6 @@ def _create_secrets(self, env_name, client=None):
         secret_data: dict = {
             App.IP_RANGES: self.app.env_allowed_ip_ranges(env_name=env_name),
             App.APP_ROLE_ARN: self.app.iam_role_arn,
-            App.DATA_ACCOUNT_ID: settings.AWS_DATA_ACCOUNT_ID
         }
         if client:
             secret_data[App.AUTH0_CLIENT_ID] = client["client_id"]
@@ -501,7 +499,7 @@ def oidc_provider_statement(self):
                 "identity_provider_arn": iam_arn(
                     f"oidc-provider/{settings.OIDC_APP_EKS_PROVIDER}"
                 ),
-                "app_name": self.app.slug,
+                "app_namespace": self.app.namespace,
             }
         )
         return json.loads(statement)
@@ -510,6 +508,8 @@ def create_iam_role(self):
         assume_role_policy = deepcopy(BASE_ASSUME_ROLE_POLICY)
         assume_role_policy["Statement"].append(self.oidc_provider_statement)
         self.aws_role_service.create_role(self.iam_role_name, assume_role_policy)
+        for env in self.get_deployment_envs():
+            self._create_secrets(env_name=env)
 
     def grant_bucket_access(self, bucket_arn, access_level, path_arns):
         self.aws_role_service.grant_bucket_access(
@@ -534,24 +534,27 @@ def format_github_key_name(key_name):
         Format the self-defined secret/variable by adding prefix if
         create/update value back to github and there is no prefix in the name
         """
-        if key_name not in settings.AUTH_SETTINGS_ENVS \
-                and key_name not in settings.AUTH_SETTINGS_SECRETS:
-            if not key_name.startswith(settings.APP_SELF_DEFINE_SETTING_PREFIX):
-                return f"{settings.APP_SELF_DEFINE_SETTING_PREFIX}{key_name}"
-        return key_name
+        if key_name in settings.AUTH_SETTINGS_ENVS:
+            return key_name
+
+        if key_name in settings.AUTH_SETTINGS_SECRETS:
+            return key_name
+
+        if key_name.startswith(settings.APP_SELF_DEFINE_SETTING_PREFIX):
+            return key_name
+
+        return f"{settings.APP_SELF_DEFINE_SETTING_PREFIX}{key_name}"
 
     @staticmethod
-    def get_github_key_display_name(key_name):
+    def get_github_key_display_name(key_name: str) -> str:
         """
         Format the self-defined secret/variable by removing the prefix
         if reading it from github and there is prefix in the name
         """
-        if key_name and key_name not in settings.AUTH_SETTINGS_ENVS \
-                and key_name not in settings.AUTH_SETTINGS_SECRETS:
-            if settings.APP_SELF_DEFINE_SETTING_PREFIX in key_name:
-                return key_name.replace(
-                    settings.APP_SELF_DEFINE_SETTING_PREFIX, "")
-        return key_name
+        if not key_name.startswith(settings.APP_SELF_DEFINE_SETTING_PREFIX):
+            return key_name
+
+        return key_name.replace(settings.APP_SELF_DEFINE_SETTING_PREFIX, "", 1)
 
     def create_or_update_secret(self, env_name, secret_key, secret_value):
         org_name, repo_name = extract_repo_info_from_url(self.app.repo_url)

controlpanel/api/migrations/0032_app_namespace.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.7 on 2024-02-20 16:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("api", "0031_add_soft_delete_fields"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="app",
+            name="namespace",
+            field=models.CharField(blank=True, max_length=63, null=True, unique=True),
+        ),
+    ]

controlpanel/api/migrations/0033_add_namespaces_values_to_apps.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.7 on 2024-02-20 16:01
+
+from django.db import migrations
+
+
+def add_namespaces(apps, schema_editor):
+    App = apps.get_model("api", "App")
+    for app in App.objects.all():
+        app.namespace = f"data-platform-app-{app.slug}"
+        app.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("api", "0032_app_namespace"),
+    ]
+
+    operations = [
+        migrations.RunPython(code=add_namespaces, reverse_code=migrations.RunPython.noop)
+    ]

controlpanel/api/migrations/0034_remove_null_blank_from_namespace.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.7 on 2024-02-20 16:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("api", "0033_add_namespaces_values_to_apps"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="app",
+            name="namespace",
+            field=models.CharField(max_length=63, unique=True),
+        ),
+    ]

controlpanel/api/models/app.py

@@ -10,10 +10,9 @@
 from django_extensions.db.models import TimeStampedModel
 
 # First-party/Local
-from controlpanel.api import auth0, cluster
+from controlpanel.api import auth0, cluster, tasks
 from controlpanel.api.models import IPAllowlist
 from controlpanel.utils import github_repository_name, s3_slugify, webapp_release_name
-from controlpanel.api import tasks
 
 
 class App(TimeStampedModel):
@@ -35,6 +34,9 @@ class App(TimeStampedModel):
     # are not within the fields which will be searched frequently
     app_conf = models.JSONField(null=True)
 
+    # Stores the Cloud Platform namespace name
+    namespace = models.CharField(unique=True, max_length=63)
+
     # Non database field just for passing extra parameters
     disable_authentication = False
     connections = {}
@@ -252,12 +254,13 @@ def auth0_client_name(self, env_name=None):
 
     def app_url_name(self, env_name):
         format_pattern = settings.APP_URL_NAME_PATTERN.get(env_name.upper())
+        namespace = self.namespace.removeprefix("data-platform-app-")
         if not format_pattern:
             format_pattern = settings.APP_URL_NAME_PATTERN.get(self.DEFAULT_SETTING_KEY_WORD)
         if format_pattern:
-            return format_pattern.format(app_name=self.slug, env=env_name)
+            return format_pattern.format(app_name=namespace, env=env_name)
         else:
-            return self.slug
+            return namespace
 
     def get_auth_client(self, env_name):
         env_name = env_name or self.DEFAULT_AUTH_CATEGORY
@@ -314,7 +317,8 @@ class DeleteCustomerError(Exception):
 App.DeleteCustomerError = DeleteCustomerError
 
 
-from django.db.models.signals import post_save, post_delete
+# Third-party
+from django.db.models.signals import post_delete, post_save
 from django.dispatch import receiver
 
 
@@ -336,6 +340,7 @@ def trigger_app_create_related_messages(sender, instance, created, **kwargs):
 
 @receiver(post_delete, sender=App)
 def remove_app_related_tasks(sender, instance, **kwargs):
+    # First-party/Local
     from controlpanel.api.models import Task
     related_app_tasks = Task.objects.filter(entity_class="App", entity_id=instance.id)
     for task in related_app_tasks:

controlpanel/api/tasks/handlers/app.py

@@ -28,5 +28,6 @@ class CreateAppAWSRole(BaseModelTaskHandler):
     name = "create_app_aws_role"
 
     def handle(self):
-        cluster.App(self.object).create_iam_role()
+        task_user = User.objects.filter(pk=self.task_user_pk).first()
+        cluster.App(self.object, task_user.github_api_token).create_iam_role()
         self.complete()

controlpanel/api/templates/assume_roles/cloud_platform_oidc.json

@@ -9,8 +9,8 @@
         "StringEquals": {
             "{{ identity_provider }}:aud": "sts.amazonaws.com",
             "{{ identity_provider }}:sub": [
-                "system:serviceaccount:data-platform-app-{{ app_name }}-dev:data-platform-app-{{ app_name }}-dev-sa",
-                "system:serviceaccount:data-platform-app-{{ app_name }}-prod:data-platform-app-{{ app_name }}-prod-sa"
+                "system:serviceaccount:{{ app_namespace }}-dev:{{ app_namespace }}-dev-sa",
+                "system:serviceaccount:{{ app_namespace }}-prod:{{ app_namespace }}-prod-sa"
             ]
         }
     }

controlpanel/frontend/forms.py

@@ -116,7 +116,7 @@ def _check_inputs_for_custom_connection(self, cleaned_data):
         return auth0_conn_data
 
 
-class CreateAppForm(AppAuth0Form):
+class CreateAppForm(forms.Form):
 
     repo_url = forms.CharField(
         max_length=512,
@@ -148,8 +148,10 @@ class CreateAppForm(AppAuth0Form):
         empty_label="Select",
         required=False,
     )
+    namespace = forms.CharField(required=True, max_length=63)
 
     def __init__(self, *args, **kwargs):
+        self.request = kwargs.pop("request", None)
         super().__init__(*args, **kwargs)
         self.fields["existing_datasource_id"].queryset = self.get_datasource_queryset()
 
@@ -204,6 +206,18 @@ def clean_repo_url(self):
 
         return repo_url
 
+    def clean_namespace(self):
+        """
+        Removes the env suffix if the user included it
+        """
+        namespace = self.cleaned_data["namespace"]
+        for suffix in ["-dev", "-prod"]:
+            if suffix in namespace:
+                namespace = namespace.removesuffix(suffix)
+                break
+
+        return namespace
+
 
 class UpdateAppAuth0ConnectionsForm(AppAuth0Form):
     env_name = forms.CharField(widget=forms.HiddenInput)

controlpanel/frontend/jinja2/customers-list.html

@@ -20,7 +20,7 @@ <h2 class="govuk-heading-m">
 
 {% if not groups_dict %}
    <h2 class="govuk-heading-s">
-      No need to manage the customers of the app on Control panel as it does not require authentication
+      Customer management is disabled. To manage customers, create an Auth0 client and enable authentication on the <a href="{{ url("manage-app", kwargs={ "pk": app.id}) }}"> manage app page.</a>
    </h2>
 {% else %}
   <section>

controlpanel/frontend/jinja2/includes/app-list.html

@@ -37,8 +37,7 @@
         {{ yes_no(user.is_app_admin(app.id)) }}
       </td>
       <td class="govuk-table__cell">
-        <a href="{{ url("appcustomers-page", kwargs={"pk": app.id, "page_no": "1"}) }}"
-          class="govuk-button govuk-button--secondary right" >Manage customers</a>
+        <a href="{{ url("appcustomers-page", kwargs={"pk": app.id, "page_no": "1"}) }}" class="govuk-button govuk-button--secondary right">Manage customers</a>
       </td>
       <td class="govuk-table__cell">
         <a class="govuk-button govuk-button--secondary right {%- if not request.user.has_perm('api.retrieve_app', app) %} govuk-visually-hidden{% endif %}"

controlpanel/frontend/jinja2/webapp-create.html

@@ -23,7 +23,12 @@
       "text": '60 chars max, only lowercase letters, numbers, periods and hyphens, auto-prefixed with "' + env + '"'
     },
     "errorMessage": {"text": form.new_datasource_name.errors|join(". ")} if form.new_datasource_name.errors else {},
-    "value": form.new_datasource_name.value()
+    "value": form.new_datasource_name.value(),
+    "attributes": {
+      "data-bucket-prefix": env + "-",
+      "pattern": "[a-z0-9.-]{1,60}",
+      "maxlength": "60",
+    }
   }) }}
   </div>
 {% endset %}
@@ -68,13 +73,28 @@ <h1 class="govuk-heading-xl">{{ page_title }}</h1>
       }) }}
     {% endif %}
 
-    <input type="text" class="govuk-input" id="display_result_repo" name="repo_url" />
+    <input type="text" class="govuk-input" id="display_result_repo" name="repo_url" required />
 
-    <br/>
-    <br/>
+  </div>
+  <div class="govuk-form-group" id="container-element">
+
+    <label class="govuk-label govuk-label--m" for="display_result_repo">Cloud Platform namespace
+    </label>
+
+    {% set error_repo_msg = form.errors.get("namespace") %}
+    {% if error_repo_msg %}
+      {% set errorId = 'namespace-error' %}
+      {{ govukErrorMessage({
+        "id": errorId,
+        "html": error_repo_msg|join(". "),
+      }) }}
+    {% endif %}
+    <span class="govuk-hint">Enter namespace with the -env suffix removed</span>
+    <input type="text" class="govuk-input" id="id_namespace" name="namespace" required />
 
   </div>
 
+
   {{ govukRadios({
     "name": "connect_bucket",
     "fieldset": {
@@ -84,7 +104,7 @@ <h1 class="govuk-heading-xl">{{ page_title }}</h1>
       },
     },
     "hint": {
-      "text": "Connect an existing app data source to your app, or create a new one.",
+      "text": "Connect an existing app data source to your app, or create a new one. If you don't need to connect to an S3 bucket, select 'Do this later'",
     },
     "items": [
       {
@@ -95,6 +115,9 @@ <h1 class="govuk-heading-xl">{{ page_title }}</h1>
       {
         "value": "existing",
         "html": existing_datasource_html|safe,
+        "hint": {
+          "text": "Only buckets that you have admin access to are displayed",
+        },
         "checked": form.connect_bucket.value() == "existing"
       },
       {

controlpanel/frontend/views/app.py

@@ -141,8 +141,6 @@ def get_form_kwargs(self):
         kwargs = FormMixin.get_form_kwargs(self)
         kwargs.update(
             request=self.request,
-            all_connections_names=auth0.ExtendedAuth0().connections.get_all_connection_names(),  # noqa: E501
-            custom_connections=auth0.ExtendedConnections.custom_connections(),
         )
         return kwargs
 
@@ -151,7 +149,7 @@ def get_success_url(self):
             self.request,
             f"Successfully registered {self.object.name} webapp",
         )
-        return reverse_lazy("list-apps")
+        return reverse_lazy("manage-app", kwargs={"pk": self.object.pk})
 
     def form_valid(self, form):
         try:

controlpanel/frontend/views/app_variables.py

@@ -37,7 +37,7 @@ def get_form_kwargs(self):
         kwargs["initial"]["env_name"] = data.get("env_name")
         kwargs["initial"]["key"] = self.kwargs.get("var_name")
         kwargs["initial"]["display_key"] = cluster.App.get_github_key_display_name(
-            self.kwargs.get("var_name"))
+            self.kwargs.get("var_name", ""))
         if kwargs["initial"]["key"]:
             try:
                 var_info = cluster.App(

controlpanel/frontend/views/apps_mng.py

@@ -31,6 +31,7 @@ def register_app(self, user, app_data):
             name=name,
             repo_url=repo_url,
             current_user=user,
+            namespace=app_data["namespace"],
         )
         self._add_app_to_users(new_app, user)
         self._create_or_link_datasource(new_app, user, app_data)

controlpanel/frontend/views/secrets.py

@@ -40,7 +40,7 @@ def get_form_kwargs(self):
         kwargs["initial"]["env_name"] = data.get("env_name")
         kwargs["initial"]["key"] = self.kwargs.get("secret_name")
         kwargs["initial"]["display_key"] = cluster.App.get_github_key_display_name(
-            self.kwargs.get("secret_name"))
+            self.kwargs.get("secret_name", ""))
         return kwargs
 
     def get_success_url(self, app_id):

settings.yaml

@@ -95,12 +95,14 @@ AUTH_SETTINGS_SECRETS:
   - AUTH0_CLIENT_ID
   - AUTH0_CLIENT_SECRET
   - IP_RANGES
+  - APP_ROLE_ARN
 
 AUTH_SETTINGS_NO_EDIT:
   - AUTH0_CLIENT_ID
   - AUTH0_CLIENT_SECRET
   - AUTH0_DOMAIN
   - AUTH0_PASSWORDLESS
+  - APP_ROLE_ARN
 
 AUTH_SETTINGS_ENVS:
   - AUTH0_DOMAIN