ANPL-1701 post migration clearup

controlpanel/api/models/app.py

@@ -249,15 +249,6 @@ def auth0_client_name(self, env_name=None):
         return settings.AUTH0_CLIENT_NAME_PATTERN.format(
             app_name=client_name, env=env_name)
 
-    @property
-    def migration_info(self):
-        # TODO: using app.description for temporary place for storing old app info,
-        #  The content of this field should be removed after app migration is completed.
-        try:
-            return json.loads(self.description).get("migration", {})
-        except ValueError:
-            return {}
-
     def app_url_name(self, env_name):
         format_pattern = settings.APP_URL_NAME_PATTERN.get(env_name.upper())
         if not format_pattern:

controlpanel/cli/management/commands/post_migration_clearup.py

@@ -0,0 +1,92 @@
+
+# Third-party
+from django.core.management.base import BaseCommand
+
+# First-party/Local
+from controlpanel.api.models import App, AppS3Bucket
+from controlpanel.api import auth0
+
+
+class Command(BaseCommand):
+    help = "Clear up the redundant resources for migrated apps " \
+           "- remove the old auth0 related resources " \
+           "- remove old auth0 information from control db" \
+           "- remove the apps which are not required any more"
+
+    SCRIPT_LOG_FILE_NAME = "./clear_up_auth0_resources_log.txt"
+
+    EXCEPTION_APPS = ["gold-scorecard-form"]
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "-a", "--apply", action="store_true", help="Apply the actions"
+        )
+
+    def _remove_old_auth0_clients(self, app, auth0_instance):
+        old_client_info = app.app_conf.get(App.KEY_WORD_FOR_AUTH_SETTINGS, {}).\
+            get(App.DEFAULT_AUTH_CATEGORY, {})
+        if not old_client_info:
+            self._log_info(f"No old client for {app.slug} - {app.repo_url}")
+            return
+
+        self._log_info(f"Removing the old client for {app.slug} - {app.repo_url}")
+        if self.apply_action:
+            auth0_instance.clear_up_app(old_client_info)
+
+    def _update_db(self, app):
+        self._log_info(f"Removing the migration info and old clients for {app.slug} - {app.repo_url}")
+        app.description = ""
+        if App.DEFAULT_AUTH_CATEGORY in app.app_conf.get(App.KEY_WORD_FOR_AUTH_SETTINGS, {}):
+            del app.app_conf[App.KEY_WORD_FOR_AUTH_SETTINGS][App.DEFAULT_AUTH_CATEGORY]
+        if self.apply_action:
+            app.save()
+
+    def _remove_application(self, app):
+        self._log_info(f"Removing the application {app.slug} - {app.repo_url}")
+
+        """ TODO: how to deal with related bucket? we will output
+        the related datasets from this script"""
+        # log the related buckets information into file
+        related_buckets = AppS3Bucket.objects.filter(app_id=app.id)
+        for item in related_buckets:
+            self._log_info(f"The app links the bucket - {item.s3bucket.name}")
+            # Remove the relationship to avoid removal of the bucket when removing the app
+            if self.apply_action:
+                item.delete()
+        if self.apply_action:
+            app.delete()
+
+    def _log_info(self, info):
+        self.stdout.write(info)
+        with open(self.SCRIPT_LOG_FILE_NAME, "a") as f:
+            f.write(info)
+            f.write("\n")
+
+    def _clear_up_resources(self, auth0_instance):
+        apps = App.objects.all()
+        counter = 1
+        for app in apps:
+            if app.slug in self.EXCEPTION_APPS:
+                self._log_info(f"Ignore the application {app.slug}")
+                continue
+
+            try:
+                self._log_info(f"{counter}--Processing the application {app.slug}")
+
+                self._remove_old_auth0_clients(app, auth0_instance)
+                if "moj-analytical-services" in app.repo_url:
+                    self._remove_application(app)
+                else:
+                    self._update_db(app)
+                self._log_info(f"{counter}--Done with the application {app.slug}")
+                counter += 1
+            except Exception as ex:
+                self._log_info(f"Failed to process {app.slug} due to error : {ex.__str__()}")
+
+    def handle(self, *args, **options):
+        self.stdout.write("start to scan the apps from database.")
+        auth0_instance = auth0.ExtendedAuth0()
+        self.apply_action = options.get('apply') or False
+        self._clear_up_resources(auth0_instance)
+        self.stdout.write("Clean up action has completed.")
+

controlpanel/frontend/jinja2/webapp-detail.html

@@ -13,7 +13,6 @@
 {% set page_title = app.name %}
 {% set app_domain = settings.APP_DOMAIN %}
 {% set app_old_url = "https://" + app.slug + "." + settings.APP_DOMAIN_BEFORE_MIGRATION %}
-{% set feature_enabled = settings.features.app_migration.enabled %}
 
 {% set app_admins_html %}
 {% include "modals/app_admins.html" %}
@@ -26,53 +25,17 @@
 {% block content %}
 
 <header>
-  {% if feature_enabled %}
-  <div class="govuk-width-container">
-    <div class="govuk-warning-text">
-      <span class="govuk-warning-text__icon" aria-hidden="true">!</span>
-      <strong class="govuk-warning-text__text">
-        <span class="govuk-warning-text__assistive">Warning</span>
-        More information and user guidance for the webapp deployment pipeline and related settings will be provided soon. If you have questions about the app migration process, please contact us on the <a href=https://mojdt.slack.com/archives/C0504KY0Z2N>#data-platform-application-migration-support</a> Slack channel.
-      </strong>
-    </div>
-    <hr class="govuk-section-break govuk-section-break--visible">
-  </div>
-  <br/>
-  {% endif %}
 
   <span class="govuk-caption-xl">Webapp</span>
   <h1 class="govuk-heading-xl">{{ page_title }}</h1>
 
-  {% if feature_enabled %}
-      <h2 class="govuk-heading-m">Migration status:</h2>
-      {% if app_migration_status == "in_progress" %}
-        <p class="govuk-body" ><b>This app is currently being migrated. It has been deployed onto Cloud Platform for testing, and the original version is still available on the Alpha cluster.</b></p>
-        <ul class="govuk-body">
-          <li>Old repository: <a href="{{ app_migration_info["repo_url"] }}">{{ app_migration_info["repo_url"] }}</a></li>
-          <li>Old app name: <strong>{{ app_migration_info["app_name"] }}</strong></li>
-          <li>Old app URL: <a href="{{ app_migration_info["app_url"] }}">{{ app_migration_info["app_url"] }}</a></li>
-        </ul>
-        <p class="govuk-body" >
-          <b style="color:red">The above information about old app will be removed once the migration process is completed.</b>
-        </p>
-      {% elif app_migration_status == "done" %}
-        <p class="govuk-body">
-          <b style="color:red">The migration for this app has been completed</b>
-        </p>
-      {% else %}
-        <p class="govuk-body">
-          <b style="color:red">The migration process for this app has not been started yet!</b>
-        </p>
-      {% endif %}
-  {% else %}
-    {% if app.description %}
-      <p class="govuk-body">
-        {{ app.description }}
-      </p>
-    {% endif %}
+  {% if app.description %}
+    <p class="govuk-body">
+      {{ app.description }}
+    </p>
   {% endif %}
 
-  {% if feature_enabled and repo_access_error_msg %}
+  {% if repo_access_error_msg %}
   <div class="govuk-grid-row">
     <div class="govuk-column-two-thirds">
 
@@ -91,14 +54,6 @@ <h2 class="govuk-heading-m govuk-error-summary-heading">
   </div>
   {% endif %}
 
-  {% if not app_migration_info and app_old_url %}
-  <h2 class="govuk-heading-m">Deployed URL</h2>
-    <p> If the app was successfully built on the alpha (original) cluster, it can be accessed here:</p>
-    <p class="govuk-body">
-      <a href="{{ app_old_url }}">{{ app_old_url }}</a>
-    </p>
-  {% endif %}
-
   <h2 class="govuk-heading-m">Source Code Repository</h2>
   <p class="govuk-body">
     {% if app.repo_url %}
@@ -108,53 +63,49 @@ <h2 class="govuk-heading-m">Source Code Repository</h2>
     {% endif %}
   </p>
 
-  {% if feature_enabled %}
-
-    <h2 class="govuk-heading-m">App logs</h2>
-    <p class="govuk-body">
-      <a href="{{ settings.KIBANA_BASE_URL }}">{{ settings.KIBANA_BASE_URL }}</a>
-    </p>
+  <h2 class="govuk-heading-m">App logs</h2>
+  <p class="govuk-body">
+    <a href="{{ settings.KIBANA_BASE_URL }}">{{ settings.KIBANA_BASE_URL }}</a>
+  </p>
 
-    <h2 class="govuk-heading-m">App resources usage dashboard</h2>
-    <p class="govuk-body">
-      <a href="{{ settings.GRAFANA_BASE_URL }}">{{ settings.GRAFANA_BASE_URL }}</a>
-    </p>
+  <h2 class="govuk-heading-m">App resources usage dashboard</h2>
+  <p class="govuk-body">
+    <a href="{{ settings.GRAFANA_BASE_URL }}">{{ settings.GRAFANA_BASE_URL }}</a>
+  </p>
 
-    <h2 class="govuk-heading-m">Deployment Pipeline</h2>
-    <p class="govuk-body"> Github workflows on app's repo are used for deploying the app.</p>
-    <p class="govuk-body"><strong>You are required to be member of admin team for this app repo in order to be able to maintain the deployments settings</strong></p>
-  {% endif %}
+  <h2 class="govuk-heading-m">Deployment Pipeline</h2>
+  <p class="govuk-body"> Github workflows on app's repo are used for deploying the app.</p>
+  <p class="govuk-body"><strong>You are required to be member of admin team for this app repo in order to be able to maintain the deployments settings</strong></p>
 
 </header>
 
 <section class="cpanel-section track_task app-auth-settings-panel">
-  {% if feature_enabled %}
-    {% if github_settings_access_error_msg %}
-      <div class="govuk-error-summary" role="alert" aria-labelledby="error-summary-heading-example-1" tabindex="-1">
-        <p style="color:red">Couldn't load the github settings</p>
-        <p style="color:red">Raw error message: {{ github_settings_access_error_msg }}</p>
-      </div>
-    {% endif %}
 
-    {% for env_name, deployment_setting in deployments_settings.items() %}
-    <h2 class="govuk-heading-m" >Deployment settings under {{ env_name }}</h2>
-      {% if deployment_setting.get('is_redundant') and request.user.has_perm('api.update_app', app) %}
-        <section class="cpanel-section">
-          <p style="color:red">It appears this deployment environment is redundant and can be removed</p>
-          <form action="{{ url('remove-app-deployment-env', kwargs={'pk': app.id, 'env_name': env_name}) }}" method="post">
-            {{ csrf_input }}
-            <button class="govuk-button cpanel-button--destructive js-confirm"
-                    data-confirm-message="Are you sure you want to remove this deployment environment?">
-              Remove redundant deployment environment
-            </button>
-          </form>
-        </section>
-      {% else %}
-        {{ app_deployment_settings(app, env_name, app_domain, deployment_setting, request, csrf_input) }}
-      {% endif %}
-    {% endfor %}
+  {% if github_settings_access_error_msg %}
+    <div class="govuk-error-summary" role="alert" aria-labelledby="error-summary-heading-example-1" tabindex="-1">
+      <p style="color:red">Couldn't load the github settings</p>
+      <p style="color:red">Raw error message: {{ github_settings_access_error_msg }}</p>
+    </div>
   {% endif %}
 
+  {% for env_name, deployment_setting in deployments_settings.items() %}
+  <h2 class="govuk-heading-m" >Deployment settings under {{ env_name }}</h2>
+    {% if deployment_setting.get('is_redundant') and request.user.has_perm('api.update_app', app) %}
+      <section class="cpanel-section">
+        <p style="color:red">It appears this deployment environment is redundant and can be removed</p>
+        <form action="{{ url('remove-app-deployment-env', kwargs={'pk': app.id, 'env_name': env_name}) }}" method="post">
+          {{ csrf_input }}
+          <button class="govuk-button cpanel-button--destructive js-confirm"
+                  data-confirm-message="Are you sure you want to remove this deployment environment?">
+            Remove redundant deployment environment
+          </button>
+        </form>
+      </section>
+    {% else %}
+      {{ app_deployment_settings(app, env_name, app_domain, deployment_setting, request, csrf_input) }}
+    {% endif %}
+  {% endfor %}
+
 </section>
 
 <section class="cpanel-section">