From b7f594e49f6468ca20c2e67040393a8117d93756 Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Mon, 9 Oct 2023 16:27:20 +0100 Subject: [PATCH 01/10] Update S3bucket model to add fields to capture soft delete details --- .../migrations/0031_add_soft_delete_fields.py | 31 +++++++++++++++++++ controlpanel/api/models/s3bucket.py | 8 +++++ 2 files changed, 39 insertions(+) create mode 100644 controlpanel/api/migrations/0031_add_soft_delete_fields.py diff --git a/controlpanel/api/migrations/0031_add_soft_delete_fields.py b/controlpanel/api/migrations/0031_add_soft_delete_fields.py new file mode 100644 index 000000000..c66261360 --- /dev/null +++ b/controlpanel/api/migrations/0031_add_soft_delete_fields.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.1 on 2023-10-09 15:33 + +# Third-party +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0030_task'), + ] + + operations = [ + migrations.AddField( + model_name='s3bucket', + name='deleted_at', + field=models.DateTimeField(null=True), + ), + migrations.AddField( + model_name='s3bucket', + name='deleted_by', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='deleted_s3buckets', to=settings.AUTH_USER_MODEL), + ), + migrations.AddField( + model_name='s3bucket', + name='is_deleted', + field=models.BooleanField(default=False), + ), + ] diff --git a/controlpanel/api/models/s3bucket.py b/controlpanel/api/models/s3bucket.py index 90de336f3..0d3aecae2 100644 --- a/controlpanel/api/models/s3bucket.py +++ b/controlpanel/api/models/s3bucket.py @@ -56,6 +56,14 @@ class S3Bucket(TimeStampedModel): is_data_warehouse = models.BooleanField(default=False) # TODO remove this field - it's unused location_url = models.CharField(max_length=128, null=True) + is_deleted = models.BooleanField(default=False) + deleted_by = models.ForeignKey( + "User", + on_delete=models.SET_NULL, + null=True, + related_name="deleted_s3buckets" + ) + deleted_at = models.DateTimeField(null=True) objects = S3BucketQuerySet.as_manager() From 80c579a3fbba168be14fc1b0a9002e68e92faaca Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Tue, 10 Oct 2023 15:40:34 +0100 Subject: [PATCH 02/10] Implement soft delete for S3Bucket, and update DeleteDatasource view to call it Changes view to use form_valid rather than delete to handle deletion, as this was a change introduced in django 4.0. See docs/release notes for full info. --- controlpanel/api/models/s3bucket.py | 14 ++++++++++ controlpanel/frontend/views/datasource.py | 17 +++++++------ tests/api/models/test_s3bucket.py | 13 ++++++++++ tests/frontend/views/test_datasource.py | 31 ++++++++++++++++++++--- 4 files changed, 63 insertions(+), 12 deletions(-) diff --git a/controlpanel/api/models/s3bucket.py b/controlpanel/api/models/s3bucket.py index 0d3aecae2..8d72ed467 100644 --- a/controlpanel/api/models/s3bucket.py +++ b/controlpanel/api/models/s3bucket.py @@ -3,10 +3,12 @@ # Third-party from django.conf import settings +from django.contrib.auth.models import User from django.core.validators import MinLengthValidator from django.db import models from django.db.models import Q from django.db.transaction import atomic +from django.utils import timezone from django_extensions.db.models import TimeStampedModel # First-party/Local @@ -166,3 +168,15 @@ def delete(self, *args, **kwargs): if not self.is_folder: self.cluster.mark_for_archival() super().delete(*args, **kwargs) + + def soft_delete(self, deleted_by: User): + """ + Mark the object as deleted, but do not remove it from the database + """ + self.is_deleted = True + self.deleted_by = deleted_by + self.deleted_at = timezone.now() + self.save() + # TODO update to handle deleting folders + if not self.is_folder: + self.cluster.mark_for_archival() diff --git a/controlpanel/frontend/views/datasource.py b/controlpanel/frontend/views/datasource.py index 5099089ea..6572b4bdd 100644 --- a/controlpanel/frontend/views/datasource.py +++ b/controlpanel/frontend/views/datasource.py @@ -6,6 +6,7 @@ from django.contrib import messages from django.core.exceptions import PermissionDenied from django.db import transaction +from django.http import HttpResponseRedirect from django.shortcuts import get_object_or_404 from django.urls import reverse_lazy from django.views.generic.base import ContextMixin @@ -198,16 +199,16 @@ class DeleteDatasource( permission_required = "api.destroy_s3bucket" success_url = reverse_lazy("list-warehouse-datasources") - def delete(self, *args, **kwargs): - bucket = self.get_object() - if not bucket.is_data_warehouse: - self.success_url = reverse_lazy("list-webapp-datasources") - - response = super().delete(*args, **kwargs) + def get_success_url(self): + if not self.object.is_data_warehouse: + return reverse_lazy("list-webapp-datasources") + return self.success_url + def form_valid(self, *args, **kwargs): + self.object = self.get_object() + self.object.soft_delete(deleted_by=self.request.user) messages.success(self.request, "Successfully deleted data source") - - return response + return HttpResponseRedirect(self.get_success_url()) class UpdateAccessLevelMixin: diff --git a/tests/api/models/test_s3bucket.py b/tests/api/models/test_s3bucket.py index 9d9d89f95..deb17e0aa 100644 --- a/tests/api/models/test_s3bucket.py +++ b/tests/api/models/test_s3bucket.py @@ -96,3 +96,16 @@ def test_is_folder(name, expected): ) def test_cluster(name, expected): assert isinstance(S3Bucket(name=name).cluster, expected) + + +def test_soft_delete(bucket, users): + user = users["superuser"] + + assert bucket.is_deleted is False + with patch("controlpanel.api.cluster.S3Bucket.mark_for_archival") as archive: + bucket.soft_delete(deleted_by=user) + + assert bucket.is_deleted is True + assert bucket.deleted_by == user + assert bucket.deleted_at is not None + archive.assert_called_once() diff --git a/tests/frontend/views/test_datasource.py b/tests/frontend/views/test_datasource.py index 1f43f02a0..dc1f05658 100644 --- a/tests/frontend/views/test_datasource.py +++ b/tests/frontend/views/test_datasource.py @@ -3,7 +3,7 @@ # Third-party import pytest -from django.urls import reverse +from django.urls import reverse, reverse_lazy from model_mommy import mommy from rest_framework import status @@ -115,9 +115,10 @@ def create(client, *args, **kwargs): return client.post(reverse("create-datasource") + "?type=warehouse", data) -def delete(client, buckets, *args): - return client.delete( - reverse("delete-datasource", kwargs={"pk": buckets["warehouse1"].id}) +def delete(client, buckets, *args, bucket=None): + bucket = bucket or buckets["warehouse1"] + return client.post( + reverse("delete-datasource", kwargs={"pk": bucket.id}) ) @@ -392,3 +393,25 @@ def test_grant_access_invalid_form(client, users3buckets, users, kwargs): assert response.status_code == 200 assert response.context_data["form"].is_valid() is False + + +@pytest.mark.parametrize( + "bucket, success_url", + [ + ("warehouse1", reverse_lazy("list-warehouse-datasources")), + ("app_data1", reverse_lazy("list-webapp-datasources")), + ] +) +def test_delete_calls_soft_delete(client, buckets, users, bucket, success_url): + admin = users["bucket_admin"] + bucket = buckets[bucket] + + client.force_login(admin) + response = delete(client, buckets, bucket=bucket) + bucket.refresh_from_db() + + assert bucket.pk is not None + assert bucket.is_deleted is True + assert bucket.deleted_by == admin + assert bucket.deleted_at is not None + assert response.url == success_url From 30430f8ebcb1ea36d7e68d7b8dcc110e231e7f41 Mon Sep 17 00:00:00 2001 From: Michael Collins <15347726+michaeljcollinsuk@users.noreply.github.com> Date: Tue, 10 Oct 2023 17:37:17 +0100 Subject: [PATCH 03/10] Update datasource list views and templates - For admins, list deleted datasources - Fix bugs in templates so that admin view displays datasource type --- controlpanel/frontend/forms.py | 8 ++++--- .../frontend/jinja2/datasource-list.html | 13 +++++++---- controlpanel/frontend/views/datasource.py | 13 +++++++++-- tests/frontend/views/test_datasource.py | 23 ++++++++++--------- 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/controlpanel/frontend/forms.py b/controlpanel/frontend/forms.py index 9341dd333..4fe857c1c 100644 --- a/controlpanel/frontend/forms.py +++ b/controlpanel/frontend/forms.py @@ -146,7 +146,7 @@ class CreateAppForm(AppAuth0Form): required=False, ) existing_datasource_id = DatasourceChoiceField( - queryset=S3Bucket.objects.filter(is_data_warehouse=False), + queryset=S3Bucket.objects.filter(is_data_warehouse=False, is_deleted=False), empty_label="Select", required=False, ) @@ -353,9 +353,11 @@ def __init__(self, *args, **kwargs): if self.exclude_connected: self.fields["datasource"].queryset = S3Bucket.objects.exclude( id__in=[a.s3bucket_id for a in self.app.apps3buckets.all()], - ) + ).filter(is_deleted=False) else: - self.fields["datasource"].queryset = S3Bucket.objects.all() + self.fields["datasource"].queryset = S3Bucket.objects.filter( + is_deleted=False, + ) class CreateIAMManagedPolicyForm(forms.Form): diff --git a/controlpanel/frontend/jinja2/datasource-list.html b/controlpanel/frontend/jinja2/datasource-list.html index 8e2027d88..7c575bc6a 100644 --- a/controlpanel/frontend/jinja2/datasource-list.html +++ b/controlpanel/frontend/jinja2/datasource-list.html @@ -4,12 +4,12 @@ {% extends "base.html" %} -{% if datasource_type %} - {% set page_name = datasource_type + "-datasource-list" %} - {% set page_title = "Your " + datasource_type + " data sources" %} -{% else %} +{% if all_datasources %} {% set page_name = "all-datasources" %} {% set page_title = "All data sources" %} +{% else %} + {% set page_name = datasource_type + "-datasource-list" %} + {% set page_title = "Your " + datasource_type + " data sources" %} {% endif %} {% set access_levels_html %} @@ -71,4 +71,9 @@
- - Open on AWS - + {% if bucket.is_deleted %} +
+ This bucket was deleted by {{ user_name(bucket.deleted_by) }} on {{ bucket.deleted_at.strftime("%Y/%m/%d %H:%M:%S") }}. +
+ {% else %} + + Open on AWS + + {% endif %}