Changed lifecycle config to use intelligent tiering. (#1391)

* Changed lifecycle config to use intelligent tiering. Added admin button to change all existing buckets to use new lifecycle config

* Fixed failing test

controlpanel/api/aws.py

@@ -659,25 +659,7 @@ def create(self, bucket_name, is_data_warehouse=False):
             # Set bucket lifecycle. Send non-current versions of files to glacier
             # storage after 30 days.
             # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.put_bucket_lifecycle_configuration  # noqa: E501
-            lifecycle_id = f"{bucket_name}_lifecycle_configuration"
-            s3_client.put_bucket_lifecycle_configuration(
-                Bucket=bucket_name,
-                LifecycleConfiguration={
-                    "Rules": [
-                        {
-                            "ID": lifecycle_id,
-                            "Status": "Enabled",
-                            "Prefix": "",
-                            "NoncurrentVersionTransitions": [
-                                {
-                                    "NoncurrentDays": 30,
-                                    "StorageClass": "GLACIER",
-                                },
-                            ],
-                        },
-                    ]
-                },
-            )
+            self.apply_lifecycle_config(bucket_name, s3_client)
             if is_data_warehouse:
                 self._tag_bucket(bucket, {"buckettype": "datawarehouse"})
 
@@ -718,6 +700,35 @@ def create(self, bucket_name, is_data_warehouse=False):
         self._apply_tls_restrictions(s3_client, bucket_name)
         return bucket
 
+    def apply_lifecycle_config(self, bucket_name, s3_client=None):
+        if not s3_client:
+            s3_client = self.boto3_session.client("s3")
+        lifecycle_id = f"{bucket_name}_lifecycle_configuration"
+
+        try:
+            s3_client.put_bucket_lifecycle_configuration(
+                Bucket=bucket_name,
+                LifecycleConfiguration={
+                    "Rules": [
+                        {
+                            "ID": lifecycle_id,
+                            "Status": "Enabled",
+                            "Prefix": "",
+                            "Transitions": [
+                                {
+                                    "Days": 0,
+                                    "StorageClass": "INTELLIGENT_TIERING",
+                                },
+                            ],
+                        },
+                    ]
+                },
+            )
+        except s3_client.exceptions.NoSuchBucket:
+            log.warning(
+                f"Skipping creating lifecycle configuration for {bucket_name}: Does not exist"
+            )
+
     def _apply_tls_restrictions(self, client, bucket_name):
         """it assumes that this is a new bucket with no policies & creates it"""
         tls_statement = deepcopy(BUCKET_TLS_STATEMENT)

controlpanel/api/cluster.py

@@ -741,6 +741,12 @@ def create(self, owner=AWSRoleCategory.user):
         )
         return self.aws_bucket_service.create(self.bucket.name, self.bucket.is_data_warehouse)
 
+    def apply_lifecycle_config(self, owner=AWSRoleCategory.user):
+        self.aws_bucket_service.assume_role_name = self.get_assume_role(
+            self.aws_service_class, aws_role_category=owner
+        )
+        return self.aws_bucket_service.apply_lifecycle_config(self.bucket.name)
+
     def mark_for_archival(self):
         self.aws_bucket_service.assume_role_name = self.get_assume_role(
             self.aws_service_class, aws_role_category=self._get_assume_role_category()

controlpanel/frontend/jinja2/datasource-list.html

@@ -19,6 +19,18 @@
 {% block content %}
 <h1 class="govuk-heading-xl">{{ page_title }}</h1>
 
+{% if all_datasources and request.user.has_perm('api.is_superuser') %}
+<h2 class="govuk-heading-m">Update all bucket lifecycle configurations</h2>
+
+<form action="{{ url("update-lifecycle") }}" method="post">
+  {{ csrf_input }}
+  <button class="govuk-button js-confirm"
+          data-confirm-message="Are you sure you want to update all bucket lifecycle configurations?">
+          Update configurations
+  </button>
+</form>
+{% endif %}
+
 {% if request.user.has_perm('api.list_s3bucket') %}
   {{ datasource_list(buckets, datasource_type|default(""), request.user) }}
 

controlpanel/frontend/urls.py

@@ -10,6 +10,11 @@
     path("oidc/entraid/auth/", views.EntraIdAuthView.as_view(), name="entraid-auth"),
     path("oidc/logout/", views.LogoutView.as_view(), name="oidc_logout"),
     path("datasources/", views.AdminBucketList.as_view(), name="list-all-datasources"),
+    path(
+        "datasources/update-lifecycle-config",
+        views.UpdateDatasourceLifecycleConfig.as_view(),
+        name="update-lifecycle",
+    ),
     path("datasources/<int:pk>/", views.BucketDetail.as_view(), name="manage-datasource"),
     path(
         "datasources/<int:pk>/access/",

controlpanel/frontend/views/__init__.py

@@ -54,6 +54,7 @@
     RevokeAccess,
     RevokeIAMManagedPolicyAccess,
     UpdateAccessLevel,
+    UpdateDatasourceLifecycleConfig,
     UpdateIAMManagedPolicyAccessLevel,
     WebappBucketList,
 )

controlpanel/frontend/views/datasource.py

@@ -9,7 +9,7 @@
 from django.http import HttpResponseRedirect
 from django.shortcuts import get_object_or_404
 from django.urls import reverse_lazy
-from django.views.generic.base import ContextMixin
+from django.views.generic.base import ContextMixin, View
 from django.views.generic.detail import DetailView
 from django.views.generic.edit import CreateView, DeleteView, FormMixin, UpdateView
 from django.views.generic.list import ListView
@@ -402,3 +402,23 @@ def values(self, form):
             "paths": form.cleaned_data["paths"],
             "policy_id": form.cleaned_data["policy_id"],
         }
+
+
+class UpdateDatasourceLifecycleConfig(
+    OIDCLoginRequiredMixin,
+    PermissionRequiredMixin,
+    View,
+):
+    permission_required = "api.is_superuser"
+
+    def get_success_url(self):
+        return reverse_lazy("list-all-datasources")
+
+    def post(self, *args, **kwargs):
+        buckets = S3Bucket.objects.all()
+
+        for bucket in buckets:
+            bucket.cluster.apply_lifecycle_config(bucket.name)
+
+        messages.success(self.request, "Successfully updated bucket lifecycle configurations")
+        return HttpResponseRedirect(self.get_success_url())

tests/api/cluster/test_bucket.py

@@ -61,6 +61,12 @@ def test_mark_for_archival(aws_tag_bucket, bucket):
     aws_tag_bucket.assert_called_with(bucket.name, {"to-archive": "true"})
 
 
+def test_apply_lifecycle_config(bucket):
+    with patch("controlpanel.api.aws.AWSBucket.apply_lifecycle_config") as apply_lifecycle_config:
+        cluster.S3Bucket(bucket).apply_lifecycle_config(bucket.name)
+        apply_lifecycle_config.assert_called_with(bucket.name)
+
+
 def test_aws_folder_exists(bucket):
     with patch("controlpanel.api.aws.AWSFolder.exists") as mock_exists:
         mock_exists.return_value = False

tests/api/test_aws.py

@@ -246,8 +246,8 @@ def test_create_bucket(logs_bucket, s3):
     rule = versioning.rules[0]
     assert rule["ID"].endswith("_lifecycle_configuration")
     assert rule["Status"] == "Enabled"
-    assert rule["NoncurrentVersionTransitions"][0]["NoncurrentDays"] == 30
-    assert rule["NoncurrentVersionTransitions"][0]["StorageClass"] == "GLACIER"
+    assert rule["Transitions"][0]["Days"] == 0
+    assert rule["Transitions"][0]["StorageClass"] == "INTELLIGENT_TIERING"
 
     # Check logging
     assert (