feat: Replace vfolder's status_history's type dict with list

changes/3205.feature.md

@@ -0,0 +1 @@
+Change the type of `vfolders.status_history` from a mapping of status and timestamps to a list of log entries containing status and timestamps, to preserve the log entries.

src/ai/backend/manager/api/schema.graphql

@@ -890,6 +890,9 @@
   cur_size: BigInt
   cloneable: Boolean
   status: String
+
+  """Added in 24.12.0."""
+  status_history: JSONString
 }
 
 type ComputeSession implements Item {

src/ai/backend/manager/models/alembic/versions/786be66ef4e5_replace_vfolders_status_history_s_type_.py

@@ -0,0 +1,66 @@
+"""Replace vfolders status_history's type map with list
+Revision ID: 786be66ef4e5
+Revises: 8c8e90aebacd
+Create Date: 2024-05-07 05:10:23.799723
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "786be66ef4e5"
+down_revision = "8c8e90aebacd"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute(
+        """
+        WITH data AS (
+            SELECT id,
+                (jsonb_each(status_history)).key AS status,
+                (jsonb_each(status_history)).value AS timestamp
+            FROM vfolders
+        )
+        UPDATE vfolders
+        SET status_history = (
+            SELECT jsonb_agg(
+                jsonb_build_object('status', status, 'timestamp', timestamp)
+                ORDER BY timestamp
+            )
+            FROM data
+            WHERE data.id = vfolders.id
+        );
+    """
+    )
+
+    op.execute("UPDATE vfolders SET status_history = '[]'::jsonb WHERE status_history IS NULL;")
+    op.alter_column(
+        "vfolders",
+        "status_history",
+        nullable=False,
+        default=[],
+    )
+
+
+def downgrade():
+    op.execute(
+        """
+        WITH data AS (
+            SELECT id,
+                jsonb_object_agg(
+                    elem->>'status', elem->>'timestamp'
+                ) AS new_status_history
+            FROM vfolders,
+            jsonb_array_elements(status_history) AS elem
+            GROUP BY id
+        )
+        UPDATE vfolders
+        SET status_history = data.new_status_history
+        FROM data
+        WHERE data.id = vfolders.id;
+    """
+    )
+
+    op.alter_column("vfolders", "status_history", nullable=True, default=None)
+    op.execute("UPDATE vfolders SET status_history = NULL WHERE status_history = '[]'::jsonb;")

src/ai/backend/manager/models/vfolder.py

@@ -100,7 +100,12 @@
 from .storage import PermissionContext as StorageHostPermissionContext
 from .storage import PermissionContextBuilder as StorageHostPermissionContextBuilder
 from .user import UserRole, UserRow
-from .utils import ExtendedAsyncSAEngine, execute_with_retry, execute_with_txn_retry, sql_json_merge
+from .utils import (
+    ExtendedAsyncSAEngine,
+    execute_with_retry,
+    execute_with_txn_retry,
+    sql_append_dict_to_list,
+)
 
 if TYPE_CHECKING:
     from ..api.context import BackgroundTaskManager
@@ -378,14 +383,14 @@ class VFolderCloneInfo(NamedTuple):
         nullable=False,
         index=True,
     ),
-    # status_history records the most recent status changes for each status
+    # status_history records the status changes of the vfolder.
     # e.g)
-    # {
-    #   "ready": "2022-10-22T10:22:30",
-    #   "delete-pending": "2022-10-22T11:40:30",
-    #   "delete-ongoing": "2022-10-25T10:22:30"
-    # }
-    sa.Column("status_history", pgsql.JSONB(), nullable=True, default=sa.null()),
+    # [
+    #   {"status": "ready", "timestamp": "2022-10-22T10:22:30"},
+    #   {"status": "delete-pending", "timestamp": "2022-10-22T11:40:30"},
+    #   {"status": "delete-ongoing", "timestamp": "2022-10-25T10:22:30"}
+    # ]
+    sa.Column("status_history", pgsql.JSONB(), nullable=False, default=[]),
     sa.Column("status_changed", sa.DateTime(timezone=True), nullable=True, index=True),
 )
 
@@ -1063,12 +1068,9 @@ async def _update() -> None:
             values = {
                 "status": update_status,
                 "status_changed": now,
-                "status_history": sql_json_merge(
+                "status_history": sql_append_dict_to_list(
                     VFolderRow.status_history,
-                    (),
-                    {
-                        update_status.name: now.isoformat(),
-                    },
+                    {"status": update_status.name, "timestamp": now.isoformat()},
                 ),
             }
             if update_status == VFolderOperationStatus.DELETE_ONGOING:
@@ -1396,6 +1398,8 @@ class Meta:
     cloneable = graphene.Boolean()
     status = graphene.String()
 
+    status_history = graphene.JSONString(description="Added in 24.12.0.")
+
     @classmethod
     def from_row(cls, ctx: GraphQueryContext, row: Row | VFolderRow) -> Optional[VirtualFolder]:
         if row is None:
@@ -1430,6 +1434,7 @@ def _get_field(name: str) -> Any:
             cloneable=row["cloneable"],
             status=row["status"],
             cur_size=row["cur_size"],
+            status_history=row["status_history"],
         )
 
     @classmethod
@@ -1455,6 +1460,7 @@ def from_orm_row(cls, row: VFolderRow) -> VirtualFolder:
             cloneable=row.cloneable,
             status=row.status,
             cur_size=row.cur_size,
+            status_history=row.status_history,
         )
 
     async def resolve_num_files(self, info: graphene.ResolveInfo) -> int:
@@ -1517,6 +1523,7 @@ async def resolve_num_files(self, info: graphene.ResolveInfo) -> int:
         "cloneable": ("vfolders_cloneable", None),
         "status": ("vfolders_status", None),
         "cur_size": ("vfolders_cur_size", None),
+        "status_history": ("vfolders_status_history", None),
     }
 
     @classmethod