Add extra and gifted execution minutes (#1361)

Fixes #1358 

- Adds `extraExecMinutes` and `giftedExecMinutes` org quotas, which are
not reset monthly but are updateable amounts that carry across months
- Adds `quotaUpdate` field to `Organization` to track when quotas were
updated with timestamp
- Adds `extraExecMinutesAvailable` and `giftedExecMinutesAvailable`
fields to `Organization` to help with tracking available time left
(includes tested migration to initialize these to 0)
- Modifies org backend to track time across multiple categories, using
monthlyExecSeconds, then giftedExecSeconds, then extraExecSeconds.
All time is also written into crawlExecSeconds, which is now the monthly
total and also contains any overage time above the quotas
- Updates Dashboard crawling meter to include all types of execution
time if `extraExecMinutes` and/or `giftedExecMinutes` are set above 0
- Updates Dashboard Usage History table to include all types of
execution time (only displaying columns that have data)
- Adds backend nightly test to check handling of quotas and execution
time
- Includes migration to add new fields and copy crawlExecSeconds to
monthlyExecSeconds for previous months

Co-authored-by: emma <[email protected]>

backend/btrixcloud/db.py

@@ -16,7 +16,7 @@
 from .migrations import BaseMigration
 
 
-CURR_DB_VERSION = "0022"
+CURR_DB_VERSION = "0023"
 
 
 # ============================================================================

backend/btrixcloud/migrations/migration_0022_partial_complete.py

@@ -1,5 +1,5 @@
 """
-Migration 00233 -- Partial Complete
+Migration 0022 -- Partial Complete
 """
 from btrixcloud.migrations import BaseMigration
 

backend/btrixcloud/migrations/migration_0023_available_extra_exec_mins.py

@@ -0,0 +1,63 @@
+"""
+Migration 0023 -- Available extra/gifted minutes
+"""
+from btrixcloud.migrations import BaseMigration
+
+
+MIGRATION_VERSION = "0023"
+
+
+class Migration(BaseMigration):
+    """Migration class."""
+
+    def __init__(self, mdb, migration_version=MIGRATION_VERSION):
+        super().__init__(mdb, migration_version)
+
+    async def migrate_up(self):
+        """Perform migration up.
+
+        Add extraExecSecondsAvailable and giftedExecSecondsAvailable to org.
+        Initialize at 0 to avoid them being None.
+
+        Also add monthlyExecSeconds and copy previous crawlExecSeconds values
+        to it.
+        """
+        # pylint: disable=duplicate-code
+        mdb_orgs = self.mdb["organizations"]
+
+        query = {
+            "extraExecSecondsAvailable": None,
+            "giftedExecSecondsAvailable": None,
+        }
+        async for org in mdb_orgs.find(query):
+            oid = org["_id"]
+            try:
+                await mdb_orgs.find_one_and_update(
+                    {"_id": oid},
+                    {
+                        "$set": {
+                            "extraExecSecondsAvailable": 0,
+                            "giftedExecSecondsAvailable": 0,
+                        }
+                    },
+                )
+            # pylint: disable=broad-exception-caught
+            except Exception as err:
+                print(
+                    f"Error adding exec seconds available fields to org {oid}: {err}",
+                    flush=True,
+                )
+
+        async for org in mdb_orgs.find({"monthlyExecSeconds": None}):
+            oid = org["_id"]
+            try:
+                await mdb_orgs.update_one(
+                    {"_id": oid},
+                    [{"$set": {"monthlyExecSeconds": "$crawlExecSeconds"}}],
+                )
+            # pylint: disable=broad-exception-caught
+            except Exception as err:
+                print(
+                    f"Error copying crawlExecSeconds to monthlyExecSeconds for org {oid}: {err}",
+                    flush=True,
+                )

backend/btrixcloud/models.py

@@ -817,6 +817,16 @@ class OrgQuotas(BaseModel):
     maxPagesPerCrawl: Optional[int] = 0
     storageQuota: Optional[int] = 0
     maxExecMinutesPerMonth: Optional[int] = 0
+    extraExecMinutes: Optional[int] = 0
+    giftedExecMinutes: Optional[int] = 0
+
+
+# ============================================================================
+class OrgQuotaUpdate(BaseModel):
+    """Organization quota update (to track changes over time)"""
+
+    modified: datetime
+    update: OrgQuotas
 
 
 # ============================================================================
@@ -841,50 +851,63 @@ class OrgOut(BaseMongoModel):
     name: str
     slug: str
     users: Optional[Dict[str, Any]]
-    usage: Optional[Dict[str, int]]
-    crawlExecSeconds: Optional[Dict[str, int]]
+
     default: bool = False
     bytesStored: int
     bytesStoredCrawls: int
     bytesStoredUploads: int
     bytesStoredProfiles: int
     origin: Optional[AnyHttpUrl] = None
 
-    webhookUrls: Optional[OrgWebhookUrls] = OrgWebhookUrls()
-    quotas: Optional[OrgQuotas] = OrgQuotas()
-
     storageQuotaReached: Optional[bool]
     execMinutesQuotaReached: Optional[bool]
 
+    usage: Optional[Dict[str, int]]
+    crawlExecSeconds: Dict[str, int] = {}
+    monthlyExecSeconds: Dict[str, int] = {}
+    extraExecSeconds: Dict[str, int] = {}
+    giftedExecSeconds: Dict[str, int] = {}
+
+    extraExecSecondsAvailable: int = 0
+    giftedExecSecondsAvailable: int = 0
+
+    quotas: Optional[OrgQuotas] = OrgQuotas()
+    quotaUpdates: Optional[List[OrgQuotaUpdate]] = []
+
+    webhookUrls: Optional[OrgWebhookUrls] = OrgWebhookUrls()
+
 
 # ============================================================================
 class Organization(BaseMongoModel):
     """Organization Base Model"""
 
     id: UUID
-
     name: str
     slug: str
-
     users: Dict[str, UserRole]
 
-    storage: StorageRef
+    default: bool = False
 
+    storage: StorageRef
     storageReplicas: List[StorageRef] = []
-
     customStorages: Dict[str, S3Storage] = {}
 
-    usage: Dict[str, int] = {}
-    crawlExecSeconds: Dict[str, int] = {}
-
     bytesStored: int = 0
     bytesStoredCrawls: int = 0
     bytesStoredUploads: int = 0
     bytesStoredProfiles: int = 0
 
-    default: bool = False
+    usage: Dict[str, int] = {}
+    crawlExecSeconds: Dict[str, int] = {}
+    monthlyExecSeconds: Dict[str, int] = {}
+    extraExecSeconds: Dict[str, int] = {}
+    giftedExecSeconds: Dict[str, int] = {}
+
+    extraExecSecondsAvailable: int = 0
+    giftedExecSecondsAvailable: int = 0
 
     quotas: Optional[OrgQuotas] = OrgQuotas()
+    quotaUpdates: Optional[List[OrgQuotaUpdate]] = []
 
     webhookUrls: Optional[OrgWebhookUrls] = OrgWebhookUrls()