From e5807ba696d5bbec2d41ed9c5e35de9ca589bf71 Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Wed, 8 Nov 2023 15:51:39 -0500 Subject: [PATCH] Implement extra and gifted execution minutes - Add new quotas - Add usage tracking and quota update history to Organization - Add tracking of available seconds for extra and gifted minutes to Organization - Modify routine to check if quota is reached to return False if extra or gifted time remains (with flag to disable checking these for internal purposes) - Update inc_org_time_stats to update used and remaining extra and gifted time on crawl completion - Update frontend execution time meter and usage history --- backend/btrixcloud/models.py | 24 +++ backend/btrixcloud/orgs.py | 132 ++++++++++++++- frontend/src/components/orgs-list.ts | 6 + frontend/src/pages/org/dashboard.ts | 229 +++++++++++++++++++++------ frontend/src/types/org.ts | 10 ++ 5 files changed, 348 insertions(+), 53 deletions(-) diff --git a/backend/btrixcloud/models.py b/backend/btrixcloud/models.py index db0a29cbd3..4b14e95777 100644 --- a/backend/btrixcloud/models.py +++ b/backend/btrixcloud/models.py @@ -816,6 +816,16 @@ class OrgQuotas(BaseModel): maxPagesPerCrawl: Optional[int] = 0 storageQuota: Optional[int] = 0 maxExecMinutesPerMonth: Optional[int] = 0 + extraExecMinutes: Optional[int] = 0 + giftedExecMinutes: Optional[int] = 0 + + +# ============================================================================ +class OrgQuotaUpdate(BaseModel): + """Organization quota update (to track changes over time)""" + + modified: datetime + update: OrgQuotas # ============================================================================ @@ -851,10 +861,17 @@ class OrgOut(BaseMongoModel): webhookUrls: Optional[OrgWebhookUrls] = OrgWebhookUrls() quotas: Optional[OrgQuotas] = OrgQuotas() + quotaUpdates: Optional[List[OrgQuotaUpdate]] = [] storageQuotaReached: Optional[bool] execMinutesQuotaReached: Optional[bool] + extraExecSeconds: Dict[str, int] = {} + giftedExecSeconds: Dict[str, int] = {} + + extraExecSecondsAvailable: int = 0 + giftedExecSecondsAvailable: int = 0 + # ============================================================================ class Organization(BaseMongoModel): @@ -875,6 +892,8 @@ class Organization(BaseMongoModel): usage: Dict[str, int] = {} crawlExecSeconds: Dict[str, int] = {} + extraExecSeconds: Dict[str, int] = {} + giftedExecSeconds: Dict[str, int] = {} bytesStored: int = 0 bytesStoredCrawls: int = 0 @@ -884,11 +903,16 @@ class Organization(BaseMongoModel): default: bool = False quotas: Optional[OrgQuotas] = OrgQuotas() + quotaUpdates: Optional[List[OrgQuotaUpdate]] = [] webhookUrls: Optional[OrgWebhookUrls] = OrgWebhookUrls() origin: Optional[AnyHttpUrl] = None + # TODO: Add migration so that we are sure this is not None + extraExecSecondsAvailable: int = 0 + giftedExecSecondsAvailable: int = 0 + def is_owner(self, user): """Check if user is owner""" return self._is_auth(user, UserRole.OWNER) diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py index 21da60708a..8067cd14a6 100644 --- a/backend/btrixcloud/orgs.py +++ b/backend/btrixcloud/orgs.py @@ -21,6 +21,7 @@ Organization, StorageRef, OrgQuotas, + OrgQuotaUpdate, OrgMetrics, OrgWebhookUrls, CreateOrg, @@ -268,17 +269,46 @@ async def update_custom_storages(self, org: Organization): async def update_quotas(self, org: Organization, quotas: OrgQuotas): """update organization quotas""" - return await self.orgs.find_one_and_update( + + previous_extra_mins = org.quotas.extraExecMinutes or 0 + previous_gifted_mins = org.quotas.giftedExecMinutes or 0 + + update = quotas.dict( + exclude_unset=True, exclude_defaults=True, exclude_none=True + ) + + quota_updates = [] + for prev_update in org.quotaUpdates: + quota_updates.append(prev_update.dict()) + quota_updates.append( + OrgQuotaUpdate(update=update, modified=datetime.now()).dict() + ) + + await self.orgs.find_one_and_update( {"_id": org.id}, { "$set": { - "quotas": quotas.dict( - exclude_unset=True, exclude_defaults=True, exclude_none=True - ) + "quotas": update, + "quotaUpdates": quota_updates, } }, ) + # Inc org available fields for extra/gifted execution time as needed + if quotas.extraExecMinutes: + extra_secs_diff = (quotas.extraExecMinutes - previous_extra_mins) * 60 + await self.orgs.find_one_and_update( + {"_id": org.id}, + {"$inc": {"extraExecSecondsAvailable": extra_secs_diff}}, + ) + + if quotas.giftedExecMinutes: + gifted_secs_diff = (quotas.giftedExecMinutes - previous_gifted_mins) * 60 + await self.orgs.find_one_and_update( + {"_id": org.id}, + {"$inc": {"giftedExecSecondsAvailable": gifted_secs_diff}}, + ) + async def update_event_webhook_urls(self, org: Organization, urls: OrgWebhookUrls): """Update organization event webhook URLs""" return await self.orgs.find_one_and_update( @@ -377,12 +407,23 @@ async def get_this_month_crawl_exec_seconds(self, oid: UUID) -> int: except KeyError: return 0 - async def exec_mins_quota_reached(self, oid: UUID) -> bool: - """Return bools for if execution minutes quota""" + async def exec_mins_quota_reached( + self, oid: UUID, include_extra: bool = True + ) -> bool: + """Return bool for if execution minutes quota is reached""" quota = await self.get_org_exec_mins_monthly_quota(oid) quota_reached = False + if include_extra: + gifted_seconds = await self.get_gifted_exec_secs_available(oid) + if gifted_seconds: + return False + + extra_seconds = await self.get_extra_exec_secs_available(oid) + if extra_seconds: + return False + if quota: monthly_exec_seconds = await self.get_this_month_crawl_exec_seconds(oid) monthly_exec_minutes = math.floor(monthly_exec_seconds / 60) @@ -390,8 +431,6 @@ async def exec_mins_quota_reached(self, oid: UUID) -> bool: if monthly_exec_minutes >= quota: quota_reached = True - # add additional quotas here - return quota_reached async def get_org_storage_quota(self, oid: UUID) -> int: @@ -410,6 +449,22 @@ async def get_org_exec_mins_monthly_quota(self, oid: UUID) -> int: return org.quotas.maxExecMinutesPerMonth return 0 + async def get_extra_exec_secs_available(self, oid: UUID) -> int: + """return extra billable rollover seconds available, if any""" + org = await self.orgs.find_one({"_id": oid}) + if org: + org = Organization.from_dict(org) + return org.extraExecSecondsAvailable + return 0 + + async def get_gifted_exec_secs_available(self, oid: UUID) -> int: + """return gifted rollover seconds available, if any""" + org = await self.orgs.find_one({"_id": oid}) + if org: + org = Organization.from_dict(org) + return org.giftedExecSecondsAvailable + return 0 + async def set_origin(self, org: Organization, request: Request): """Get origin from request and store in db for use in event webhooks""" headers = request.headers @@ -434,6 +489,67 @@ async def inc_org_time_stats(self, oid, duration, is_exec_time=False): {"_id": oid}, {"$inc": {f"{key}.{yymm}": duration}} ) + org = await self.get_org_by_id(oid) + monthly_base_quota_passed = await self.exec_mins_quota_reached( + oid, include_extra=False + ) + + # If we're still within our monthly base quota, nothing to do + if not monthly_base_quota_passed: + return + + # If we've surpassed monthly base quota, use gifted and extra exec minutes + # in that order if available, track their usage per month, and recalculate + # extraExecSecondsAvailable and giftedExecSecondsAvailable as needed + monthly_quota_mins = await self.get_org_exec_mins_monthly_quota(oid) + monthly_quota_secs = monthly_quota_mins * 60 + + current_monthly_exec_secs = await self.get_this_month_crawl_exec_seconds(oid) + secs_over_quota = current_monthly_exec_secs - monthly_quota_secs + + gifted_secs_available = org.giftedExecSecondsAvailable or 0 + if gifted_secs_available: + if secs_over_quota <= gifted_secs_available: + return await self.orgs.find_one_and_update( + {"_id": oid}, + { + "$inc": { + f"giftedExecSeconds.{yymm}": secs_over_quota, + "giftedExecSecondsAvailable": -secs_over_quota, + } + }, + ) + + # If seconds over quota is higher than gifted minutes, use all of the + # gifted minutes and then move on to extra minutes + await self.orgs.find_one_and_update( + {"_id": oid}, + { + "$inc": { + f"giftedExecSeconds.{yymm}": gifted_secs_available, + "giftedExecSecondsAvailable": -gifted_secs_available, + } + }, + ) + + secs_still_over_quota = current_monthly_exec_secs - ( + monthly_quota_secs + gifted_secs_available + ) + extra_secs_available = org.extraExecSecondsAvailable + if extra_secs_available: + return await self.orgs.find_one_and_update( + {"_id": oid}, + { + "$inc": { + f"extraExecSeconds.{yymm}": secs_still_over_quota, + # Don't let extra seconds available fall below 0 + "extraExecSecondsAvailable": -min( + secs_still_over_quota, extra_secs_available + ), + } + }, + ) + async def get_max_concurrent_crawls(self, oid): """return max allowed concurrent crawls, if any""" org = await self.orgs.find_one({"_id": oid}) diff --git a/frontend/src/components/orgs-list.ts b/frontend/src/components/orgs-list.ts index 1c3afa87a0..6473982aca 100644 --- a/frontend/src/components/orgs-list.ts +++ b/frontend/src/components/orgs-list.ts @@ -64,6 +64,12 @@ export class OrgsList extends LiteElement { case "maxExecMinutesPerMonth": label = msg("Max Execution Minutes Per Month"); break; + case "extraExecMinutes": + label = msg("Extra Execution Minutes"); + break; + case "giftedExecMinutes": + label = msg("Gifted Execution Minutes"); + break; default: label = msg("Unlabeled"); } diff --git a/frontend/src/pages/org/dashboard.ts b/frontend/src/pages/org/dashboard.ts index 440c80a364..0d1354d5f2 100644 --- a/frontend/src/pages/org/dashboard.ts +++ b/frontend/src/pages/org/dashboard.ts @@ -59,7 +59,10 @@ export class Dashboard extends LiteElement { } } - private humanizeExecutionSeconds = (seconds: number) => { + private formatExecutionSeconds = ( + seconds: number, + humanize: boolean = true + ) => { const minutes = Math.ceil(seconds / 60); const locale = getLocale(); @@ -78,9 +81,12 @@ export class Dashboard extends LiteElement { }); return html` - ${compactFormatter.format(minutes)} - (${humanizeMilliseconds(seconds * 1000)})`; + ${compactFormatter.format(minutes)} + + ${when( + humanize, + () => html` (${humanizeMilliseconds(seconds * 1000)})` + )} `; }; render() { @@ -365,12 +371,28 @@ export class Dashboard extends LiteElement { private renderCrawlingMeter(metrics: Metrics) { let quotaSeconds = 0; + if (this.org!.quotas && this.org!.quotas.maxExecMinutesPerMonth) { quotaSeconds = this.org!.quotas.maxExecMinutesPerMonth * 60; } - let usageSeconds = 0; + let quotaSecondsAllTypes = quotaSeconds; + + let quotaSecondsExtra = 0; + if (this.org!.extraExecSecondsAvailable) { + quotaSecondsExtra = this.org!.extraExecSecondsAvailable; + quotaSecondsAllTypes += this.org!.extraExecSecondsAvailable; + } + + let quotaSecondsGifted = 0; + if (this.org!.giftedExecSecondsAvailable) { + quotaSecondsGifted = this.org!.giftedExecSecondsAvailable; + quotaSecondsAllTypes += this.org!.giftedExecSecondsAvailable; + } + const now = new Date(); + + let usageSeconds = 0; if (this.org!.crawlExecSeconds) { const actualUsage = this.org!.crawlExecSeconds[ @@ -381,11 +403,43 @@ export class Dashboard extends LiteElement { } } - const hasQuota = Boolean(quotaSeconds); - const isReached = hasQuota && usageSeconds >= quotaSeconds; + let usageSecondsAllTypes = usageSeconds; + + let usageSecondsExtra = 0; + if (this.org!.extraExecSeconds) { + usageSecondsExtra = + this.org!.extraExecSeconds[ + `${now.getFullYear()}-${now.getUTCMonth() + 1}` + ]; + } + if (usageSecondsExtra) { + usageSecondsAllTypes += usageSecondsExtra; + // Quota for extra = this month's usage + remaining available + quotaSecondsAllTypes += usageSecondsExtra; + quotaSecondsExtra += usageSecondsExtra; + } + + let usageSecondsGifted = 0; + if (this.org!.giftedExecSeconds) { + usageSecondsGifted = + this.org!.giftedExecSeconds[ + `${now.getFullYear()}-${now.getUTCMonth() + 1}` + ]; + } + if (usageSecondsGifted) { + usageSecondsAllTypes += usageSecondsGifted; + // Quota for gifted = this month's usage + remaining available + quotaSecondsAllTypes += usageSecondsGifted; + quotaSecondsGifted += usageSecondsGifted; + } + + const hasQuota = Boolean(quotaSecondsAllTypes); + const isReached = hasQuota && usageSecondsAllTypes >= quotaSecondsAllTypes; + const maxTotalTime = quotaSeconds + quotaSecondsExtra + quotaSecondsGifted; if (isReached) { - usageSeconds = quotaSeconds; + usageSecondsAllTypes = maxTotalTime; + quotaSecondsAllTypes = maxTotalTime; } const renderBar = ( @@ -395,14 +449,13 @@ export class Dashboard extends LiteElement { color: string ) => html`
${label}
- ${humanizeMilliseconds(value * 1000)} | - ${this.renderPercentage(value / quotaSeconds)} + ${this.formatExecutionSeconds(value, false)}
@@ -417,15 +470,15 @@ export class Dashboard extends LiteElement { class="text-danger" name="exclamation-triangle" > - ${msg("Monthly Execution Minutes Quota Reached")} + ${msg("Execution Minutes Quota Reached")} `, () => hasQuota ? html` - ${this.humanizeExecutionSeconds( - quotaSeconds - usageSeconds + ${this.formatExecutionSeconds( + quotaSecondsAllTypes - usageSecondsAllTypes )} ${msg("Available")} @@ -438,39 +491,61 @@ export class Dashboard extends LiteElement { () => html`
${when(usageSeconds, () => renderBar( - usageSeconds, - msg("Monthly Execution Time Used"), - isReached ? "warning" : this.colors.runningTime + usageSeconds > quotaSeconds ? quotaSeconds : usageSeconds, + msg("Monthly Execution Time"), + "green-400" + ) + )} + ${when(usageSecondsGifted, () => + renderBar( + usageSecondsGifted > quotaSecondsGifted + ? quotaSecondsGifted + : usageSecondsGifted, + msg("Gifted Execution Time"), + "blue-400" + ) + )} + ${when(usageSecondsExtra, () => + renderBar( + usageSecondsExtra > quotaSecondsExtra + ? quotaSecondsExtra + : usageSecondsExtra, + msg("Extra Execution Time"), + "red-400" + ) + )} + ${when(quotaSeconds && usageSeconds < quotaSeconds, () => + renderBar( + quotaSeconds - usageSeconds, + msg("Monthly Execution Time Available"), + "green-100" + ) + )} + ${when(this.org!.giftedExecSecondsAvailable, () => + renderBar( + this.org!.giftedExecSecondsAvailable, + msg("Gifted Execution Time Available"), + "blue-100" + ) + )} + ${when(this.org!.extraExecSecondsAvailable, () => + renderBar( + this.org!.extraExecSecondsAvailable, + msg("Extra Execution Time Available"), + "red-100" ) )} -
- -
-
${msg("Monthly Execution Time Available")}
-
- ${this.humanizeExecutionSeconds( - quotaSeconds - usageSeconds - )} - | - ${this.renderPercentage( - (quotaSeconds - usageSeconds) / quotaSeconds - )} -
-
-
- -
- ${this.humanizeExecutionSeconds(usageSeconds)} + ${this.formatExecutionSeconds(usageSecondsAllTypes, false)} - ${this.humanizeExecutionSeconds(quotaSeconds)} + ${this.formatExecutionSeconds(quotaSecondsAllTypes, false)}
@@ -553,7 +628,16 @@ export class Dashboard extends LiteElement { readonly usageTableCols = [ msg("Month"), html` - ${msg("Execution Time")} + ${msg("Elapsed Time")} + +
+ ${msg("Total time elapsed between when crawls started and ended")} +
+ + + `, + html` + ${msg("Total Execution Time")}
${msg("Total running time of all crawler instances")} @@ -562,10 +646,28 @@ export class Dashboard extends LiteElement { `, html` - ${msg("Elapsed Time")} + ${msg("Execution: Monthly")}
- ${msg("Total time elapsed between when crawls started and ended")} + ${msg("Monthly execution time used on crawls this month")} +
+ + + `, + html` + ${msg("Execution: Extra")} + +
+ ${msg("Billable rollover execution time used on crawls this month")} +
+ + + `, + html` + ${msg("Execution: Gifted")} + +
+ ${msg("Gifted execution time used on crawls this month")}
@@ -578,7 +680,35 @@ export class Dashboard extends LiteElement { // Sort latest .reverse() .map(([mY, crawlTime]) => { - const value = this.org!.crawlExecSeconds?.[mY]; + let value = this.org!.crawlExecSeconds?.[mY] || 0; + let maxMonthlySeconds = 0; + if (this.org!.quotas.maxExecMinutesPerMonth) { + maxMonthlySeconds = this.org!.quotas.maxExecMinutesPerMonth * 60; + } + if (value > maxMonthlySeconds) { + value = maxMonthlySeconds; + } + + let extraSecondsUsed = this.org!.extraExecSeconds?.[mY] || 0; + let maxExtraSeconds = 0; + if (this.org!.quotas.extraExecMinutes) { + maxExtraSeconds = this.org!.quotas.extraExecMinutes * 60; + } + if (extraSecondsUsed > maxExtraSeconds) { + extraSecondsUsed = maxExtraSeconds; + } + + let giftedSecondsUsed = this.org!.giftedExecSeconds?.[mY] || 0; + let maxGiftedSeconds = 0; + if (this.org!.quotas.giftedExecMinutes) { + maxGiftedSeconds = this.org!.quotas.giftedExecMinutes * 60; + } + if (giftedSecondsUsed > maxGiftedSeconds) { + giftedSecondsUsed = maxGiftedSeconds; + } + + const totalTimeSeconds = value + extraSecondsUsed + giftedSecondsUsed; + return [ html` `, - value ? this.humanizeExecutionSeconds(value) : "--", - humanizeMilliseconds(crawlTime * 1000 || 0), + humanizeMilliseconds((crawlTime || 0) * 1000), + totalTimeSeconds + ? humanizeMilliseconds(totalTimeSeconds * 1000) + : "--", + value ? humanizeMilliseconds(value * 1000) : "--", + extraSecondsUsed + ? humanizeMilliseconds(extraSecondsUsed * 1000) + : "--", + giftedSecondsUsed + ? humanizeMilliseconds(giftedSecondsUsed * 1000) + : "--", ]; }); return html` diff --git a/frontend/src/types/org.ts b/frontend/src/types/org.ts index 586ffb23cc..e01ef5636f 100644 --- a/frontend/src/types/org.ts +++ b/frontend/src/types/org.ts @@ -22,6 +22,16 @@ export type OrgData = { // Keyed by {4-digit year}-{2-digit month} [key: string]: number; } | null; + extraExecSeconds: { + // Keyed by {4-digit year}-{2-digit month} + [key: string]: number; + } | null; + giftedExecSeconds: { + // Keyed by {4-digit year}-{2-digit month} + [key: string]: number; + } | null; + extraExecSecondsAvailable: number; + giftedExecSecondsAvailable: number; storageQuotaReached?: boolean; execMinutesQuotaReached?: boolean; users?: {