From bc9975d34837a02d89c80ebe13e12e0e26e4a6f8 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 23 Jul 2024 12:43:30 +0200 Subject: [PATCH 1/4] Use the correct database for the compact action. --- packages/service-core/src/entry/commands/compact-action.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/service-core/src/entry/commands/compact-action.ts b/packages/service-core/src/entry/commands/compact-action.ts index 51871fb..7c8d45b 100644 --- a/packages/service-core/src/entry/commands/compact-action.ts +++ b/packages/service-core/src/entry/commands/compact-action.ts @@ -34,7 +34,7 @@ export function registerCompactAction(program: Command) { const client = mongo.createMongoClient(storage); await client.connect(); try { - const psdb = new PowerSyncMongo(client); + const psdb = new PowerSyncMongo(client, { database: storage.database }); const bucketStorage = new MongoBucketStorage(psdb, { slot_name_prefix: config.slot_name_prefix }); const active = await bucketStorage.getActiveSyncRules(); if (active == null) { From 4aaa2a3ff4f1f06a615b7d561bd9f6a89208482c Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 23 Jul 2024 12:44:14 +0200 Subject: [PATCH 2/4] Expose API to compact specific buckets. --- .../service-core/src/storage/BucketStorage.ts | 7 +++++++ .../src/storage/mongo/MongoCompactor.ts | 19 +++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/packages/service-core/src/storage/BucketStorage.ts b/packages/service-core/src/storage/BucketStorage.ts index 9665c80..24d46e2 100644 --- a/packages/service-core/src/storage/BucketStorage.ts +++ b/packages/service-core/src/storage/BucketStorage.ts @@ -426,4 +426,11 @@ export interface CompactOptions { * not be compacted, to avoid invalidating checkpoints in use. */ maxOpId?: bigint; + + /** + * If specified, compact only the specific buckets. + * + * If not specified, compacts all buckets. + */ + compactBuckets?: string[]; } diff --git a/packages/service-core/src/storage/mongo/MongoCompactor.ts b/packages/service-core/src/storage/mongo/MongoCompactor.ts index f5e8d39..3c52936 100644 --- a/packages/service-core/src/storage/mongo/MongoCompactor.ts +++ b/packages/service-core/src/storage/mongo/MongoCompactor.ts @@ -55,6 +55,7 @@ export class MongoCompactor { private moveBatchQueryLimit: number; private clearBatchLimit: number; private maxOpId: bigint | undefined; + private buckets: string[] | undefined; constructor(private db: PowerSyncMongo, private group_id: number, options?: MongoCompactOptions) { this.idLimitBytes = (options?.memoryLimitMB ?? DEFAULT_MEMORY_LIMIT_MB) * 1024 * 1024; @@ -62,6 +63,7 @@ export class MongoCompactor { this.moveBatchQueryLimit = options?.moveBatchQueryLimit ?? DEFAULT_MOVE_BATCH_QUERY_LIMIT; this.clearBatchLimit = options?.clearBatchLimit ?? DEFAULT_CLEAR_BATCH_LIMIT; this.maxOpId = options?.maxOpId; + this.buckets = options?.compactBuckets; } /** @@ -70,6 +72,19 @@ export class MongoCompactor { * See /docs/compacting-operations.md for details. */ async compact() { + if (this.buckets) { + for (let bucket of this.buckets) { + // We can make this more efficient later on by iterating + // through the buckets in a single query. + // That makes batching more tricky, so we leave for later. + await this.compactInternal(bucket); + } + } else { + await this.compactInternal(undefined); + } + } + + async compactInternal(bucket: string | undefined) { const idLimitBytes = this.idLimitBytes; let currentState: CurrentBucketState | null = null; @@ -77,14 +92,14 @@ export class MongoCompactor { // Constant lower bound const lowerBound: BucketDataKey = { g: this.group_id, - b: new MinKey() as any, + b: bucket ?? (new MinKey() as any), o: new MinKey() as any }; // Upper bound is adjusted for each batch let upperBound: BucketDataKey = { g: this.group_id, - b: new MaxKey() as any, + b: bucket ?? (new MaxKey() as any), o: new MaxKey() as any }; From bfe0e6472eb0eec201a28217b7f86b5603937fc6 Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 23 Jul 2024 12:46:15 +0200 Subject: [PATCH 3/4] Add changeset. --- .changeset/purple-moles-shave.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/purple-moles-shave.md diff --git a/.changeset/purple-moles-shave.md b/.changeset/purple-moles-shave.md new file mode 100644 index 0000000..b07257c --- /dev/null +++ b/.changeset/purple-moles-shave.md @@ -0,0 +1,6 @@ +--- +'@powersync/service-core': patch +'@powersync/service-image': patch +--- + +Fix compact command to use the correct database From 00da76872ae524d003c1c61f12f8cf97a11507db Mon Sep 17 00:00:00 2001 From: Ralf Kistner Date: Tue, 23 Jul 2024 13:01:55 +0200 Subject: [PATCH 4/4] Use higher-level createPowerSyncMongo() instead. --- packages/service-core/src/entry/commands/compact-action.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/service-core/src/entry/commands/compact-action.ts b/packages/service-core/src/entry/commands/compact-action.ts index 7c8d45b..2fff227 100644 --- a/packages/service-core/src/entry/commands/compact-action.ts +++ b/packages/service-core/src/entry/commands/compact-action.ts @@ -2,8 +2,7 @@ import { Command } from 'commander'; import { logger } from '@powersync/lib-services-framework'; import * as v8 from 'v8'; -import { mongo } from '../../db/db-index.js'; -import { MongoBucketStorage, PowerSyncMongo } from '../../storage/storage-index.js'; +import { createPowerSyncMongo, MongoBucketStorage } from '../../storage/storage-index.js'; import { loadConfig } from '../../util/config.js'; import { extractRunnerOptions, wrapConfigCommand } from './config-command.js'; @@ -31,10 +30,10 @@ export function registerCompactAction(program: Command) { const config = await loadConfig(runnerConfig); const { storage } = config; - const client = mongo.createMongoClient(storage); + const psdb = createPowerSyncMongo(storage); + const client = psdb.client; await client.connect(); try { - const psdb = new PowerSyncMongo(client, { database: storage.database }); const bucketStorage = new MongoBucketStorage(psdb, { slot_name_prefix: config.slot_name_prefix }); const active = await bucketStorage.getActiveSyncRules(); if (active == null) {