From baf1c4330eefabf3c0df1c0b5cf30a7579efcdc7 Mon Sep 17 00:00:00 2001 From: Karl Prieb Date: Wed, 18 Sep 2024 01:03:53 -0300 Subject: [PATCH 1/4] cache for saveNestedDataHash calls to prevent duplicate writes Add a cache for saveNestedDataHash calls to prevent duplicate writes. Add a metrics counter for the number of duplicate saveNestedDataHash calls. In an environment with ~830 calls to saveNestedDataHash per second after 7 minutes of operation(cache entry TTL), the amount of cached keys was ~349k, with a total of ~33MB of data stored in the cache. Total amount of duplicated calls was 1428. --- src/database/standalone-sqlite.ts | 26 ++++++++++++++++++++++++++ src/metrics.ts | 6 ++++++ 2 files changed, 32 insertions(+) diff --git a/src/database/standalone-sqlite.ts b/src/database/standalone-sqlite.ts index f1443476..aa77cb78 100644 --- a/src/database/standalone-sqlite.ts +++ b/src/database/standalone-sqlite.ts @@ -29,6 +29,7 @@ import * as R from 'ramda'; import sql from 'sql-bricks'; import * as winston from 'winston'; import CircuitBreaker from 'opossum'; +import NodeCache from 'node-cache'; // TODO enable eslint /* eslint-disable */ @@ -2432,6 +2433,8 @@ export class StandaloneSqliteDatabase Awaited> >; + private saveNestedDataHashCache: NodeCache; + constructor({ log, coreDbPath, @@ -2505,6 +2508,16 @@ export class StandaloneSqliteDatabase this.getTransactionAttributesCircuitBreaker, ]); + // + // Initialize save nested data hash cache + // + + this.saveNestedDataHashCache = new NodeCache({ + stdTTL: 60 * 7, // 7 minutes + checkperiod: 60, // 1 minute + useClones: false, + }); + // // Initialize workers // @@ -2924,6 +2937,19 @@ export class StandaloneSqliteDatabase parentId: string; dataOffset: number; }): Promise { + const key = `${hash}:${parentId}`; + + console.log('cache size:', this.saveNestedDataHashCache.getStats()); + + if (this.saveNestedDataHashCache.get(key)) { + metrics.saveMethodsDuplicateCounter.inc({ + method: 'saveNestedDataHash', + }); + + return; + } + this.saveNestedDataHashCache.set(key, true); + return this.queueWrite('data', 'saveNestedDataHash', [ { hash, diff --git a/src/metrics.ts b/src/metrics.ts index 6baf0cff..f4ab855c 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -124,6 +124,12 @@ export const methodDurationSummary = new promClient.Summary({ labelNames: ['worker', 'role', 'method'], }); +export const saveMethodsDuplicateCounter = new promClient.Counter({ + name: 'save_methods_duplicate_total', + help: 'Count of duplicate calls to save methods', + labelNames: ['method'], +}); + // // Block importer metrics // From 937fe82fea1b267dab623d5cb4593ea1f1e65628 Mon Sep 17 00:00:00 2001 From: Karl Prieb Date: Wed, 18 Sep 2024 15:21:24 -0300 Subject: [PATCH 2/4] cache for saveDataContentAttributes calls to prevent duplicate writes --- src/database/standalone-sqlite.ts | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/database/standalone-sqlite.ts b/src/database/standalone-sqlite.ts index aa77cb78..394e61a0 100644 --- a/src/database/standalone-sqlite.ts +++ b/src/database/standalone-sqlite.ts @@ -2434,6 +2434,7 @@ export class StandaloneSqliteDatabase >; private saveNestedDataHashCache: NodeCache; + private saveDataContentAttributesCache: NodeCache; constructor({ log, @@ -2509,7 +2510,7 @@ export class StandaloneSqliteDatabase ]); // - // Initialize save nested data hash cache + // Initialize method caches // this.saveNestedDataHashCache = new NodeCache({ @@ -2518,6 +2519,12 @@ export class StandaloneSqliteDatabase useClones: false, }); + this.saveDataContentAttributesCache = new NodeCache({ + stdTTL: 60 * 7, // 7 minutes + checkperiod: 60, // 1 minute + useClones: false, + }); + // // Initialize workers // @@ -2793,6 +2800,15 @@ export class StandaloneSqliteDatabase dataSize: number; contentType?: string; }) { + if (this.saveDataContentAttributesCache.get(id)) { + metrics.saveMethodsDuplicateCounter.inc({ + method: 'saveDataContentAttributes', + }); + return Promise.resolve(); + } + + this.saveDataContentAttributesCache.set(id, true); + return this.queueWrite('data', 'saveDataContentAttributes', [ { id, @@ -2939,8 +2955,6 @@ export class StandaloneSqliteDatabase }): Promise { const key = `${hash}:${parentId}`; - console.log('cache size:', this.saveNestedDataHashCache.getStats()); - if (this.saveNestedDataHashCache.get(key)) { metrics.saveMethodsDuplicateCounter.inc({ method: 'saveNestedDataHash', From 9a4e827c044429abf530537a661086cea8eba365 Mon Sep 17 00:00:00 2001 From: Karl Prieb Date: Wed, 18 Sep 2024 15:26:43 -0300 Subject: [PATCH 3/4] cache for insertDataHash calls to prevent duplicate writes --- src/database/standalone-sqlite.ts | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/src/database/standalone-sqlite.ts b/src/database/standalone-sqlite.ts index 394e61a0..c9cfc4aa 100644 --- a/src/database/standalone-sqlite.ts +++ b/src/database/standalone-sqlite.ts @@ -406,6 +406,8 @@ export class StandaloneSqliteDatabaseWorker { private bundleFormatIds: { [filter: string]: number } = {}; private filterIds: { [filter: string]: number } = {}; + private insertDataHashCache: NodeCache; + // Transactions resetBundlesToHeightFn: Sqlite.Transaction; resetCoreToHeightFn: Sqlite.Transaction; @@ -762,6 +764,12 @@ export class StandaloneSqliteDatabaseWorker { }); }, ); + + this.insertDataHashCache = new NodeCache({ + stdTTL: 60 * 7, // 7 minutes + checkperiod: 60, // 1 minute + useClones: false, + }); } getMaxHeight() { @@ -1133,13 +1141,7 @@ export class StandaloneSqliteDatabaseWorker { cachedAt?: number; }) { const hashBuffer = fromB64Url(hash); - this.stmts.data.insertDataHash.run({ - hash: hashBuffer, - data_size: dataSize, - original_source_content_type: contentType, - indexed_at: currentUnixTimestamp(), - cached_at: cachedAt, - }); + this.stmts.data.insertDataId.run({ id: fromB64Url(id), contiguous_data_hash: hashBuffer, @@ -1152,6 +1154,19 @@ export class StandaloneSqliteDatabaseWorker { indexed_at: currentUnixTimestamp(), }); } + + if (this.insertDataHashCache.get(hash)) { + return; + } + this.insertDataHashCache.set(hash, true); + + this.stmts.data.insertDataHash.run({ + hash: hashBuffer, + data_size: dataSize, + original_source_content_type: contentType, + indexed_at: currentUnixTimestamp(), + cached_at: cachedAt, + }); } getGqlNewTransactionTags(txId: Buffer) { From 089ee33e2e41ec9da315f646bd3b9fd9f27c4419 Mon Sep 17 00:00:00 2001 From: Karl Prieb Date: Fri, 20 Sep 2024 12:22:53 -0300 Subject: [PATCH 4/4] revert saveNestedDataHash cache --- src/database/standalone-sqlite.ts | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/database/standalone-sqlite.ts b/src/database/standalone-sqlite.ts index c9cfc4aa..9aa48b54 100644 --- a/src/database/standalone-sqlite.ts +++ b/src/database/standalone-sqlite.ts @@ -2448,7 +2448,6 @@ export class StandaloneSqliteDatabase Awaited> >; - private saveNestedDataHashCache: NodeCache; private saveDataContentAttributesCache: NodeCache; constructor({ @@ -2528,12 +2527,6 @@ export class StandaloneSqliteDatabase // Initialize method caches // - this.saveNestedDataHashCache = new NodeCache({ - stdTTL: 60 * 7, // 7 minutes - checkperiod: 60, // 1 minute - useClones: false, - }); - this.saveDataContentAttributesCache = new NodeCache({ stdTTL: 60 * 7, // 7 minutes checkperiod: 60, // 1 minute @@ -2968,17 +2961,6 @@ export class StandaloneSqliteDatabase parentId: string; dataOffset: number; }): Promise { - const key = `${hash}:${parentId}`; - - if (this.saveNestedDataHashCache.get(key)) { - metrics.saveMethodsDuplicateCounter.inc({ - method: 'saveNestedDataHash', - }); - - return; - } - this.saveNestedDataHashCache.set(key, true); - return this.queueWrite('data', 'saveNestedDataHash', [ { hash,