From 766c1bbb082debe9d6fa94e08466d3596c971843 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Thu, 5 Sep 2024 14:04:12 -0400 Subject: [PATCH] feat(swing-store): Limit item deletion to the previously-current transcript span Fixes #9387 --- packages/swing-store/src/transcriptStore.js | 39 +++-- packages/swing-store/test/state.test.js | 158 +++++++++++--------- 2 files changed, 114 insertions(+), 83 deletions(-) diff --git a/packages/swing-store/src/transcriptStore.js b/packages/swing-store/src/transcriptStore.js index 73145cfe9e2..4a7ae9c47d6 100644 --- a/packages/swing-store/src/transcriptStore.js +++ b/packages/swing-store/src/transcriptStore.js @@ -80,15 +80,21 @@ export function makeTranscriptStore( ) `); - // Transcripts are broken up into "spans", delimited by heap snapshots. If we - // take heap snapshots after deliveries 100 and 200, and have not yet - // performed delivery 201, we'll have two non-current (i.e., isCurrent=null) - // spans (one with startPos=0, endPos=100, the second with startPos=100, - // endPos=200), and a single empty isCurrent==1 span with startPos=200 and - // endPos=200. After we perform delivery 201, the single isCurrent=1 span - // will will still have startPos=200 but will now have endPos=201. For every - // vatID, there will be exactly one isCurrent=1 span, and zero or more - // non-current (historical) spans. + // Transcripts are broken up into "spans", delimited by heap snapshots. + // The items of each transcript consist of deliveries and pseudo-deliveries + // such as initialize-worker and load-snapshot. + // For every vatID, there will be exactly one current span (with isCurrent=1), + // and zero or more non-current (historical) spans (with isCurrent=null). + // If we take a heap snapshot after the first hundred items and again after + // the second hundred (i.e., after zero-indexed items 99 and 199), + // and have not yet extended the transcript after the second snapshot, we'll + // have two historical spans (one with startPos=0 and endPos=100, the second + // with startPos=100 and endPos=200) and a single empty current span with + // startPos=200 and endPos=200. But this situation is transient, and will + // generally be followed by a load-snapshot pseudo-delivery before the next + // commit (at which point the single current span will still have startPos=200 + // but will have endPos=201). After we perform the next delivery, the single + // current span will still have startPos=200 but will now have endPos=202. // // The transcriptItems associated with historical spans may or may not exist, // depending on pruning. However, the items associated with the current span @@ -246,7 +252,7 @@ export function makeTranscriptStore( const sqlDeleteOldItems = db.prepare(` DELETE FROM transcriptItems - WHERE vatID = ? AND position < ? + WHERE vatID = ? AND position >= ? AND position < ? `); function doSpanRollover(vatID, isNewIncarnation) { @@ -276,12 +282,13 @@ export function makeTranscriptStore( noteExport(spanMetadataKey(newRec), JSON.stringify(newRec)); if (!keepTranscripts) { - // TODO: for #9174 (delete historical transcript spans), we need - // this DB statement to only delete the items of the old span - // (startPos..endPos), not all previous items, otherwise the - // first rollover after switching to keepTranscripts=false will - // do a huge DB commit and probably explode - sqlDeleteOldItems.run(vatID, endPos); + // Delete items of the previously-current span. + // There may still be items associated with even older spans, but we leave + // those, to avoid excessive DB churn (for details, see #9387 and #9174). + // Recovery of space claimed by such ancient items is expected to use an + // external mechanism such as restoration from an operational snapshot + // that doesn't include them. + sqlDeleteOldItems.run(vatID, startPos, endPos); } return incarnationToUse; } diff --git a/packages/swing-store/test/state.test.js b/packages/swing-store/test/state.test.js index 490f7953035..d9bf119a83e 100644 --- a/packages/swing-store/test/state.test.js +++ b/packages/swing-store/test/state.test.js @@ -155,82 +155,106 @@ test('persistent kvStore maxKeySize write', async t => { await hostStorage.close(); }); -async function testTranscriptStore(t, dbDir) { - const exportLog = makeExportLog(); - const { kernelStorage, hostStorage } = initSwingStore(dbDir, { - exportCallback: exportLog.callback, - keepTranscripts: true, // XXX need to vary - }); - const { transcriptStore } = kernelStorage; - const { commit, close } = hostStorage; +const testTranscriptStore = test.macro({ + title(prefix = '', { ephemeral, keepTranscripts }) { + const type = ephemeral ? 'in-memory' : 'persistent'; + const detail = keepTranscripts ? 'with retention' : 'without retention'; + return `${prefix.replace(/.$/, '$& ')}${type} transcriptStore ${detail}`; + }, + async exec(t, { ephemeral, keepTranscripts }) { + let dbDir = null; + if (!ephemeral) { + const [tmpPath, cleanup] = await tmpDir('testdb'); + t.teardown(cleanup); + t.is(isSwingStore(tmpPath), false); + dbDir = tmpPath; + } - transcriptStore.initTranscript('st1'); - transcriptStore.initTranscript('st2'); - transcriptStore.addItem('st1', 'first'); - transcriptStore.addItem('st1', 'second'); - transcriptStore.rolloverSpan('st1'); - transcriptStore.addItem('st1', 'third'); - transcriptStore.addItem('st2', 'oneth'); - transcriptStore.addItem('st1', 'fourth'); - transcriptStore.addItem('st2', 'twoth'); - transcriptStore.addItem('st2', 'threeth'); - transcriptStore.addItem('st2', 'fourst'); - const reader1 = transcriptStore.readSpan('st1', 0); - t.deepEqual(Array.from(reader1), ['first', 'second']); - const reader2 = transcriptStore.readSpan('st2', 0); - t.deepEqual(Array.from(reader2), ['oneth', 'twoth', 'threeth', 'fourst']); - - t.throws(() => transcriptStore.readSpan('st2', 3), { - message: 'no transcript span for "st2" at 3', - }); + const exportLog = makeExportLog(); + const { kernelStorage, hostStorage } = initSwingStore(dbDir, { + exportCallback: exportLog.callback, + keepTranscripts, + }); + const { transcriptStore } = kernelStorage; + const { commit, close } = hostStorage; + + transcriptStore.initTranscript('st1'); + transcriptStore.initTranscript('st2'); + transcriptStore.addItem('st1', 'zeroth'); + transcriptStore.rolloverSpan('st1'); + transcriptStore.addItem('st1', 'first'); + transcriptStore.addItem('st1', 'second'); + transcriptStore.rolloverSpan('st1'); + transcriptStore.addItem('st1', 'third'); + transcriptStore.addItem('st2', 'oneth'); + transcriptStore.addItem('st1', 'fourth'); + transcriptStore.addItem('st2', 'twoth'); + transcriptStore.addItem('st2', 'threeth'); + transcriptStore.addItem('st2', 'fourst'); + const reader1a = transcriptStore.readSpan('st1', 0); + const reader1b = transcriptStore.readSpan('st1', 1); + if (keepTranscripts) { + t.deepEqual(Array.from(reader1a), ['zeroth']); + t.deepEqual(Array.from(reader1b), ['first', 'second']); + } else { + const fna = async () => Array.from(reader1a); + const fnb = async () => Array.from(reader1b); + await t.throwsAsync(fna, undefined, 'pruned spans must not be readable'); + await t.throwsAsync(fnb, undefined, 'pruned spans must not be readable'); + } + const reader2 = transcriptStore.readSpan('st2', 0); + t.deepEqual(Array.from(reader2), ['oneth', 'twoth', 'threeth', 'fourst']); - const reader1alt = transcriptStore.readSpan('st1'); - t.deepEqual(Array.from(reader1alt), ['third', 'fourth']); - const reader1alt2 = transcriptStore.readSpan('st1', 2); - t.deepEqual(Array.from(reader1alt2), ['third', 'fourth']); + t.throws(() => transcriptStore.readSpan('st2', 3), { + message: 'no transcript span for "st2" at 3', + }); - transcriptStore.initTranscript('empty'); - const readerEmpty = transcriptStore.readSpan('empty'); - t.deepEqual(Array.from(readerEmpty), []); + const reader1alt = transcriptStore.readSpan('st1'); + t.deepEqual(Array.from(reader1alt), ['third', 'fourth']); + const reader1alt2 = transcriptStore.readSpan('st1', 3); + t.deepEqual(Array.from(reader1alt2), ['third', 'fourth']); - t.throws(() => transcriptStore.readSpan('nonexistent'), { - message: 'no current transcript for "nonexistent"', - }); + transcriptStore.initTranscript('empty'); + const readerEmpty = transcriptStore.readSpan('empty'); + t.deepEqual(Array.from(readerEmpty), []); - await commit(); - t.deepEqual(exportLog.getLog(), [ - [ - [ - 'transcript.empty.current', - '{"vatID":"empty","startPos":0,"endPos":0,"hash":"43e6be43a3a34d60c0ebeb8498b5849b094fc20fc68483a7aeb3624fa10f79f6","isCurrent":1,"incarnation":0}', - ], - [ - 'transcript.st1.0', - '{"vatID":"st1","startPos":0,"endPos":2,"hash":"d385c43882cfb5611d255e362a9a98626ba4e55dfc308fc346c144c696ae734e","isCurrent":0,"incarnation":0}', - ], - [ - 'transcript.st1.current', - '{"vatID":"st1","startPos":2,"endPos":4,"hash":"789342fab468506c624c713c46953992f53a7eaae390d634790d791636b96cab","isCurrent":1,"incarnation":0}', - ], + t.throws(() => transcriptStore.readSpan('nonexistent'), { + message: 'no current transcript for "nonexistent"', + }); + + await commit(); + t.deepEqual(exportLog.getLog(), [ [ - 'transcript.st2.current', - '{"vatID":"st2","startPos":0,"endPos":4,"hash":"45de7ae9d2be34148f9cf3000052e5d1374932d663442fe9f39a342d221cebf1","isCurrent":1,"incarnation":0}', + [ + 'transcript.empty.current', + '{"vatID":"empty","startPos":0,"endPos":0,"hash":"43e6be43a3a34d60c0ebeb8498b5849b094fc20fc68483a7aeb3624fa10f79f6","isCurrent":1,"incarnation":0}', + ], + [ + 'transcript.st1.0', + '{"vatID":"st1","startPos":0,"endPos":1,"hash":"92d0cf6ecd39b60b4e32dc65d4c6f343495928cb041f25b19e2825b17f4daa9a","isCurrent":0,"incarnation":0}', + ], + [ + 'transcript.st1.1', + '{"vatID":"st1","startPos":1,"endPos":3,"hash":"d385c43882cfb5611d255e362a9a98626ba4e55dfc308fc346c144c696ae734e","isCurrent":0,"incarnation":0}', + ], + [ + 'transcript.st1.current', + '{"vatID":"st1","startPos":3,"endPos":5,"hash":"789342fab468506c624c713c46953992f53a7eaae390d634790d791636b96cab","isCurrent":1,"incarnation":0}', + ], + [ + 'transcript.st2.current', + '{"vatID":"st2","startPos":0,"endPos":4,"hash":"45de7ae9d2be34148f9cf3000052e5d1374932d663442fe9f39a342d221cebf1","isCurrent":1,"incarnation":0}', + ], ], - ], - ]); - await close(); -} - -test('in-memory transcriptStore read/write', async t => { - await testTranscriptStore(t, null); + ]); + await close(); + }, }); -test('persistent transcriptStore read/write', async t => { - const [dbDir, cleanup] = await tmpDir('testdb'); - t.teardown(cleanup); - t.is(isSwingStore(dbDir), false); - await testTranscriptStore(t, dbDir); -}); +test(testTranscriptStore, { ephemeral: true, keepTranscripts: true }); +test(testTranscriptStore, { ephemeral: true, keepTranscripts: false }); +test(testTranscriptStore, { ephemeral: false, keepTranscripts: true }); +test(testTranscriptStore, { ephemeral: false, keepTranscripts: false }); test('transcriptStore abort', async t => { const [dbDir, cleanup] = await tmpDir('testdb');