Skip to content

Commit

Permalink
feat(swing-store): Limit item deletion to the previously-current tran…
Browse files Browse the repository at this point in the history 
…script span

Fixes #9387
  • Loading branch information
@gibson042
gibson042 committed Sep 6, 2024
1 parent 3cf6b57 commit 766c1bb
Show file tree
Hide file tree
Showing 2 changed files with 114 additions and 83 deletions.
39 changes: 23 additions & 16 deletions packages/swing-store/src/transcriptStore.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,15 +80,21 @@ export function makeTranscriptStore(
)
`);

// Transcripts are broken up into "spans", delimited by heap snapshots. If we
// take heap snapshots after deliveries 100 and 200, and have not yet
// performed delivery 201, we'll have two non-current (i.e., isCurrent=null)
// spans (one with startPos=0, endPos=100, the second with startPos=100,
// endPos=200), and a single empty isCurrent==1 span with startPos=200 and
// endPos=200. After we perform delivery 201, the single isCurrent=1 span
// will will still have startPos=200 but will now have endPos=201. For every
// vatID, there will be exactly one isCurrent=1 span, and zero or more
// non-current (historical) spans.
// Transcripts are broken up into "spans", delimited by heap snapshots.
// The items of each transcript consist of deliveries and pseudo-deliveries
// such as initialize-worker and load-snapshot.
// For every vatID, there will be exactly one current span (with isCurrent=1),
// and zero or more non-current (historical) spans (with isCurrent=null).
// If we take a heap snapshot after the first hundred items and again after
// the second hundred (i.e., after zero-indexed items 99 and 199),
// and have not yet extended the transcript after the second snapshot, we'll
// have two historical spans (one with startPos=0 and endPos=100, the second
// with startPos=100 and endPos=200) and a single empty current span with
// startPos=200 and endPos=200. But this situation is transient, and will
// generally be followed by a load-snapshot pseudo-delivery before the next
// commit (at which point the single current span will still have startPos=200
// but will have endPos=201). After we perform the next delivery, the single
// current span will still have startPos=200 but will now have endPos=202.
//
// The transcriptItems associated with historical spans may or may not exist,
// depending on pruning. However, the items associated with the current span
Expand Down Expand Up @@ -246,7 +252,7 @@ export function makeTranscriptStore(

const sqlDeleteOldItems = db.prepare(`
DELETE FROM transcriptItems
WHERE vatID = ? AND position < ?
WHERE vatID = ? AND position >= ? AND position < ?
`);

function doSpanRollover(vatID, isNewIncarnation) {
Expand Down Expand Up @@ -276,12 +282,13 @@ export function makeTranscriptStore(
noteExport(spanMetadataKey(newRec), JSON.stringify(newRec));

if (!keepTranscripts) {
// TODO: for #9174 (delete historical transcript spans), we need
// this DB statement to only delete the items of the old span
// (startPos..endPos), not all previous items, otherwise the
// first rollover after switching to keepTranscripts=false will
// do a huge DB commit and probably explode
sqlDeleteOldItems.run(vatID, endPos);
// Delete items of the previously-current span.
// There may still be items associated with even older spans, but we leave
// those, to avoid excessive DB churn (for details, see #9387 and #9174).
// Recovery of space claimed by such ancient items is expected to use an
// external mechanism such as restoration from an operational snapshot
// that doesn't include them.
sqlDeleteOldItems.run(vatID, startPos, endPos);
}
return incarnationToUse;
}
Expand Down
158 changes: 91 additions & 67 deletions packages/swing-store/test/state.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,82 +155,106 @@ test('persistent kvStore maxKeySize write', async t => {
await hostStorage.close();
});

async function testTranscriptStore(t, dbDir) {
const exportLog = makeExportLog();
const { kernelStorage, hostStorage } = initSwingStore(dbDir, {
exportCallback: exportLog.callback,
keepTranscripts: true, // XXX need to vary
});
const { transcriptStore } = kernelStorage;
const { commit, close } = hostStorage;
const testTranscriptStore = test.macro({
title(prefix = '', { ephemeral, keepTranscripts }) {
const type = ephemeral ? 'in-memory' : 'persistent';
const detail = keepTranscripts ? 'with retention' : 'without retention';
return `${prefix.replace(/.$/, '$& ')}${type} transcriptStore ${detail}`;
},
async exec(t, { ephemeral, keepTranscripts }) {
let dbDir = null;
if (!ephemeral) {
const [tmpPath, cleanup] = await tmpDir('testdb');
t.teardown(cleanup);
t.is(isSwingStore(tmpPath), false);
dbDir = tmpPath;
}

transcriptStore.initTranscript('st1');
transcriptStore.initTranscript('st2');
transcriptStore.addItem('st1', 'first');
transcriptStore.addItem('st1', 'second');
transcriptStore.rolloverSpan('st1');
transcriptStore.addItem('st1', 'third');
transcriptStore.addItem('st2', 'oneth');
transcriptStore.addItem('st1', 'fourth');
transcriptStore.addItem('st2', 'twoth');
transcriptStore.addItem('st2', 'threeth');
transcriptStore.addItem('st2', 'fourst');
const reader1 = transcriptStore.readSpan('st1', 0);
t.deepEqual(Array.from(reader1), ['first', 'second']);
const reader2 = transcriptStore.readSpan('st2', 0);
t.deepEqual(Array.from(reader2), ['oneth', 'twoth', 'threeth', 'fourst']);

t.throws(() => transcriptStore.readSpan('st2', 3), {
message: 'no transcript span for "st2" at 3',
});
const exportLog = makeExportLog();
const { kernelStorage, hostStorage } = initSwingStore(dbDir, {
exportCallback: exportLog.callback,
keepTranscripts,
});
const { transcriptStore } = kernelStorage;
const { commit, close } = hostStorage;

transcriptStore.initTranscript('st1');
transcriptStore.initTranscript('st2');
transcriptStore.addItem('st1', 'zeroth');
transcriptStore.rolloverSpan('st1');
transcriptStore.addItem('st1', 'first');
transcriptStore.addItem('st1', 'second');
transcriptStore.rolloverSpan('st1');
transcriptStore.addItem('st1', 'third');
transcriptStore.addItem('st2', 'oneth');
transcriptStore.addItem('st1', 'fourth');
transcriptStore.addItem('st2', 'twoth');
transcriptStore.addItem('st2', 'threeth');
transcriptStore.addItem('st2', 'fourst');
const reader1a = transcriptStore.readSpan('st1', 0);
const reader1b = transcriptStore.readSpan('st1', 1);
if (keepTranscripts) {
t.deepEqual(Array.from(reader1a), ['zeroth']);
t.deepEqual(Array.from(reader1b), ['first', 'second']);
} else {
const fna = async () => Array.from(reader1a);
const fnb = async () => Array.from(reader1b);
await t.throwsAsync(fna, undefined, 'pruned spans must not be readable');
await t.throwsAsync(fnb, undefined, 'pruned spans must not be readable');
}
const reader2 = transcriptStore.readSpan('st2', 0);
t.deepEqual(Array.from(reader2), ['oneth', 'twoth', 'threeth', 'fourst']);

const reader1alt = transcriptStore.readSpan('st1');
t.deepEqual(Array.from(reader1alt), ['third', 'fourth']);
const reader1alt2 = transcriptStore.readSpan('st1', 2);
t.deepEqual(Array.from(reader1alt2), ['third', 'fourth']);
t.throws(() => transcriptStore.readSpan('st2', 3), {
message: 'no transcript span for "st2" at 3',
});

transcriptStore.initTranscript('empty');
const readerEmpty = transcriptStore.readSpan('empty');
t.deepEqual(Array.from(readerEmpty), []);
const reader1alt = transcriptStore.readSpan('st1');
t.deepEqual(Array.from(reader1alt), ['third', 'fourth']);
const reader1alt2 = transcriptStore.readSpan('st1', 3);
t.deepEqual(Array.from(reader1alt2), ['third', 'fourth']);

t.throws(() => transcriptStore.readSpan('nonexistent'), {
message: 'no current transcript for "nonexistent"',
});
transcriptStore.initTranscript('empty');
const readerEmpty = transcriptStore.readSpan('empty');
t.deepEqual(Array.from(readerEmpty), []);

await commit();
t.deepEqual(exportLog.getLog(), [
[
[
'transcript.empty.current',
'{"vatID":"empty","startPos":0,"endPos":0,"hash":"43e6be43a3a34d60c0ebeb8498b5849b094fc20fc68483a7aeb3624fa10f79f6","isCurrent":1,"incarnation":0}',
],
[
'transcript.st1.0',
'{"vatID":"st1","startPos":0,"endPos":2,"hash":"d385c43882cfb5611d255e362a9a98626ba4e55dfc308fc346c144c696ae734e","isCurrent":0,"incarnation":0}',
],
[
'transcript.st1.current',
'{"vatID":"st1","startPos":2,"endPos":4,"hash":"789342fab468506c624c713c46953992f53a7eaae390d634790d791636b96cab","isCurrent":1,"incarnation":0}',
],
t.throws(() => transcriptStore.readSpan('nonexistent'), {
message: 'no current transcript for "nonexistent"',
});

await commit();
t.deepEqual(exportLog.getLog(), [
[
'transcript.st2.current',
'{"vatID":"st2","startPos":0,"endPos":4,"hash":"45de7ae9d2be34148f9cf3000052e5d1374932d663442fe9f39a342d221cebf1","isCurrent":1,"incarnation":0}',
[
'transcript.empty.current',
'{"vatID":"empty","startPos":0,"endPos":0,"hash":"43e6be43a3a34d60c0ebeb8498b5849b094fc20fc68483a7aeb3624fa10f79f6","isCurrent":1,"incarnation":0}',
],
[
'transcript.st1.0',
'{"vatID":"st1","startPos":0,"endPos":1,"hash":"92d0cf6ecd39b60b4e32dc65d4c6f343495928cb041f25b19e2825b17f4daa9a","isCurrent":0,"incarnation":0}',
],
[
'transcript.st1.1',
'{"vatID":"st1","startPos":1,"endPos":3,"hash":"d385c43882cfb5611d255e362a9a98626ba4e55dfc308fc346c144c696ae734e","isCurrent":0,"incarnation":0}',
],
[
'transcript.st1.current',
'{"vatID":"st1","startPos":3,"endPos":5,"hash":"789342fab468506c624c713c46953992f53a7eaae390d634790d791636b96cab","isCurrent":1,"incarnation":0}',
],
[
'transcript.st2.current',
'{"vatID":"st2","startPos":0,"endPos":4,"hash":"45de7ae9d2be34148f9cf3000052e5d1374932d663442fe9f39a342d221cebf1","isCurrent":1,"incarnation":0}',
],
],
],
]);
await close();
}

test('in-memory transcriptStore read/write', async t => {
await testTranscriptStore(t, null);
]);
await close();
},
});

test('persistent transcriptStore read/write', async t => {
const [dbDir, cleanup] = await tmpDir('testdb');
t.teardown(cleanup);
t.is(isSwingStore(dbDir), false);
await testTranscriptStore(t, dbDir);
});
test(testTranscriptStore, { ephemeral: true, keepTranscripts: true });
test(testTranscriptStore, { ephemeral: true, keepTranscripts: false });
test(testTranscriptStore, { ephemeral: false, keepTranscripts: true });
test(testTranscriptStore, { ephemeral: false, keepTranscripts: false });

test('transcriptStore abort', async t => {
const [dbDir, cleanup] = await tmpDir('testdb');
Expand Down

0 comments on commit 766c1bb

Please sign in to comment.