From 3cc8ce70421920b2ccc20463ce5f14ff5d14a6ec Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Thu, 5 Sep 2024 11:42:51 -0400 Subject: [PATCH 1/7] chore(cosmic-swingset): Validate AG_COSMOS_INIT `resolvedConfig` --- golang/cosmos/x/swingset/config.go | 4 +++ packages/cosmic-swingset/package.json | 7 ++-- packages/cosmic-swingset/src/chain-main.js | 41 +++++++++++++--------- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/golang/cosmos/x/swingset/config.go b/golang/cosmos/x/swingset/config.go index d96b53e2a13..11c898845bb 100644 --- a/golang/cosmos/x/swingset/config.go +++ b/golang/cosmos/x/swingset/config.go @@ -39,6 +39,10 @@ max_vats_online = {{ .Swingset.MaxVatsOnline }} ` // SwingsetConfig defines configuration for the SwingSet VM. +// "mapstructure" tag data is used to direct reads from app.toml; +// "json" tag data is used to populate init messages for the VM. +// This should be kept in sync with SwingsetConfigShape in +// ../../../../packages/cosmic-swingset/src/chain-main.js. // TODO: Consider extensions from docs/env.md. type SwingsetConfig struct { // SlogFile is the absolute path at which a SwingSet log "slog" file should be written. diff --git a/packages/cosmic-swingset/package.json b/packages/cosmic-swingset/package.json index abd5f3a0b51..339579e6740 100644 --- a/packages/cosmic-swingset/package.json +++ b/packages/cosmic-swingset/package.json @@ -22,7 +22,6 @@ "author": "Agoric", "license": "Apache-2.0", "dependencies": { - "@endo/errors": "^1.2.5", "@agoric/builders": "^0.1.0", "@agoric/cosmos": "^0.34.1", "@agoric/deploy-script-support": "^0.10.3", @@ -34,15 +33,17 @@ "@agoric/vm-config": "^0.1.0", "@endo/bundle-source": "^3.4.0", "@endo/env-options": "^1.1.6", - "@endo/far": "^1.1.5", + "@endo/errors": "^1.2.5", "@endo/import-bundle": "^1.2.2", "@endo/init": "^1.1.4", + "@endo/far": "^1.1.5", "@endo/marshal": "^1.5.3", "@endo/nat": "^5.0.10", + "@endo/patterns": "^1.4.3", "@endo/promise-kit": "^1.1.5", "@iarna/toml": "^2.2.3", - "@opentelemetry/sdk-metrics": "~1.9.0", "@opentelemetry/api": "~1.3.0", + "@opentelemetry/sdk-metrics": "~1.9.0", "anylogger": "^0.21.0", "deterministic-json": "^1.0.5", "import-meta-resolve": "^2.2.1", diff --git a/packages/cosmic-swingset/src/chain-main.js b/packages/cosmic-swingset/src/chain-main.js index 95dd92f4e6f..9cf098c3584 100644 --- a/packages/cosmic-swingset/src/chain-main.js +++ b/packages/cosmic-swingset/src/chain-main.js @@ -12,6 +12,9 @@ import { fork } from 'node:child_process'; import { Fail, q } from '@endo/errors'; import { E } from '@endo/far'; +import { makeMarshal } from '@endo/marshal'; +import { isNat } from '@endo/nat'; +import { M, mustMatch } from '@endo/patterns'; import engineGC from '@agoric/internal/src/lib-nodejs/engine-gc.js'; import { waitUntilQuiescent } from '@agoric/internal/src/lib-nodejs/waitUntilQuiescent.js'; import { @@ -25,7 +28,6 @@ import { makeChainStorageRoot, makeSerializeToStorage, } from '@agoric/internal/src/lib-chainStorage.js'; -import { makeMarshal } from '@endo/marshal'; import { makeShutdown } from '@agoric/internal/src/node/shutdown.js'; import * as STORAGE_PATH from '@agoric/internal/src/chain-storage-paths.js'; @@ -73,6 +75,22 @@ const toNumber = specimen => { * @property {string} [slogfile] * @property {number} [maxVatsOnline] */ +const SwingsetConfigShape = M.splitRecord( + // All known properties are optional, but unknown properties are not allowed. + {}, + { + slogfile: M.string(), + maxVatsOnline: M.number(), + }, + {}, +); +const validateSwingsetConfig = swingsetConfig => { + mustMatch(swingsetConfig, SwingsetConfigShape); + const { maxVatsOnline } = swingsetConfig; + maxVatsOnline === undefined || + (isNat(maxVatsOnline) && maxVatsOnline > 0) || + Fail`maxVatsOnline must be a positive integer`; +}; /** * A boot message consists of cosmosInitAction fields that are subject to @@ -100,19 +118,6 @@ const makeBootMsg = initAction => { }; }; -/** - * Extract local Swingset-specific configuration which is - * not part of the consensus. - * - * @param {CosmosSwingsetConfig} [resolvedConfig] - */ -const makeSwingsetConfig = resolvedConfig => { - const { maxVatsOnline } = resolvedConfig || {}; - return { - maxVatsOnline, - }; -}; - /** * @template {unknown} [T=unknown] * @param {(req: string) => string} call @@ -301,13 +306,15 @@ export default async function main(progname, args, { env, homedir, agcc }) { // here so 'sendToChainStorage' can close over the single mutable instance, // when we updated the 'portNums.storage' value each time toSwingSet was called. async function launchAndInitializeSwingSet(initAction) { + /** @type {CosmosSwingsetConfig} */ + const swingsetConfig = harden(initAction.resolvedConfig || {}); + validateSwingsetConfig(swingsetConfig); + const { slogfile } = swingsetConfig; + // As a kludge, back-propagate selected configuration into environment variables. - const { slogfile } = initAction.resolvedConfig || {}; // eslint-disable-next-line dot-notation if (slogfile) env['SLOGFILE'] = slogfile; - const swingsetConfig = makeSwingsetConfig(initAction.resolvedConfig); - const sendToChainStorage = msg => chainSend(portNums.storage, msg); // this object is used to store the mailbox state. const fromBridgeMailbox = data => { From ea99c6d49633846fbb3e434aac2b901612bf7679 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Thu, 5 Sep 2024 11:49:18 -0400 Subject: [PATCH 2/7] chore(golang/cosmos): Align `[swingset]` TOML with cosmos-sdk The preferred separator is "-" rather than "_", cf. https://github.com/cosmos/cosmos-sdk/blob/a57b25418a594ae023274673f07b72611ccd2744/server/config/config.toml.tpl --- golang/cosmos/x/swingset/config.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/golang/cosmos/x/swingset/config.go b/golang/cosmos/x/swingset/config.go index 11c898845bb..3ed38a37635 100644 --- a/golang/cosmos/x/swingset/config.go +++ b/golang/cosmos/x/swingset/config.go @@ -27,15 +27,17 @@ const DefaultConfigTemplate = ` ############################################################################### [swingset] -# slogfile is the path at which a SwingSet log "slog" file should be written. +# The path at which a SwingSet log "slog" file should be written. # If relative, it is interpreted against the application home directory # (e.g., ~/.agoric). +# May be overridden by a SLOGFILE environment variable, which if relative is +# interpreted against the working directory. slogfile = "{{ .Swingset.SlogFile }}" # The maximum number of vats that the SwingSet kernel will bring online. A lower number # requires less memory but may have a negative performance impact if vats need to # be frequently paged out to remain under this limit. -max_vats_online = {{ .Swingset.MaxVatsOnline }} +max-vats-online = {{ .Swingset.MaxVatsOnline }} ` // SwingsetConfig defines configuration for the SwingSet VM. @@ -45,11 +47,12 @@ max_vats_online = {{ .Swingset.MaxVatsOnline }} // ../../../../packages/cosmic-swingset/src/chain-main.js. // TODO: Consider extensions from docs/env.md. type SwingsetConfig struct { - // SlogFile is the absolute path at which a SwingSet log "slog" file should be written. + // SlogFile is the path at which a SwingSet log "slog" file should be written. + // If relative, it is interpreted against the application home directory SlogFile string `mapstructure:"slogfile" json:"slogfile,omitempty"` // MaxVatsOnline is the maximum number of vats that the SwingSet kernel will have online // at any given time. - MaxVatsOnline int `mapstructure:"max_vats_online" json:"maxVatsOnline,omitempty"` + MaxVatsOnline int `mapstructure:"max-vats-online" json:"maxVatsOnline,omitempty"` } var DefaultSwingsetConfig = SwingsetConfig{ From 3cf6b57d9e1968c6197147419d5d177b5c42e62b Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Thu, 5 Sep 2024 12:23:20 -0400 Subject: [PATCH 3/7] feat: Add consensus-independent vat transcript span retention configuration to AG_COSMOS_INIT Ref #9174 Ref #9386 --- golang/cosmos/util/util.go | 9 +++ golang/cosmos/x/swingset/config.go | 67 +++++++++++++++++--- packages/cosmic-swingset/src/chain-main.js | 7 +- packages/cosmic-swingset/src/launch-chain.js | 2 + 4 files changed, 76 insertions(+), 9 deletions(-) diff --git a/golang/cosmos/util/util.go b/golang/cosmos/util/util.go index c23ad788a18..c2d7533a078 100644 --- a/golang/cosmos/util/util.go +++ b/golang/cosmos/util/util.go @@ -4,6 +4,15 @@ import ( "github.com/spf13/viper" ) +func IndexOf[T comparable](a []T, x T) int { + for i, s := range a { + if s == x { + return i + } + } + return -1 +} + func NewFileOnlyViper(v1 *viper.Viper) (*viper.Viper, error) { v2 := viper.New() v2.SetConfigFile(v1.ConfigFileUsed()) diff --git a/golang/cosmos/x/swingset/config.go b/golang/cosmos/x/swingset/config.go index 3ed38a37635..3a93c7fa92d 100644 --- a/golang/cosmos/x/swingset/config.go +++ b/golang/cosmos/x/swingset/config.go @@ -7,6 +7,8 @@ import ( "github.com/spf13/viper" "github.com/cosmos/cosmos-sdk/client/flags" + pruningtypes "github.com/cosmos/cosmos-sdk/pruning/types" + serverconfig "github.com/cosmos/cosmos-sdk/server/config" servertypes "github.com/cosmos/cosmos-sdk/server/types" "github.com/Agoric/agoric-sdk/golang/cosmos/util" @@ -15,8 +17,16 @@ import ( const ( ConfigPrefix = "swingset" FlagSlogfile = ConfigPrefix + ".slogfile" + + TranscriptRetentionOptionArchival = "archival" + TranscriptRetentionOptionOperational = "operational" ) +var transcriptRetentionValues []string = []string{ + TranscriptRetentionOptionArchival, + TranscriptRetentionOptionOperational, +} + // DefaultConfigTemplate defines a default TOML configuration section for the SwingSet VM. // Values are pulled from a "Swingset" property, in accord with CustomAppConfig from // ../../daemon/cmd/root.go. @@ -38,6 +48,16 @@ slogfile = "{{ .Swingset.SlogFile }}" # requires less memory but may have a negative performance impact if vats need to # be frequently paged out to remain under this limit. max-vats-online = {{ .Swingset.MaxVatsOnline }} + +# Retention of vat transcript spans, with values analogous to those of export +# ` + "`artifactMode`" + ` (cf. +# https://github.com/Agoric/agoric-sdk/blob/master/packages/swing-store/docs/data-export.md#optional--historical-data +# * "archival": keep all transcript spans +# * "operational": keep only necessary transcript spans (i.e., since the +# last snapshot of their vat) +# * "default": determined by 'pruning' ("archival" if 'pruning' is "nothing", +# otherwise "operational") +vat-transcript-retention = "{{ .Swingset.VatTranscriptRetention }}" ` // SwingsetConfig defines configuration for the SwingSet VM. @@ -53,11 +73,21 @@ type SwingsetConfig struct { // MaxVatsOnline is the maximum number of vats that the SwingSet kernel will have online // at any given time. MaxVatsOnline int `mapstructure:"max-vats-online" json:"maxVatsOnline,omitempty"` + // VatTranscriptRetention controls retention of vat transcript spans, + // and has values analogous to those of export `artifactMode` (cf. + // ../../../../packages/swing-store/docs/data-export.md#optional--historical-data ). + // * "archival": keep all transcript spans + // * "operational": keep only necessary transcript spans (i.e., since the + // last snapshot of their vat) + // * "default": determined by `pruning` ("archival" if `pruning` is + // "nothing", otherwise "operational") + VatTranscriptRetention string `mapstructure:"vat-transcript-retention" json:"vatTranscriptRetention,omitempty"` } var DefaultSwingsetConfig = SwingsetConfig{ - SlogFile: "", - MaxVatsOnline: 50, + SlogFile: "", + MaxVatsOnline: 50, + VatTranscriptRetention: "default", } func SwingsetConfigFromViper(resolvedConfig servertypes.AppOptions) (*SwingsetConfig, error) { @@ -73,11 +103,32 @@ func SwingsetConfigFromViper(resolvedConfig servertypes.AppOptions) (*SwingsetCo return nil, nil } v.MustBindEnv(FlagSlogfile, "SLOGFILE") - wrapper := struct{ Swingset SwingsetConfig }{} - if err := v.Unmarshal(&wrapper); err != nil { + // See CustomAppConfig in ../../daemon/cmd/root.go. + type ExtendedConfig struct { + serverconfig.Config `mapstructure:",squash"` + Swingset SwingsetConfig `mapstructure:"swingset"` + } + extendedConfig := ExtendedConfig{} + if err := v.Unmarshal(&extendedConfig); err != nil { + return nil, err + } + ssConfig := &extendedConfig.Swingset + + // Default/validate transcript retention. + if ssConfig.VatTranscriptRetention == "" || ssConfig.VatTranscriptRetention == "default" { + if extendedConfig.Pruning == pruningtypes.PruningOptionNothing { + ssConfig.VatTranscriptRetention = TranscriptRetentionOptionArchival + } else { + ssConfig.VatTranscriptRetention = TranscriptRetentionOptionOperational + } + } + if util.IndexOf(transcriptRetentionValues, ssConfig.VatTranscriptRetention) == -1 { + err := fmt.Errorf( + "value for vat-transcript-retention must be in %q", + transcriptRetentionValues, + ) return nil, err } - config := &wrapper.Swingset // Interpret relative paths from config files against the application home // directory and from other sources (e.g. env vars) against the current @@ -108,11 +159,11 @@ func SwingsetConfigFromViper(resolvedConfig servertypes.AppOptions) (*SwingsetCo return filepath.Abs(path) } - resolvedSlogFile, err := resolvePath(config.SlogFile, FlagSlogfile) + resolvedSlogFile, err := resolvePath(ssConfig.SlogFile, FlagSlogfile) if err != nil { return nil, err } - config.SlogFile = resolvedSlogFile + ssConfig.SlogFile = resolvedSlogFile - return config, nil + return ssConfig, nil } diff --git a/packages/cosmic-swingset/src/chain-main.js b/packages/cosmic-swingset/src/chain-main.js index 9cf098c3584..8b8af2b36d7 100644 --- a/packages/cosmic-swingset/src/chain-main.js +++ b/packages/cosmic-swingset/src/chain-main.js @@ -74,6 +74,7 @@ const toNumber = specimen => { * @typedef {object} CosmosSwingsetConfig * @property {string} [slogfile] * @property {number} [maxVatsOnline] + * @property {'archival' | 'operational'} [vatTranscriptRetention] */ const SwingsetConfigShape = M.splitRecord( // All known properties are optional, but unknown properties are not allowed. @@ -81,6 +82,7 @@ const SwingsetConfigShape = M.splitRecord( { slogfile: M.string(), maxVatsOnline: M.number(), + vatTranscriptRetention: M.or('archival', 'operational'), }, {}, ); @@ -309,7 +311,8 @@ export default async function main(progname, args, { env, homedir, agcc }) { /** @type {CosmosSwingsetConfig} */ const swingsetConfig = harden(initAction.resolvedConfig || {}); validateSwingsetConfig(swingsetConfig); - const { slogfile } = swingsetConfig; + const { slogfile, vatTranscriptRetention } = swingsetConfig; + const keepTranscripts = vatTranscriptRetention === 'archival'; // As a kludge, back-propagate selected configuration into environment variables. // eslint-disable-next-line dot-notation @@ -462,6 +465,7 @@ export default async function main(progname, args, { env, homedir, agcc }) { trueValue: pathResolve(stateDBDir, 'store-trace.log'), }); + // TODO: Add to SwingsetConfig (#9386) const keepSnapshots = XSNAP_KEEP_SNAPSHOTS === '1' || XSNAP_KEEP_SNAPSHOTS === 'true'; @@ -546,6 +550,7 @@ export default async function main(progname, args, { env, homedir, agcc }) { swingStoreExportCallback, swingStoreTraceFile, keepSnapshots, + keepTranscripts, afterCommitCallback, swingsetConfig, }); diff --git a/packages/cosmic-swingset/src/launch-chain.js b/packages/cosmic-swingset/src/launch-chain.js index 2fbcc151542..e10cf93285c 100644 --- a/packages/cosmic-swingset/src/launch-chain.js +++ b/packages/cosmic-swingset/src/launch-chain.js @@ -331,6 +331,7 @@ export async function launch({ swingStoreTraceFile, swingStoreExportCallback, keepSnapshots, + keepTranscripts, afterCommitCallback = async () => ({}), swingsetConfig, }) { @@ -373,6 +374,7 @@ export async function launch({ traceFile: swingStoreTraceFile, exportCallback: swingStoreExportSyncCallback, keepSnapshots, + keepTranscripts, }); const { kvStore, commit } = hostStorage; From 766c1bbb082debe9d6fa94e08466d3596c971843 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Thu, 5 Sep 2024 14:04:12 -0400 Subject: [PATCH 4/7] feat(swing-store): Limit item deletion to the previously-current transcript span Fixes #9387 --- packages/swing-store/src/transcriptStore.js | 39 +++-- packages/swing-store/test/state.test.js | 158 +++++++++++--------- 2 files changed, 114 insertions(+), 83 deletions(-) diff --git a/packages/swing-store/src/transcriptStore.js b/packages/swing-store/src/transcriptStore.js index 73145cfe9e2..4a7ae9c47d6 100644 --- a/packages/swing-store/src/transcriptStore.js +++ b/packages/swing-store/src/transcriptStore.js @@ -80,15 +80,21 @@ export function makeTranscriptStore( ) `); - // Transcripts are broken up into "spans", delimited by heap snapshots. If we - // take heap snapshots after deliveries 100 and 200, and have not yet - // performed delivery 201, we'll have two non-current (i.e., isCurrent=null) - // spans (one with startPos=0, endPos=100, the second with startPos=100, - // endPos=200), and a single empty isCurrent==1 span with startPos=200 and - // endPos=200. After we perform delivery 201, the single isCurrent=1 span - // will will still have startPos=200 but will now have endPos=201. For every - // vatID, there will be exactly one isCurrent=1 span, and zero or more - // non-current (historical) spans. + // Transcripts are broken up into "spans", delimited by heap snapshots. + // The items of each transcript consist of deliveries and pseudo-deliveries + // such as initialize-worker and load-snapshot. + // For every vatID, there will be exactly one current span (with isCurrent=1), + // and zero or more non-current (historical) spans (with isCurrent=null). + // If we take a heap snapshot after the first hundred items and again after + // the second hundred (i.e., after zero-indexed items 99 and 199), + // and have not yet extended the transcript after the second snapshot, we'll + // have two historical spans (one with startPos=0 and endPos=100, the second + // with startPos=100 and endPos=200) and a single empty current span with + // startPos=200 and endPos=200. But this situation is transient, and will + // generally be followed by a load-snapshot pseudo-delivery before the next + // commit (at which point the single current span will still have startPos=200 + // but will have endPos=201). After we perform the next delivery, the single + // current span will still have startPos=200 but will now have endPos=202. // // The transcriptItems associated with historical spans may or may not exist, // depending on pruning. However, the items associated with the current span @@ -246,7 +252,7 @@ export function makeTranscriptStore( const sqlDeleteOldItems = db.prepare(` DELETE FROM transcriptItems - WHERE vatID = ? AND position < ? + WHERE vatID = ? AND position >= ? AND position < ? `); function doSpanRollover(vatID, isNewIncarnation) { @@ -276,12 +282,13 @@ export function makeTranscriptStore( noteExport(spanMetadataKey(newRec), JSON.stringify(newRec)); if (!keepTranscripts) { - // TODO: for #9174 (delete historical transcript spans), we need - // this DB statement to only delete the items of the old span - // (startPos..endPos), not all previous items, otherwise the - // first rollover after switching to keepTranscripts=false will - // do a huge DB commit and probably explode - sqlDeleteOldItems.run(vatID, endPos); + // Delete items of the previously-current span. + // There may still be items associated with even older spans, but we leave + // those, to avoid excessive DB churn (for details, see #9387 and #9174). + // Recovery of space claimed by such ancient items is expected to use an + // external mechanism such as restoration from an operational snapshot + // that doesn't include them. + sqlDeleteOldItems.run(vatID, startPos, endPos); } return incarnationToUse; } diff --git a/packages/swing-store/test/state.test.js b/packages/swing-store/test/state.test.js index 490f7953035..d9bf119a83e 100644 --- a/packages/swing-store/test/state.test.js +++ b/packages/swing-store/test/state.test.js @@ -155,82 +155,106 @@ test('persistent kvStore maxKeySize write', async t => { await hostStorage.close(); }); -async function testTranscriptStore(t, dbDir) { - const exportLog = makeExportLog(); - const { kernelStorage, hostStorage } = initSwingStore(dbDir, { - exportCallback: exportLog.callback, - keepTranscripts: true, // XXX need to vary - }); - const { transcriptStore } = kernelStorage; - const { commit, close } = hostStorage; +const testTranscriptStore = test.macro({ + title(prefix = '', { ephemeral, keepTranscripts }) { + const type = ephemeral ? 'in-memory' : 'persistent'; + const detail = keepTranscripts ? 'with retention' : 'without retention'; + return `${prefix.replace(/.$/, '$& ')}${type} transcriptStore ${detail}`; + }, + async exec(t, { ephemeral, keepTranscripts }) { + let dbDir = null; + if (!ephemeral) { + const [tmpPath, cleanup] = await tmpDir('testdb'); + t.teardown(cleanup); + t.is(isSwingStore(tmpPath), false); + dbDir = tmpPath; + } - transcriptStore.initTranscript('st1'); - transcriptStore.initTranscript('st2'); - transcriptStore.addItem('st1', 'first'); - transcriptStore.addItem('st1', 'second'); - transcriptStore.rolloverSpan('st1'); - transcriptStore.addItem('st1', 'third'); - transcriptStore.addItem('st2', 'oneth'); - transcriptStore.addItem('st1', 'fourth'); - transcriptStore.addItem('st2', 'twoth'); - transcriptStore.addItem('st2', 'threeth'); - transcriptStore.addItem('st2', 'fourst'); - const reader1 = transcriptStore.readSpan('st1', 0); - t.deepEqual(Array.from(reader1), ['first', 'second']); - const reader2 = transcriptStore.readSpan('st2', 0); - t.deepEqual(Array.from(reader2), ['oneth', 'twoth', 'threeth', 'fourst']); - - t.throws(() => transcriptStore.readSpan('st2', 3), { - message: 'no transcript span for "st2" at 3', - }); + const exportLog = makeExportLog(); + const { kernelStorage, hostStorage } = initSwingStore(dbDir, { + exportCallback: exportLog.callback, + keepTranscripts, + }); + const { transcriptStore } = kernelStorage; + const { commit, close } = hostStorage; + + transcriptStore.initTranscript('st1'); + transcriptStore.initTranscript('st2'); + transcriptStore.addItem('st1', 'zeroth'); + transcriptStore.rolloverSpan('st1'); + transcriptStore.addItem('st1', 'first'); + transcriptStore.addItem('st1', 'second'); + transcriptStore.rolloverSpan('st1'); + transcriptStore.addItem('st1', 'third'); + transcriptStore.addItem('st2', 'oneth'); + transcriptStore.addItem('st1', 'fourth'); + transcriptStore.addItem('st2', 'twoth'); + transcriptStore.addItem('st2', 'threeth'); + transcriptStore.addItem('st2', 'fourst'); + const reader1a = transcriptStore.readSpan('st1', 0); + const reader1b = transcriptStore.readSpan('st1', 1); + if (keepTranscripts) { + t.deepEqual(Array.from(reader1a), ['zeroth']); + t.deepEqual(Array.from(reader1b), ['first', 'second']); + } else { + const fna = async () => Array.from(reader1a); + const fnb = async () => Array.from(reader1b); + await t.throwsAsync(fna, undefined, 'pruned spans must not be readable'); + await t.throwsAsync(fnb, undefined, 'pruned spans must not be readable'); + } + const reader2 = transcriptStore.readSpan('st2', 0); + t.deepEqual(Array.from(reader2), ['oneth', 'twoth', 'threeth', 'fourst']); - const reader1alt = transcriptStore.readSpan('st1'); - t.deepEqual(Array.from(reader1alt), ['third', 'fourth']); - const reader1alt2 = transcriptStore.readSpan('st1', 2); - t.deepEqual(Array.from(reader1alt2), ['third', 'fourth']); + t.throws(() => transcriptStore.readSpan('st2', 3), { + message: 'no transcript span for "st2" at 3', + }); - transcriptStore.initTranscript('empty'); - const readerEmpty = transcriptStore.readSpan('empty'); - t.deepEqual(Array.from(readerEmpty), []); + const reader1alt = transcriptStore.readSpan('st1'); + t.deepEqual(Array.from(reader1alt), ['third', 'fourth']); + const reader1alt2 = transcriptStore.readSpan('st1', 3); + t.deepEqual(Array.from(reader1alt2), ['third', 'fourth']); - t.throws(() => transcriptStore.readSpan('nonexistent'), { - message: 'no current transcript for "nonexistent"', - }); + transcriptStore.initTranscript('empty'); + const readerEmpty = transcriptStore.readSpan('empty'); + t.deepEqual(Array.from(readerEmpty), []); - await commit(); - t.deepEqual(exportLog.getLog(), [ - [ - [ - 'transcript.empty.current', - '{"vatID":"empty","startPos":0,"endPos":0,"hash":"43e6be43a3a34d60c0ebeb8498b5849b094fc20fc68483a7aeb3624fa10f79f6","isCurrent":1,"incarnation":0}', - ], - [ - 'transcript.st1.0', - '{"vatID":"st1","startPos":0,"endPos":2,"hash":"d385c43882cfb5611d255e362a9a98626ba4e55dfc308fc346c144c696ae734e","isCurrent":0,"incarnation":0}', - ], - [ - 'transcript.st1.current', - '{"vatID":"st1","startPos":2,"endPos":4,"hash":"789342fab468506c624c713c46953992f53a7eaae390d634790d791636b96cab","isCurrent":1,"incarnation":0}', - ], + t.throws(() => transcriptStore.readSpan('nonexistent'), { + message: 'no current transcript for "nonexistent"', + }); + + await commit(); + t.deepEqual(exportLog.getLog(), [ [ - 'transcript.st2.current', - '{"vatID":"st2","startPos":0,"endPos":4,"hash":"45de7ae9d2be34148f9cf3000052e5d1374932d663442fe9f39a342d221cebf1","isCurrent":1,"incarnation":0}', + [ + 'transcript.empty.current', + '{"vatID":"empty","startPos":0,"endPos":0,"hash":"43e6be43a3a34d60c0ebeb8498b5849b094fc20fc68483a7aeb3624fa10f79f6","isCurrent":1,"incarnation":0}', + ], + [ + 'transcript.st1.0', + '{"vatID":"st1","startPos":0,"endPos":1,"hash":"92d0cf6ecd39b60b4e32dc65d4c6f343495928cb041f25b19e2825b17f4daa9a","isCurrent":0,"incarnation":0}', + ], + [ + 'transcript.st1.1', + '{"vatID":"st1","startPos":1,"endPos":3,"hash":"d385c43882cfb5611d255e362a9a98626ba4e55dfc308fc346c144c696ae734e","isCurrent":0,"incarnation":0}', + ], + [ + 'transcript.st1.current', + '{"vatID":"st1","startPos":3,"endPos":5,"hash":"789342fab468506c624c713c46953992f53a7eaae390d634790d791636b96cab","isCurrent":1,"incarnation":0}', + ], + [ + 'transcript.st2.current', + '{"vatID":"st2","startPos":0,"endPos":4,"hash":"45de7ae9d2be34148f9cf3000052e5d1374932d663442fe9f39a342d221cebf1","isCurrent":1,"incarnation":0}', + ], ], - ], - ]); - await close(); -} - -test('in-memory transcriptStore read/write', async t => { - await testTranscriptStore(t, null); + ]); + await close(); + }, }); -test('persistent transcriptStore read/write', async t => { - const [dbDir, cleanup] = await tmpDir('testdb'); - t.teardown(cleanup); - t.is(isSwingStore(dbDir), false); - await testTranscriptStore(t, dbDir); -}); +test(testTranscriptStore, { ephemeral: true, keepTranscripts: true }); +test(testTranscriptStore, { ephemeral: true, keepTranscripts: false }); +test(testTranscriptStore, { ephemeral: false, keepTranscripts: true }); +test(testTranscriptStore, { ephemeral: false, keepTranscripts: false }); test('transcriptStore abort', async t => { const [dbDir, cleanup] = await tmpDir('testdb'); From a5311b5a9eb257d4dfb4f18272608f00c1616abb Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Thu, 5 Sep 2024 14:41:22 -0400 Subject: [PATCH 5/7] feat: Add consensus-independent vat snapshot retention configuration to AG_COSMOS_INIT Fixes #9386 --- golang/cosmos/x/swingset/config.go | 43 ++++++++++++++++++++-- packages/cosmic-swingset/src/chain-main.js | 15 +++++--- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/golang/cosmos/x/swingset/config.go b/golang/cosmos/x/swingset/config.go index 3a93c7fa92d..c3b3bf7886e 100644 --- a/golang/cosmos/x/swingset/config.go +++ b/golang/cosmos/x/swingset/config.go @@ -18,10 +18,18 @@ const ( ConfigPrefix = "swingset" FlagSlogfile = ConfigPrefix + ".slogfile" + SnapshotRetentionOptionDebug = "debug" + SnapshotRetentionOptionOperational = "operational" + TranscriptRetentionOptionArchival = "archival" TranscriptRetentionOptionOperational = "operational" ) +var snapshotRetentionValues []string = []string{ + SnapshotRetentionOptionDebug, + SnapshotRetentionOptionOperational, +} + var transcriptRetentionValues []string = []string{ TranscriptRetentionOptionArchival, TranscriptRetentionOptionOperational, @@ -49,9 +57,16 @@ slogfile = "{{ .Swingset.SlogFile }}" # be frequently paged out to remain under this limit. max-vats-online = {{ .Swingset.MaxVatsOnline }} +# Retention of vat snapshots, with values analogous to those of export +# 'artifactMode' (cf. +# https://github.com/Agoric/agoric-sdk/blob/master/packages/swing-store/docs/data-export.md#optional--historical-data ). +# * "debug": keep all snapshots +# * "operational": keep only the last snapshot +vat-snapshot-retention = "{{ .Swingset.VatSnapshotRetention }}" + # Retention of vat transcript spans, with values analogous to those of export -# ` + "`artifactMode`" + ` (cf. -# https://github.com/Agoric/agoric-sdk/blob/master/packages/swing-store/docs/data-export.md#optional--historical-data +# 'artifactMode' (cf. +# https://github.com/Agoric/agoric-sdk/blob/master/packages/swing-store/docs/data-export.md#optional--historical-data ). # * "archival": keep all transcript spans # * "operational": keep only necessary transcript spans (i.e., since the # last snapshot of their vat) @@ -70,9 +85,18 @@ type SwingsetConfig struct { // SlogFile is the path at which a SwingSet log "slog" file should be written. // If relative, it is interpreted against the application home directory SlogFile string `mapstructure:"slogfile" json:"slogfile,omitempty"` + // MaxVatsOnline is the maximum number of vats that the SwingSet kernel will have online // at any given time. MaxVatsOnline int `mapstructure:"max-vats-online" json:"maxVatsOnline,omitempty"` + + // VatSnapshotRetention controls retention of vat snapshots, + // and has values analogous to those of export `artifactMode` (cf. + // ../../../../packages/swing-store/docs/data-export.md#optional--historical-data ). + // * "debug": keep all snapshots + // * "operational": keep only the last snapshot + VatSnapshotRetention string `mapstructure:"vat-snapshot-retention" json:"vatSnapshotRetention,omitempty"` + // VatTranscriptRetention controls retention of vat transcript spans, // and has values analogous to those of export `artifactMode` (cf. // ../../../../packages/swing-store/docs/data-export.md#optional--historical-data ). @@ -87,6 +111,7 @@ type SwingsetConfig struct { var DefaultSwingsetConfig = SwingsetConfig{ SlogFile: "", MaxVatsOnline: 50, + VatSnapshotRetention: "operational", VatTranscriptRetention: "default", } @@ -114,7 +139,19 @@ func SwingsetConfigFromViper(resolvedConfig servertypes.AppOptions) (*SwingsetCo } ssConfig := &extendedConfig.Swingset - // Default/validate transcript retention. + // Validate vat snapshot retention only if non-empty (because otherwise it + // it will be omitted, leaving the VM to apply its own defaults). + if ssConfig.VatSnapshotRetention != "" { + if util.IndexOf(snapshotRetentionValues, ssConfig.VatSnapshotRetention) == -1 { + err := fmt.Errorf( + "value for vat-snapshot-retention must be in %q", + snapshotRetentionValues, + ) + return nil, err + } + } + + // Default/validate vat transcript retention. if ssConfig.VatTranscriptRetention == "" || ssConfig.VatTranscriptRetention == "default" { if extendedConfig.Pruning == pruningtypes.PruningOptionNothing { ssConfig.VatTranscriptRetention = TranscriptRetentionOptionArchival diff --git a/packages/cosmic-swingset/src/chain-main.js b/packages/cosmic-swingset/src/chain-main.js index 8b8af2b36d7..cb56f8ec0ae 100644 --- a/packages/cosmic-swingset/src/chain-main.js +++ b/packages/cosmic-swingset/src/chain-main.js @@ -74,6 +74,7 @@ const toNumber = specimen => { * @typedef {object} CosmosSwingsetConfig * @property {string} [slogfile] * @property {number} [maxVatsOnline] + * @property {'debug' | 'operational'} [vatSnapshotRetention] * @property {'archival' | 'operational'} [vatTranscriptRetention] */ const SwingsetConfigShape = M.splitRecord( @@ -82,6 +83,7 @@ const SwingsetConfigShape = M.splitRecord( { slogfile: M.string(), maxVatsOnline: M.number(), + vatSnapshotRetention: M.or('debug', 'operational'), vatTranscriptRetention: M.or('archival', 'operational'), }, {}, @@ -308,10 +310,16 @@ export default async function main(progname, args, { env, homedir, agcc }) { // here so 'sendToChainStorage' can close over the single mutable instance, // when we updated the 'portNums.storage' value each time toSwingSet was called. async function launchAndInitializeSwingSet(initAction) { + const { XSNAP_KEEP_SNAPSHOTS, NODE_HEAP_SNAPSHOTS = -1 } = env; + /** @type {CosmosSwingsetConfig} */ const swingsetConfig = harden(initAction.resolvedConfig || {}); validateSwingsetConfig(swingsetConfig); - const { slogfile, vatTranscriptRetention } = swingsetConfig; + const { slogfile, vatSnapshotRetention, vatTranscriptRetention } = + swingsetConfig; + const keepSnapshots = + vatSnapshotRetention === 'debug' || + (!vatSnapshotRetention && ['1', 'true'].includes(XSNAP_KEEP_SNAPSHOTS)); const keepTranscripts = vatTranscriptRetention === 'archival'; // As a kludge, back-propagate selected configuration into environment variables. @@ -452,7 +460,6 @@ export default async function main(progname, args, { env, homedir, agcc }) { serviceName: TELEMETRY_SERVICE_NAME, }); - const { XSNAP_KEEP_SNAPSHOTS, NODE_HEAP_SNAPSHOTS = -1 } = env; const slogSender = await makeSlogSender({ stateDir: stateDBDir, env, @@ -465,10 +472,6 @@ export default async function main(progname, args, { env, homedir, agcc }) { trueValue: pathResolve(stateDBDir, 'store-trace.log'), }); - // TODO: Add to SwingsetConfig (#9386) - const keepSnapshots = - XSNAP_KEEP_SNAPSHOTS === '1' || XSNAP_KEEP_SNAPSHOTS === 'true'; - const nodeHeapSnapshots = Number.parseInt(NODE_HEAP_SNAPSHOTS, 10); let lastCommitTime = 0; From b01c0332991a243860fd5a0447ff520e80f2e2d1 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Thu, 5 Sep 2024 22:06:01 -0400 Subject: [PATCH 6/7] docs(swing-store): Update for changes --- packages/swing-store/docs/data-export.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/swing-store/docs/data-export.md b/packages/swing-store/docs/data-export.md index 9fd1ef21288..bf0ca524da3 100644 --- a/packages/swing-store/docs/data-export.md +++ b/packages/swing-store/docs/data-export.md @@ -188,13 +188,13 @@ Once the new SwingStore is fully populated with the previously-exported data, th Some of the data maintained by SwingStore is not strictly necessary for kernel execution, at least under normal circumstances. For example, once a vat worker performs a heap snapshot, we no longer need the transcript entries from before the snapshot was taken, since vat replay will start from the snapshot point. We split each vat's transcript into "spans", delimited by heap snapshot events, and the "current span" is the most recent one (still growing), whereas the "historical spans" are all closed and immutable. Likewise, we only really need the most recent heap snapshot for each vat: older snapshots might be interesting for experiments that replay old transcripts with different versions of the XS engine, but no normal kernel will ever need them. -Most validators would prefer to prune this data, to reduce their storage needs. But we can imagine some [extreme upgrade scenarios](https://github.com/Agoric/agoric-sdk/issues/1691) that would require access to these historical transcript spans. Our compromise is to record *validation data* for these historical spans in the export data, but omit the spans themselves from the export artifacts. Validators can delete the old spans at will, and if we ever need them in the future, we can add code that will fetch copies from an archive service, validate them against the export data hashes, and re-insert the relevant entries into the SwingStore. +Most blockchain validator nodes would prefer to prune this data, to reduce their storage needs. But we can imagine some [extreme upgrade scenarios](https://github.com/Agoric/agoric-sdk/issues/1691) that would require access to these historical transcript spans. Our compromise is to record *validation data* for these historical spans in the export data, but omit the spans themselves from the export artifacts. Validators can delete the old spans at will, and if we ever need them in the future, we can add code that will fetch copies from an archive service, validate them against the export data hashes, and re-insert the relevant entries into the SwingStore. Likewise, each time a heap snapshot is recorded, we cease to need any previous snapshot. And again, as a hedge against even more drastic recovery scenarios, we strike a compromise between minimizing retained data and the ability to validate old snapshots, by retaining only their hashes. -As a result, for each active vat, the first-stage Export Data contains a record for every old transcript span, plus one for the current span. It also contains a record for every old heap snapshot, plus one for the most recent heap snapshot, plus a `.current` record that points to the most recent snapshot. However the exported artifacts may or may not include blobs for the old transcript spans, or for the old heap snapshots. +As a result, for each active vat, the first-stage Export Data contains a record for every old heap snapshot, plus one for the most recent heap snapshot, plus a `.current` record that points to the most recent snapshot. It also contains a record for every old transcript span, plus one for the current span. However the exported artifacts may or may not include blobs for the old heap snapshots, or for the old transcript spans. -The `openSwingStore()` function has an option named `keepTranscripts` (which defaults to `true`), which causes the transcriptStore to retain the old transcript items. A second option named `keepSnapshots` (which defaults to `false`) causes the snapStore to retain the old heap snapshots. Opening the swingStore with a `false` option does not necessarily delete the old items immediately, but they'll probably get deleted the next time the kernel triggers a heap snapshot or transcript-span rollover. Validators who care about minimizing their disk usage will want to set both to `false`. In the future, we will arrange the SwingStore SQLite tables to provide easy `sqlite3` CLI commands that will delete the old data, so validators can also periodically use the CLI command to prune it. +The `openSwingStore()` function has an option named `keepSnapshots` (which defaults to `false`), which causes the snapStore to retain the old heap snapshots. A second option named `keepTranscripts` (which defaults to `true`) causes the transcriptStore to retain the old transcript items. Opening the swingStore with a `false` option does not necessarily delete the old items immediately, but they may get deleted the next time the kernel triggers a heap snapshot or transcript-span rollover. Hosts who care about minimizing their disk usage will want to set both to `false`. In the future, we will arrange the SwingStore SQLite tables to provide easy `sqlite3` CLI commands that will delete the old data, for use in periodic pruning. When exporting, the `makeSwingStoreExporter()` function takes an `artifactMode` option (in an options bag). This serves to both limit, and provide some minimal guarantees about, the set of artifacts that will be provided in the export. The defined values of `artifactMode` each build upon the previous one: @@ -218,9 +218,9 @@ While `importSwingStore()`'s options bag accepts the same options as `openSwingS So, to avoid pruning current-incarnation historical transcript spans when exporting from one swingstore to another, you must set (or avoid overriding) the following options along the way: * the original swingstore must not be opened with `{ keepTranscripts: false }`, otherwise the old spans will be pruned immediately -* the export must use `makeSwingStoreExporter(dirpath, { artifactMode: 'replay'})`, otherwise the export will omit the old spans -* the import must use `importSwingStore(exporter, dirPath, { artifactMode: 'replay'})`, otherwise the import will ignore the old spans - * subsequent `openSwingStore` calls must not use `keepTranscripts: false`, otherwise the new swingstore will prune historical spans as new ones are created (during `rolloverSpan`). +* the export must use `makeSwingStoreExporter(dirpath, { artifactMode: 'replay' })`, otherwise the export will omit the old spans +* the import must use `importSwingStore(exporter, dirPath, { artifactMode: 'replay' })`, otherwise the import will ignore the old spans + * subsequent `openSwingStore` calls must not use `keepTranscripts: false`, otherwise the new swingstore will prune historical spans they are replaced during `rolloverSpan`. ## Implementation Details From 8b3a6d499f9ef58c9f4110f3ca7f165ff65c37c3 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Thu, 5 Sep 2024 22:37:34 -0400 Subject: [PATCH 7/7] refactor(cosmic-swingset): Clarify keep{Snapshots,Transcripts} determination --- packages/cosmic-swingset/src/chain-main.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/cosmic-swingset/src/chain-main.js b/packages/cosmic-swingset/src/chain-main.js index cb56f8ec0ae..c1c3427e0d4 100644 --- a/packages/cosmic-swingset/src/chain-main.js +++ b/packages/cosmic-swingset/src/chain-main.js @@ -317,10 +317,12 @@ export default async function main(progname, args, { env, homedir, agcc }) { validateSwingsetConfig(swingsetConfig); const { slogfile, vatSnapshotRetention, vatTranscriptRetention } = swingsetConfig; - const keepSnapshots = - vatSnapshotRetention === 'debug' || - (!vatSnapshotRetention && ['1', 'true'].includes(XSNAP_KEEP_SNAPSHOTS)); - const keepTranscripts = vatTranscriptRetention === 'archival'; + const keepSnapshots = vatSnapshotRetention + ? vatSnapshotRetention !== 'operational' + : ['1', 'true'].includes(XSNAP_KEEP_SNAPSHOTS); + const keepTranscripts = vatTranscriptRetention + ? vatTranscriptRetention !== 'operational' + : false; // As a kludge, back-propagate selected configuration into environment variables. // eslint-disable-next-line dot-notation