diff --git a/golang/cosmos/app/app.go b/golang/cosmos/app/app.go index 9668dfd37eb..82db55ed54f 100644 --- a/golang/cosmos/app/app.go +++ b/golang/cosmos/app/app.go @@ -810,6 +810,42 @@ func NewAgoricApp( return app } +type swingStoreMigrationEventHandler struct { + swingStore sdk.KVStore +} + +func (eventHandler swingStoreMigrationEventHandler) OnExportStarted(height uint64, retrieveSwingStoreExport func() error) error { + return retrieveSwingStoreExport() +} + +func (eventHandler swingStoreMigrationEventHandler) OnExportRetrieved(provider swingsetkeeper.SwingStoreExportProvider) (err error) { + exportDataReader, err := provider.GetExportDataReader() + if err != nil { + return err + } + defer exportDataReader.Close() + + var hasExportData bool + + for { + entry, err := exportDataReader.Read() + if err == io.EOF { + break + } else if err != nil { + return err + } + hasExportData = true + if !entry.HasValue() { + return fmt.Errorf("no value for export data key %s", entry.Key()) + } + eventHandler.swingStore.Set([]byte(entry.Key()), []byte(entry.StringValue())) + } + if !hasExportData { + return fmt.Errorf("export data had no entries") + } + return nil +} + // upgrade11Handler performs standard upgrade actions plus custom actions for upgrade-11. func upgrade11Handler(app *GaiaApp, targetUpgrade string) func(sdk.Context, upgradetypes.Plan, module.VersionMap) (module.VersionMap, error) { return func(ctx sdk.Context, plan upgradetypes.Plan, fromVm module.VersionMap) (module.VersionMap, error) { @@ -817,7 +853,64 @@ func upgrade11Handler(app *GaiaApp, targetUpgrade string) func(sdk.Context, upgr // Record the plan to send to SwingSet app.upgradePlan = &plan - // TODO: Migrate x/vstorage swingStore to x/swingset SwingStore + // Perform swing-store migrations. We do this in the app upgrade handler + // since it involves multiple modules (x/vstorage and x/swingset) which + // don't strictly have a version change on their own. + + // We are at the begining of the upgrade block, so all stores are commited + // as of the end of the previous block + savedBlockHeight := uint64(ctx.BlockHeight() - 1) + + // First, repair swing-store metadata in case this node was previously + // initialized from a state-sync snapshot. This is done with a check on the + // block height to catch early any hangover related mismatch. + // Only entries related to missing historical metadata are imported, but we + // don't know what these look like here, so we provide it all. + getSwingStoreExportDataFromVstorage := func() (reader agorictypes.KVEntryReader, err error) { + return agorictypes.NewVstorageDataEntriesReader( + app.VstorageKeeper.ExportStorageFromPrefix(ctx, swingsetkeeper.StoragePathSwingStore), + ), nil + } + + // We're not restoring any artifact to swing-store, nor have any to provide + readNoArtifact := func() (artifact swingsettypes.SwingStoreArtifact, err error) { + return artifact, io.EOF + } + + err := app.SwingStoreExportsHandler.RestoreExport( + swingsetkeeper.SwingStoreExportProvider{ + BlockHeight: savedBlockHeight, + GetExportDataReader: getSwingStoreExportDataFromVstorage, + ReadNextArtifact: readNoArtifact, + }, + swingsetkeeper.SwingStoreRestoreOptions{ + ArtifactMode: swingsetkeeper.SwingStoreArtifactModeNone, + ExportDataMode: swingsetkeeper.SwingStoreExportDataModeRepairMetadata, + }, + ) + if err != nil { + return nil, err + } + + // Then migrate the swing-store shadow copy: + // 1. Remove the swing-store "export data" shadow-copy entries from vstorage. + // 2. Export swing-store "export-data" (as of the previous block) through a + // handler that writes every entry into the swingset module's new Store. + app.VstorageKeeper.RemoveEntriesWithPrefix(ctx, swingsetkeeper.StoragePathSwingStore) + err = app.SwingStoreExportsHandler.InitiateExport( + savedBlockHeight, + swingStoreMigrationEventHandler{swingStore: app.SwingSetKeeper.GetSwingStore(ctx)}, + swingsetkeeper.SwingStoreExportOptions{ + ArtifactMode: swingsetkeeper.SwingStoreArtifactModeNone, + ExportDataMode: swingsetkeeper.SwingStoreExportDataModeAll, + }, + ) + if err == nil { + err = swingsetkeeper.WaitUntilSwingStoreExportDone() + } + if err != nil { + return nil, err + } // Always run module migrations mvm, err := app.mm.RunMigrations(ctx, app.configurator, fromVm) diff --git a/golang/cosmos/x/swingset/keeper/extension_snapshotter.go b/golang/cosmos/x/swingset/keeper/extension_snapshotter.go index 8e4c1fc0f2e..0e73dc59970 100644 --- a/golang/cosmos/x/swingset/keeper/extension_snapshotter.go +++ b/golang/cosmos/x/swingset/keeper/extension_snapshotter.go @@ -125,8 +125,8 @@ func (snapshotter *ExtensionSnapshotter) InitiateSnapshot(height int64) error { blockHeight := uint64(height) return snapshotter.swingStoreExportsHandler.InitiateExport(blockHeight, snapshotter, SwingStoreExportOptions{ - ExportMode: SwingStoreExportModeCurrent, - IncludeExportData: false, + ArtifactMode: SwingStoreArtifactModeReplay, + ExportDataMode: SwingStoreExportDataModeSkip, }) } @@ -304,6 +304,6 @@ func (snapshotter *ExtensionSnapshotter) RestoreExtension(blockHeight uint64, fo return snapshotter.swingStoreExportsHandler.RestoreExport( SwingStoreExportProvider{BlockHeight: blockHeight, GetExportDataReader: getExportDataReader, ReadNextArtifact: readNextArtifact}, - SwingStoreRestoreOptions{IncludeHistorical: false}, + SwingStoreRestoreOptions{ArtifactMode: SwingStoreArtifactModeReplay, ExportDataMode: SwingStoreExportDataModeAll}, ) } diff --git a/golang/cosmos/x/swingset/keeper/swing_store_exports_handler.go b/golang/cosmos/x/swingset/keeper/swing_store_exports_handler.go index a0c34268102..564db5de630 100644 --- a/golang/cosmos/x/swingset/keeper/swing_store_exports_handler.go +++ b/golang/cosmos/x/swingset/keeper/swing_store_exports_handler.go @@ -72,7 +72,7 @@ import ( // - OnExportRetrieved reads the export using the provider. // // Restoring a swing-store export does not have similar non-blocking requirements. -// The component simply invokes swingStoreExportHandler.RestoreExport with a +// The component simply invokes swingStoreExportsHandler.RestoreExport with a // SwingStoreExportProvider representing the swing-store export to // be restored, and RestoreExport will consume it and block until the JS side // has completed the restore before returning. @@ -157,44 +157,81 @@ type swingStoreRestoreExportAction struct { Args [1]swingStoreImportOptions `json:"args"` } -// SwingStoreExportModeCurrent represents the minimal set of artifacts needed -// to operate a node. -const SwingStoreExportModeCurrent = "current" +const ( + // SwingStoreArtifactModeNone means that no artifacts are part of the + // export / import. + SwingStoreArtifactModeNone = "none" -// SwingStoreExportModeArchival represents the set of all artifacts needed to -// not lose any historical state. -const SwingStoreExportModeArchival = "archival" + // SwingStoreArtifactModeOperational represents the minimal set of artifacts + // needed to operate a node. + SwingStoreArtifactModeOperational = "operational" -// SwingStoreExportModeDebug represents the maximal set of artifacts available -// in the JS swing-store, including any kept around for debugging purposed only -// (like previous XS heap snapshots) -const SwingStoreExportModeDebug = "debug" + // SwingStoreArtifactModeReplay represents the set of artifacts needed to + // replay the current incarnation of every vat. + SwingStoreArtifactModeReplay = "replay" + + // SwingStoreArtifactModeArchival represents the set of all artifacts + // providing all available historical state. + SwingStoreArtifactModeArchival = "archival" + + // SwingStoreArtifactModeDebug represents the maximal set of artifacts + // available in the JS swing-store, including any kept around for debugging + // purposes only (like previous XS heap snapshots) + SwingStoreArtifactModeDebug = "debug" +) + +const ( + // SwingStoreExportDataModeSkip indicates "export data" should be excluded from + // an export. ArtifactMode cannot be "none" in this case. + SwingStoreExportDataModeSkip = "skip" + + // SwingStoreExportDataModeRepairMetadata indicates the "export data" should be + // used to repair the metadata of an existing swing-store for an import + // operation. ArtifactMode must be "none" in this case. + SwingStoreExportDataModeRepairMetadata = "repair-metadata" + + // SwingStoreExportDataModeAll indicates "export data" should be part of the + // export or import. For import, ArtifactMode cannot be "none". + SwingStoreExportDataModeAll = "all" +) // SwingStoreExportOptions are configurable options provided to the JS swing-store export type SwingStoreExportOptions struct { - // The export mode can be "current", "archival" or "debug" (SwingStoreExportMode* const) - // See packages/cosmic-swingset/src/export-kernel-db.js initiateSwingStoreExport and - // packages/swing-store/src/swingStore.js makeSwingStoreExporter - ExportMode string `json:"exportMode,omitempty"` - // A flag indicating whether "export data" should be part of the swing-store export - // If false, the resulting SwingStoreExportProvider's GetExportDataReader - // will return nil - IncludeExportData bool `json:"includeExportData,omitempty"` + // ArtifactMode controls the set of artifacts that should be included in the + // swing-store export. Any SwingStoreArtifactMode* const value can be used + // (None, Operational, Replay, Archival, Debug). + // See packages/cosmic-swingset/src/export-kernel-db.js initiateSwingStoreExport + ArtifactMode string `json:"artifactMode,omitempty"` + // ExportDataMode selects whether to include "export data" in the swing-store + // export or not. Use the value SwingStoreExportDataModeSkip or + // SwingStoreExportDataModeAll. If "skip", the reader returned by + // SwingStoreExportProvider's GetExportDataReader will be nil. + ExportDataMode string `json:"exportDataMode,omitempty"` } // SwingStoreRestoreOptions are configurable options provided to the JS swing-store import type SwingStoreRestoreOptions struct { - // A flag indicating whether the swing-store import should attempt to load - // all historical artifacts available from the export provider - IncludeHistorical bool `json:"includeHistorical,omitempty"` + // ArtifactMode controls the set of artifacts that should be restored in + // swing-store. Any SwingStoreArtifactMode* const value can be used + // (None, Operational, Replay, Archival, Debug). + // See packages/cosmic-swingset/src/import-kernel-db.js performStateSyncImport + ArtifactMode string `json:"artifactMode,omitempty"` + // ExportDataMode selects the purpose of the restore, to recreate a + // swing-store (SwingStoreExportDataModeAll), or just to import missing + // metadata (SwingStoreExportDataModeRepairMetadata). + // If RepairMetadata, ArtifactMode should be SwingStoreArtifactModeNone. + // If All, ArtifactMode must be at least SwingStoreArtifactModeOperational. + ExportDataMode string `json:"exportDataMode,omitempty"` } type swingStoreImportOptions struct { // ExportDir is the directory created by RestoreExport that JS swing-store // should import from. ExportDir string `json:"exportDir"` - // IncludeHistorical is a copy of SwingStoreRestoreOptions.IncludeHistorical - IncludeHistorical bool `json:"includeHistorical,omitempty"` + // ArtifactMode is a copy of SwingStoreRestoreOptions.ArtifactMode + ArtifactMode string `json:"artifactMode,omitempty"` + // ExportDataMode is a copy of SwingStoreRestoreOptions.ExportDataMode + ExportDataMode string `json:"exportDataMode,omitempty"` } var disallowedArtifactNameChar = regexp.MustCompile(`[^-_.a-zA-Z0-9]`) @@ -781,8 +818,9 @@ func (exportsHandler SwingStoreExportsHandler) RestoreExport(provider SwingStore BlockHeight: blockHeight, Request: restoreRequest, Args: [1]swingStoreImportOptions{{ - ExportDir: exportDir, - IncludeHistorical: restoreOptions.IncludeHistorical, + ExportDir: exportDir, + ArtifactMode: restoreOptions.ArtifactMode, + ExportDataMode: restoreOptions.ExportDataMode, }}, } diff --git a/golang/cosmos/x/vstorage/keeper/keeper.go b/golang/cosmos/x/vstorage/keeper/keeper.go index cc0e9d3298c..fb5a831fe8b 100644 --- a/golang/cosmos/x/vstorage/keeper/keeper.go +++ b/golang/cosmos/x/vstorage/keeper/keeper.go @@ -181,6 +181,54 @@ func (k Keeper) ImportStorage(ctx sdk.Context, entries []*types.DataEntry) { } } +func getEncodedKeysWithPrefixFromIterator(iterator sdk.Iterator, prefix string) [][]byte { + keys := make([][]byte, 0) + defer iterator.Close() + for ; iterator.Valid(); iterator.Next() { + key := iterator.Key() + path := types.EncodedKeyToPath(key) + if strings.HasPrefix(path, prefix) { + keys = append(keys, key) + } + } + return keys +} + +// RemoveEntriesWithPrefix removes all storage entries starting with the +// supplied pathPrefix, which may not be empty. +// It has the same effect as listing children of the prefix and removing each +// descendant recursively. +func (k Keeper) RemoveEntriesWithPrefix(ctx sdk.Context, pathPrefix string) { + store := ctx.KVStore(k.storeKey) + + if len(pathPrefix) == 0 { + panic("cannot remove all content") + } + if err := types.ValidatePath(pathPrefix); err != nil { + panic(err) + } + descendantPrefix := pathPrefix + types.PathSeparator + + // since vstorage encodes keys with a prefix indicating the number of path + // elements, we cannot use a simple prefix iterator. + // Instead we iterate over the whole vstorage content and check + // whether each entry matches the descendantPrefix. This choice assumes most + // entries will be deleted. An alternative implementation would be to + // recursively list all children under the descendantPrefix, and delete them. + + iterator := sdk.KVStorePrefixIterator(store, nil) + + keys := getEncodedKeysWithPrefixFromIterator(iterator, descendantPrefix) + + for _, key := range keys { + store.Delete(key) + } + + // Update the prefix entry itself with SetStorage, which will effectively + // delete it and all necessary ancestors. + k.SetStorage(ctx, agoric.NewKVEntryWithNoValue(pathPrefix)) +} + func (k Keeper) EmitChange(ctx sdk.Context, change *ProposedChange) { if change.NewValue == change.ValueFromLastBlock { // No change. diff --git a/golang/cosmos/x/vstorage/keeper/keeper_test.go b/golang/cosmos/x/vstorage/keeper/keeper_test.go index 120e707b64b..38fcdb7e7f6 100644 --- a/golang/cosmos/x/vstorage/keeper/keeper_test.go +++ b/golang/cosmos/x/vstorage/keeper/keeper_test.go @@ -185,7 +185,25 @@ func TestStorage(t *testing.T) { t.Errorf("got export %q, want %q", got, expectedKey2Export) } + keeper.RemoveEntriesWithPrefix(ctx, "key2.child2") + if keeper.HasEntry(ctx, "key2") { + t.Errorf("got leftover entries for key2 after removal") + } + expectedRemainingExport := []*types.DataEntry{ + {Path: "alpha2", Value: "value2"}, + {Path: "beta3", Value: "value3"}, + {Path: "inited", Value: ""}, + } + gotRemainingExport := keeper.ExportStorage(ctx) + if !reflect.DeepEqual(gotRemainingExport, expectedRemainingExport) { + t.Errorf("got remaining export %q, want %q", expectedRemainingExport, expectedRemainingExport) + } + keeper.ImportStorage(ctx, gotExport) + gotExport = keeper.ExportStorage(ctx) + if !reflect.DeepEqual(gotExport, expectedExport) { + t.Errorf("got export %q after import, want %q", gotExport, expectedExport) + } } func TestStorageNotify(t *testing.T) { diff --git a/packages/cosmic-swingset/src/export-kernel-db.js b/packages/cosmic-swingset/src/export-kernel-db.js index 3da83614a08..2878658fe12 100755 --- a/packages/cosmic-swingset/src/export-kernel-db.js +++ b/packages/cosmic-swingset/src/export-kernel-db.js @@ -23,30 +23,64 @@ import { makeProcessValue } from './helpers/process-value.js'; // with the golang SwingStoreExportsHandler in golang/cosmos/x/swingset/keeper/swing_store_exports_handler.go export const ExportManifestFileName = 'export-manifest.json'; -/** @typedef {'current' | 'archival' | 'debug'} SwingStoreExportMode */ +/** + * @typedef {'none' // No artifacts included + * | import("@agoric/swing-store").ArtifactMode + * } SwingStoreArtifactMode + */ /** - * @param {SwingStoreExportMode | undefined} exportMode + * @typedef {'skip' // Do not include any "export data" (artifacts only) + * | 'repair-metadata' // Add missing artifact metadata (import only) + * | 'all' // Include all export data, create new swing-store on import + * } SwingStoreExportDataMode + */ + +/** + * @param {SwingStoreArtifactMode | undefined} artifactMode * @returns {import("@agoric/swing-store").ArtifactMode} */ -const getArtifactModeFromExportMode = exportMode => { - switch (exportMode) { - case 'current': - case undefined: +export const getEffectiveArtifactMode = artifactMode => { + switch (artifactMode) { + case 'none': + case 'operational': return 'operational'; + case undefined: + case 'replay': + return 'replay'; case 'archival': - return 'archival'; case 'debug': - return 'debug'; + return artifactMode; default: - throw Fail`Invalid value ${q(exportMode)} for "export-mode"`; + throw Fail`Invalid value ${q(artifactMode)} for "artifact-mode"`; } }; +/** @type {(artifactMode: string | undefined) => asserts artifactMode is SwingStoreArtifactMode | undefined} */ +export const checkArtifactMode = getEffectiveArtifactMode; + /** - * @type {(exportMode: string | undefined) => asserts exportMode is SwingStoreExportMode} + * @param {string | undefined} mode + * @param {boolean} [isImport] + * @returns {asserts mode is SwingStoreExportDataMode | undefined} */ -const checkExportMode = getArtifactModeFromExportMode; +export const checkExportDataMode = (mode, isImport = false) => { + switch (mode) { + case 'skip': + case undefined: + break; + case 'all': + break; + case 'repair-metadata': { + if (isImport) { + break; + } + // Fall through + } + default: + throw Fail`Invalid value ${q(mode)} for "export-data-mode"`; + } +}; /** * A state-sync manifest is a representation of the information contained in a @@ -60,7 +94,7 @@ const checkExportMode = getArtifactModeFromExportMode; * * @typedef {object} StateSyncManifest * @property {number} blockHeight the block height corresponding to this export - * @property {SwingStoreExportMode} [mode] + * @property {SwingStoreArtifactMode} [artifactMode] * @property {string} [data] file name containing the swingStore "export data" * @property {Array<[artifactName: string, fileName: string]>} artifacts * List of swingStore export artifacts which can be validated by the export data @@ -79,8 +113,8 @@ const checkExportMode = getArtifactModeFromExportMode; * @property {string} stateDir the directory containing the SwingStore to export * @property {string} exportDir the directory in which to place the exported artifacts and manifest * @property {number} [blockHeight] block height to check for - * @property {SwingStoreExportMode} [exportMode] whether to include historical or debug artifacts in the export - * @property {boolean} [includeExportData] whether to include an artifact for the export data in the export + * @property {SwingStoreArtifactMode} [artifactMode] the level of artifacts to include in the export + * @property {SwingStoreExportDataMode} [exportDataMode] include a synthetic artifact for the export data in the export */ /** @@ -96,10 +130,12 @@ export const validateExporterOptions = options => { options.blockHeight == null || typeof options.blockHeight === 'number' || Fail`optional blockHeight option not a number`; - checkExportMode(options.exportMode); - options.includeExportData == null || - typeof options.includeExportData === 'boolean' || - Fail`optional includeExportData option not a boolean`; + checkArtifactMode(options.artifactMode); + checkExportDataMode(options.exportDataMode); + + options.includeExportData === undefined || + Fail`deprecated includeExportData option found`; + options.exportMode === undefined || Fail`deprecated exportMode option found`; }; /** @@ -113,7 +149,7 @@ export const validateExporterOptions = options => { * @returns {StateSyncExporter} */ export const initiateSwingStoreExport = ( - { stateDir, exportDir, blockHeight, exportMode, includeExportData }, + { stateDir, exportDir, blockHeight, artifactMode, exportDataMode }, { fs: { open, writeFile }, pathResolve, @@ -122,8 +158,7 @@ export const initiateSwingStoreExport = ( log = console.log, }, ) => { - const artifactMode = getArtifactModeFromExportMode(exportMode); - + const effectiveArtifactMode = getEffectiveArtifactMode(artifactMode); /** @type {number | undefined} */ let savedBlockHeight; @@ -143,7 +178,9 @@ export const initiateSwingStoreExport = ( const manifestFile = await open(manifestPath, 'wx'); cleanup.push(async () => manifestFile.close()); - const swingStoreExporter = makeExporter(stateDir, { artifactMode }); + const swingStoreExporter = makeExporter(stateDir, { + artifactMode: effectiveArtifactMode, + }); cleanup.push(async () => swingStoreExporter.close()); const { hostStorage } = openDB(stateDir); @@ -153,7 +190,9 @@ export const initiateSwingStoreExport = ( if (blockHeight) { blockHeight === savedBlockHeight || - Fail`DB at unexpected block height ${savedBlockHeight} (expected ${blockHeight})`; + Fail`DB at unexpected block height ${q(savedBlockHeight)} (expected ${q( + blockHeight, + )})`; } abortIfStopped(); @@ -163,11 +202,11 @@ export const initiateSwingStoreExport = ( /** @type {StateSyncManifest} */ const manifest = { blockHeight: savedBlockHeight, - mode: exportMode, + artifactMode: artifactMode || effectiveArtifactMode, artifacts: [], }; - if (includeExportData) { + if (exportDataMode === 'all') { log?.(`Writing Export Data`); const fileName = `export-data.jsonl`; // eslint-disable-next-line @jessie.js/no-nested-await @@ -183,14 +222,16 @@ export const initiateSwingStoreExport = ( } abortIfStopped(); - for await (const artifactName of swingStoreExporter.getArtifactNames()) { - abortIfStopped(); - log?.(`Writing artifact: ${artifactName}`); - const artifactData = swingStoreExporter.getArtifact(artifactName); - // Use artifactName as the file name as we trust swingStore to generate - // artifact names that are valid file names. - await writeFile(pathResolve(exportDir, artifactName), artifactData); - manifest.artifacts.push([artifactName, artifactName]); + if (artifactMode !== 'none') { + for await (const artifactName of swingStoreExporter.getArtifactNames()) { + abortIfStopped(); + log?.(`Writing artifact: ${artifactName}`); + const artifactData = swingStoreExporter.getArtifact(artifactName); + // Use artifactName as the file name as we trust swingStore to generate + // artifact names that are valid file names. + await writeFile(pathResolve(exportDir, artifactName), artifactData); + manifest.artifacts.push([artifactName, artifactName]); + } } await manifestFile.write(JSON.stringify(manifest, null, 2)); @@ -274,11 +315,22 @@ export const main = async ( /** @type {string} */ (processValue.getFlag('export-dir', '.')), ); - const includeExportData = processValue.getBoolean({ - flagName: 'include-export-data', - }); - const exportMode = processValue.getFlag('export-mode'); - checkExportMode(exportMode); + const artifactMode = /** @type {SwingStoreArtifactMode | undefined} */ ( + processValue.getFlag('artifact-mode') + ); + checkArtifactMode(artifactMode); + + const exportDataMode = processValue.getFlag('export-data-mode'); + checkExportDataMode(exportDataMode); + + if ( + processValue.getBoolean({ flagName: 'include-export-data' }) !== undefined + ) { + throw Fail`deprecated "include-export-data" options, use "export-data-mode" instead`; + } + if (processValue.getFlag('export-mode') !== undefined) { + throw Fail`deprecated "export-mode" options, use "artifact-mode" instead`; + } const checkBlockHeight = processValue.getInteger({ flagName: 'check-block-height', @@ -295,8 +347,8 @@ export const main = async ( stateDir, exportDir, blockHeight: checkBlockHeight, - exportMode, - includeExportData, + artifactMode, + exportDataMode, }, { fs, @@ -337,7 +389,7 @@ export const main = async ( * @returns {StateSyncExporter} */ export const spawnSwingStoreExport = ( - { stateDir, exportDir, blockHeight, exportMode, includeExportData }, + { stateDir, exportDir, blockHeight, artifactMode, exportDataMode }, { fork, verbose }, ) => { const args = ['--state-dir', stateDir, '--export-dir', exportDir]; @@ -346,12 +398,12 @@ export const spawnSwingStoreExport = ( args.push('--check-block-height', String(blockHeight)); } - if (exportMode) { - args.push('--export-mode', exportMode); + if (artifactMode) { + args.push('--artifact-mode', artifactMode); } - if (includeExportData) { - args.push('--include-export-data'); + if (exportDataMode) { + args.push('--export-data-mode', exportDataMode); } if (verbose) { @@ -403,7 +455,7 @@ export const spawnSwingStoreExport = ( } default: { // @ts-expect-error exhaustive check - Fail`Unexpected ${msg.type} message`; + Fail`Unexpected ${q(msg.type)} message`; } } }; diff --git a/packages/cosmic-swingset/src/import-kernel-db.js b/packages/cosmic-swingset/src/import-kernel-db.js index bed9d42cfb8..d41e6309cd1 100755 --- a/packages/cosmic-swingset/src/import-kernel-db.js +++ b/packages/cosmic-swingset/src/import-kernel-db.js @@ -12,19 +12,24 @@ import fsPromisesPower from 'fs/promises'; import pathPower from 'path'; import BufferLineTransform from '@agoric/internal/src/node/buffer-line-transform.js'; -import { Fail } from '@agoric/assert'; -import { importSwingStore } from '@agoric/swing-store'; +import { Fail, q } from '@agoric/assert'; +import { importSwingStore, openSwingStore } from '@agoric/swing-store'; import { isEntrypoint } from './helpers/is-entrypoint.js'; import { makeProcessValue } from './helpers/process-value.js'; -import { ExportManifestFileName } from './export-kernel-db.js'; +import { + ExportManifestFileName, + checkExportDataMode, + checkArtifactMode, +} from './export-kernel-db.js'; /** * @typedef {object} StateSyncImporterOptions * @property {string} stateDir the directory containing the SwingStore to export * @property {string} exportDir the directory where to place the exported artifacts and manifest * @property {number} [blockHeight] block height to check for - * @property {boolean} [includeHistorical] whether to include historical artifacts in the export + * @property {import('./export-kernel-db.js').SwingStoreExportDataMode} [exportDataMode] how to handle export data + * @property {import('./export-kernel-db.js').SwingStoreArtifactMode} [artifactMode] the level of historical artifacts to import */ /** @@ -40,9 +45,57 @@ export const validateImporterOptions = options => { options.blockHeight == null || typeof options.blockHeight === 'number' || Fail`optional blockHeight option not a number`; - options.includeHistorical == null || - typeof options.includeHistorical === 'boolean' || - Fail`optional includeHistorical option not a boolean`; + checkExportDataMode(options.exportDataMode, true); + checkArtifactMode(options.artifactMode); + options.includeHistorical === undefined || + Fail`deprecated includeHistorical option found`; +}; + +/** + * @param {Pick} options + * @param {Readonly} manifest + * @returns {import('@agoric/swing-store').ImportSwingStoreOptions} + */ +const checkAndGetImportSwingStoreOptions = (options, manifest) => { + typeof manifest.blockHeight === 'number' || + Fail`Cannot restore snapshot without block height`; + + manifest.data || Fail`State-sync manifest missing export data`; + + const { artifactMode = manifest.artifactMode || 'replay' } = options; + + if (artifactMode === 'none') { + throw Fail`Cannot import "export data" without at least "operational" artifacts`; + } + + manifest.artifacts?.length || + Fail`State-sync manifest missing required artifacts`; + + switch (artifactMode) { + case 'debug': + // eslint-disable-next-line no-fallthrough + case 'operational': + if (manifest.artifactMode === 'operational') break; + // eslint-disable-next-line no-fallthrough + case 'replay': + if (manifest.artifactMode === 'replay') break; + // eslint-disable-next-line no-fallthrough + case 'archival': + if (manifest.artifactMode === 'archival') break; + if ( + manifest.artifactMode === undefined || + manifest.artifactMode === 'debug' + ) { + // assume the export has sufficient data + break; + } + throw Fail`State-sync manifest has insufficient artifacts: requested import artifact mode: ${q( + artifactMode, + )}, manifest has ${q(manifest.artifactMode)} artifacts`; + default: + throw Fail`Unexpected artifactMode ${q(artifactMode)}`; + } + return { artifactMode }; }; /** @@ -51,15 +104,17 @@ export const validateImporterOptions = options => { * @param {Pick & Pick} powers.fs * @param {import('path')['resolve']} powers.pathResolve * @param {typeof import('@agoric/swing-store')['importSwingStore']} [powers.importSwingStore] + * @param {typeof import('@agoric/swing-store')['openSwingStore']} [powers.openSwingStore] * @param {null | ((...args: any[]) => void)} [powers.log] * @returns {Promise} */ export const performStateSyncImport = async ( - { stateDir, exportDir, blockHeight, includeHistorical }, + { stateDir, exportDir, blockHeight, exportDataMode = 'all', artifactMode }, { fs: { createReadStream, readFile }, pathResolve, importSwingStore: importDB = importSwingStore, + openSwingStore: openDB = openSwingStore, log = console.log, }, ) => { @@ -67,7 +122,9 @@ export const performStateSyncImport = async ( const safeExportFileResolve = allegedRelativeFilename => { const resolvedPath = pathResolve(exportDir, allegedRelativeFilename); resolvedPath.startsWith(exportDir) || - Fail`Exported file ${allegedRelativeFilename} must be in export dir ${exportDir}`; + Fail`Exported file ${q( + allegedRelativeFilename, + )} must be in export dir ${q(exportDir)}`; return resolvedPath; }; @@ -78,26 +135,12 @@ export const performStateSyncImport = async ( ); if (blockHeight !== undefined && manifest.blockHeight !== blockHeight) { - Fail`State-sync manifest for unexpected block height ${manifest.blockHeight} (expected ${blockHeight})`; - } - - if (!manifest.data) { - throw Fail`State-sync manifest missing export data`; - } - - if (!manifest.artifacts) { - throw Fail`State-sync manifest missing required artifacts`; + Fail`State-sync manifest for unexpected block height ${q( + manifest.blockHeight, + )} (expected ${q(blockHeight)})`; } - const artifacts = harden(Object.fromEntries(manifest.artifacts)); - - if ( - includeHistorical && - manifest.mode !== 'archival' && - manifest.mode !== 'debug' - ) { - throw Fail`State-sync manifest missing historical artifacts`; - } + const artifacts = harden(Object.fromEntries(manifest.artifacts || [])); // Represent the data in `exportDir` as a SwingSetExporter object. /** @type {import('@agoric/swing-store').SwingStoreExporter} */ @@ -128,7 +171,7 @@ export const performStateSyncImport = async ( log?.(`importing artifact ${name}`); const fileName = artifacts[name]; if (!fileName) { - Fail`invalid artifact ${name}`; + Fail`invalid artifact ${q(name)}`; } const stream = createReadStream(safeExportFileResolve(fileName)); yield* stream; @@ -141,17 +184,47 @@ export const performStateSyncImport = async ( }, }); - const artifactMode = includeHistorical - ? 'debug' // for now don't enforce completeness but allow importing all provided artifacts - : 'operational'; + if (exportDataMode === 'all') { + const importOptions = checkAndGetImportSwingStoreOptions( + { artifactMode, exportDataMode }, + manifest, + ); + + const swingstore = await importDB(exporter, stateDir, importOptions); + + const { hostStorage } = swingstore; + + hostStorage.kvStore.set('host.height', String(manifest.blockHeight)); + await hostStorage.commit(); + await hostStorage.close(); + } else if (exportDataMode === 'repair-metadata') { + blockHeight !== 0 || Fail`repair metadata requires a block height`; + + manifest.data || Fail`State-sync manifest missing export data`; + + artifactMode === 'none' || + Fail`Cannot restore artifacts while repairing metadata`; + + const { hostStorage } = openDB(stateDir); - const swingstore = await importDB(exporter, stateDir, { artifactMode }); + const savedBlockHeight = + Number(hostStorage.kvStore.get('host.height')) || 0; - const { hostStorage } = swingstore; + if (blockHeight !== savedBlockHeight) { + throw Fail`block height doesn't match. requested=${q( + blockHeight, + )}, current=${q(savedBlockHeight)}`; + } - hostStorage.kvStore.set('host.height', String(manifest.blockHeight)); - await hostStorage.commit(); - await hostStorage.close(); + await hostStorage.repairMetadata(exporter); + + await hostStorage.commit(); + await hostStorage.close(); + } else if (exportDataMode === 'skip') { + throw Fail`Repopulation of artifacts not yet supported`; + } else { + throw Fail`Unknown export-data-mode ${exportDataMode}`; + } }; /** @@ -186,9 +259,20 @@ export const main = async ( /** @type {string} */ (processValue.getFlag('export-dir', '.')), ); - const includeHistorical = processValue.getBoolean({ - flagName: 'include-historical', - }); + const artifactMode = + /** @type {import('./export-kernel-db.js').SwingStoreArtifactMode | undefined} */ ( + processValue.getFlag('artifact-mode') + ); + checkArtifactMode(artifactMode); + + const exportDataMode = processValue.getFlag('export-data-mode'); + checkExportDataMode(exportDataMode, true); + + if ( + processValue.getBoolean({ flagName: 'include-historical' }) !== undefined + ) { + throw Fail`deprecated "include-historical" options, use "artifact-mode" instead`; + } const checkBlockHeight = processValue.getInteger({ flagName: 'check-block-height', @@ -203,7 +287,8 @@ export const main = async ( stateDir, exportDir, blockHeight: checkBlockHeight, - includeHistorical, + artifactMode, + exportDataMode, }, { fs, diff --git a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-10/actions.sh b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-10/actions.sh index 0aa7e992a95..cc53553a99b 100644 --- a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-10/actions.sh +++ b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-10/actions.sh @@ -176,12 +176,24 @@ agops perf satisfaction --from "$USER2ADDR" --executeOffer "$OFFER" --keyring-ba # replicate state-sync of node # this will cause the swing-store to prune some data +# we will save the pruned artifact for later killAgd EXPORT_DIR=$(mktemp -t -d swing-store-export-upgrade-10-XXX) make_swing_store_snapshot $EXPORT_DIR || fail "Couldn't make swing-store snapshot" test_val "$(compare_swing_store_export_data $EXPORT_DIR)" "match" "swing-store export data" +EXPORT_DIR_ALL_ARTIFACTS=$(mktemp -t -d swing-store-export-upgrade-10-all-artifacts-XXX) +make_swing_store_snapshot $EXPORT_DIR_ALL_ARTIFACTS --export-mode archival || fail "Couldn't make swing-store snapshot for historical artifacts" restore_swing_store_snapshot $EXPORT_DIR || fail "Couldn't restore swing-store snapshot" +( + cd $EXPORT_DIR_ALL_ARTIFACTS + mkdir $HOME/.agoric/data/agoric/swing-store-historical-artifacts + for i in *; do + [ -f $EXPORT_DIR/$i ] && continue + mv $i $HOME/.agoric/data/agoric/swing-store-historical-artifacts/ + done +) rm -rf $EXPORT_DIR +rm -rf $EXPORT_DIR_ALL_ARTIFACTS startAgd # # TODO fully test bidding diff --git a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-10/env_setup.sh b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-10/env_setup.sh index ad484864d3a..22d6425887f 100644 --- a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-10/env_setup.sh +++ b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-10/env_setup.sh @@ -109,7 +109,8 @@ submitDeliverInbound() { make_swing_store_snapshot() {( set -euo pipefail EXPORT_DIR="$1" - /usr/src/agoric-sdk/packages/cosmic-swingset/src/export-kernel-db.js --home "$HOME/.agoric" --export-dir "$EXPORT_DIR" --verbose --include-export-data + shift + /usr/src/agoric-sdk/packages/cosmic-swingset/src/export-kernel-db.js --home "$HOME/.agoric" --export-dir "$EXPORT_DIR" --verbose --include-export-data "$@" EXPORT_MANIFEST_FILE="$EXPORT_DIR/export-manifest.json" EXPORT_DATA_FILE="$EXPORT_DIR/$(cat "$EXPORT_MANIFEST_FILE" | jq -r .data)" diff --git a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/actions.sh b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/actions.sh index 53a9292d65a..f29d64f6e65 100644 --- a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/actions.sh +++ b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/actions.sh @@ -2,4 +2,22 @@ . ./upgrade-test-scripts/env_setup.sh -# Core-eval contract upgrade +# Enable debugging +set -x + +# CWD is agoric-sdk +upgrade11=./upgrade-test-scripts/agoric-upgrade-11 + +# hacky restore of pruned artifacts +killAgd +EXPORT_DIR=$(mktemp -t -d swing-store-export-upgrade-11-XXX) +make_swing_store_snapshot $EXPORT_DIR --artifact-mode debug || fail "Couldn't make swing-store snapshot" +test_val "$(compare_swing_store_export_data $EXPORT_DIR)" "match" "swing-store export data" +HISTORICAL_ARTIFACTS="$(cd $HOME/.agoric/data/agoric/swing-store-historical-artifacts/; for i in *; do echo -n "[\"$i\",\"$i\"],"; done)" +mv -n $HOME/.agoric/data/agoric/swing-store-historical-artifacts/* $EXPORT_DIR || fail "some historical artifacts not pruned" +mv $EXPORT_DIR/export-manifest.json $EXPORT_DIR/export-manifest-original.json +cat $EXPORT_DIR/export-manifest-original.json | jq -r ".artifacts = .artifacts + [${HISTORICAL_ARTIFACTS%%,}] | del(.artifactMode)" > $EXPORT_DIR/export-manifest.json +restore_swing_store_snapshot $EXPORT_DIR || fail "Couldn't restore swing-store snapshot" +rmdir $HOME/.agoric/data/agoric/swing-store-historical-artifacts +rm -rf $EXPORT_DIR +startAgd diff --git a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/env_setup.sh b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/env_setup.sh index 11e8e98ba3f..0faaf3a66b5 100644 --- a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/env_setup.sh +++ b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/env_setup.sh @@ -83,4 +83,65 @@ pushPriceOnce () { else echo "ERROR: pushPrice failed (using $nextOracle)" fi -} \ No newline at end of file +} + +export_genesis() { + HEIGHT_ARG= + + if [ -n "$1" ]; then + HEIGHT_ARG="--height $1" + shift + fi + + agd export $HEIGHT_ARG "$@" +} + +make_swing_store_snapshot() {( set -euo pipefail + EXPORT_DIR="$1" + shift + /usr/src/agoric-sdk/packages/cosmic-swingset/src/export-kernel-db.js --home "$HOME/.agoric" --export-dir "$EXPORT_DIR" --verbose --artifact-mode replay --export-data-mode all "$@" + + EXPORT_MANIFEST_FILE="$EXPORT_DIR/export-manifest.json" + EXPORT_DATA_FILE="$EXPORT_DIR/$(cat "$EXPORT_MANIFEST_FILE" | jq -r .data)" + EXPORT_DATA_UNTRUSTED_FILE="${EXPORT_DATA_FILE%.*}-untrusted.jsonl" + EXPORT_HEIGHT=$(cat "$EXPORT_MANIFEST_FILE" | jq -r .blockHeight) + EXPORT_MANIFEST="$(cat $EXPORT_MANIFEST_FILE)" + + mv "$EXPORT_DATA_FILE" "$EXPORT_DATA_UNTRUSTED_FILE" + export_genesis $EXPORT_HEIGHT | jq -cr '.app_state.swingset.swing_store_export_data[] | [.key,.value]' > "$EXPORT_DATA_FILE" + + jq -n "$EXPORT_MANIFEST | .untrustedData=\"$(basename -- "$EXPORT_DATA_UNTRUSTED_FILE")\"" > "$EXPORT_MANIFEST_FILE" + + echo "Successful swing-store export for block $EXPORT_HEIGHT" +)} + +restore_swing_store_snapshot() {( set -euo pipefail + rm -f $HOME/.agoric/data/agoric/swingstore.sqlite + EXPORT_DIR="$1" + shift + + /usr/src/agoric-sdk/packages/cosmic-swingset/src/import-kernel-db.js --home "$HOME/.agoric" --export-dir "$EXPORT_DIR" --verbose --artifact-mode replay --export-data-mode all "$@" +)} + +compare_swing_store_export_data() { + EXPORT_DIR="$1" + EXPORT_MANIFEST_FILE="$EXPORT_DIR/export-manifest.json" + EXPORT_DATA_FILE="$(cat "$EXPORT_MANIFEST_FILE" | jq -r .data)" + EXPORT_DATA_UNTRUSTED_FILE="$(cat "$EXPORT_MANIFEST_FILE" | jq -r .untrustedData)" + + if [ -z "$EXPORT_DATA_FILE" ]; then + echo "missing-export-data" + return + fi + + if [ -z "$EXPORT_DATA_UNTRUSTED_FILE" ]; then + echo "missing-untrusted-export-data" + return + fi + + diff <(cat "$EXPORT_DIR/$EXPORT_DATA_FILE" | sort) <(cat "$EXPORT_DIR/$EXPORT_DATA_UNTRUSTED_FILE" | sort) >&2 && { + echo "match" + } || { + echo "mismatch" + } +} diff --git a/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/test.sh b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/test.sh new file mode 100755 index 00000000000..35b279e25cf --- /dev/null +++ b/packages/deployment/upgrade-test/upgrade-test-scripts/agoric-upgrade-11/test.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +. ./upgrade-test-scripts/env_setup.sh + +echo Wait for actions to settle +waitForBlock 2 + +# CWD is agoric-sdk +upgrade11=./upgrade-test-scripts/agoric-upgrade-11 + +# verify swing-store export-data is consistent +killAgd +EXPORT_DIR=$(mktemp -t -d swing-store-export-upgrade-11-XXX) +make_swing_store_snapshot $EXPORT_DIR --artifact-mode none || fail "Couldn't make swing-store snapshot" +test_val "$(compare_swing_store_export_data $EXPORT_DIR)" "match" "swing-store consistent state-sync" +rm -rf $EXPORT_DIR +startAgd