From c05be7678ba6d0631e803c76d6d2699cdbed22fa Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Wed, 27 Mar 2024 10:43:15 -0700 Subject: [PATCH] fix(swingset): use "dirt" to schedule vat reap/bringOutYourDead `dispatch.bringOutYourDead()`, aka "reap", triggers garbage collection inside a vat, and gives it a chance to drop imported c-list vrefs that are no longer referenced by anything inside the vat. Previously, each vat has a configurable parameter named `reapInterval`, which defaults to a kernel-wide `defaultReapInterval` (but can be set separately for each vat). This defaults to 1, mainly for unit testing, but real applications set it to something like 1000. This caused BOYD to happen once every 1000 deliveries, plus an extra BOYD just before we save an XS heap-state snapshot. This commit switches to a "dirt"-based BOYD scheduler, wherein we consider the vat to get more and more dirty as it does work, and eventually it reaches a `reapDirtThreshold` that triggers the BOYD (which resets the dirt counter). We continue to track `dirt.deliveries` as before, with the same defaults. But we add a new `dirt.gcKrefs` counter, which is incremented by the krefs we submit to the vat in GC deliveries. For example, calling `dispatch.dropImports([kref1, kref2])` would increase `dirt.gcKrefs` by two. The `reapDirtThreshold.gcKrefs` limit defaults to 20. For normal use patterns, this will trigger a BOYD after ten krefs have been dropped and retired. We choose this value to allow the #8928 slow vat termination process to trigger BOYD frequently enough to keep the BOYD cranks small: since these will be happening constantly (in the "background"), we don't want them to take more than 500ms or so. Given the current size of the large vats that #8928 seeks to terminate, 10 krefs seems like a reasonable limit. And of course we don't want to perform too many BOYDs, so `gcKrefs: 20` is about the smallest threshold we'd want to use. External APIs continue to accept `reapInterval`, and now also accept `reapGCKrefs`, and `neverReap` (a boolean which inhibits all BOYD, even new forms of dirt added in the future). * kernel config record * takes `config.defaultReapInterval` and `defaultReapGCKrefs` * takes `vat.NAME.creationOptions.reapInterval` and `.reapGCKrefs` and `.neverReap` * `controller.changeKernelOptions()` still takes `defaultReapInterval` but now also accepts `defaultReapGCKrefs` The APIs available to userspace code (through `vatAdminSvc`) are unchanged (partially due to upgrade/backwards-compatibility limitations), and continue to only support setting `reapInterval`. Internally, this just modifies `reapDirtThreshold.deliveries`. * `E(vatAdminSvc).createVat(bcap, { reapInterval })` * `E(adminNode).upgrade(bcap, { reapInterval })` * `E(adminNode).changeOptions({ reapInterval })` Internally, the kernel-wide state records `defaultReapDirtThreshold` instead of `defaultReapInterval`, and each vat records `.reapDirtThreshold` in their `vNN.options` key instead of `vNN.reapInterval`. The vat-level records override the kernel-wide values. The current dirt level is recorded in `vNN.reapDirt`. The kernel will automatically upgrade both the kernel-wide and the per-vat state upon the first reboot with the new kernel code. The old `reapCountdown` value is used to initialize the vat's `reapDirt.deliveries` counter, so the upgrade shouldn't disrupt the existing schedule. Vats which used `reapInterval = 'never'` (eg comms) will get a `reapDirtThreshold.never = true`, so they continue to inhibit BOYD. Any per-vat settings that match the kernel-wide settings are removed, allowing the kernel values to take precedence (as well as changes to the kernel-wide values; i.e. the per-vat settings are not sticky). We do not track dirt when the corresponding threshold is 'never', or if `neverReap` is true, to avoid incrementing the comms dirt counters forever. This design leaves room for adding `.computrons` to the dirt record, as well as tracking a separate `snapshotDirt` counter (to trigger XS heap snapshots, ala #6786). We add `reapDirtThreshold.computrons`, but do not yet expose an API to set it. Future work includes: * upgrade vat-vat-admin to let userspace set `reapDirtThreshold` New tests were added to exercise the upgrade process, and other tests were updated to match the new internal initialization pattern. We now reset the dirt counter upon any BOYD, so this also happens to help with #8665 (doing a `reapAllVats()` resets the delivery counters, so future BOYDs will be delayed, which is what we want). But we should still change `controller.reapAllVats()` to avoid BOYDs on vats which haven't received any deliveries. closes #8980 --- .../src/controller/initializeKernel.js | 37 ++- .../src/controller/initializeSwingset.js | 2 +- .../src/controller/upgradeSwingset.js | 193 +++++++++++++++ packages/SwingSet/src/index.js | 2 +- packages/SwingSet/src/kernel/kernel.js | 102 ++++++-- .../SwingSet/src/kernel/state/kernelKeeper.js | 149 +++++++++--- .../SwingSet/src/kernel/state/vatKeeper.js | 124 +++++++--- .../src/kernel/vat-loader/manager-factory.js | 1 - .../src/kernel/vat-loader/vat-loader.js | 2 +- packages/SwingSet/src/lib/recordVatOptions.js | 18 +- packages/SwingSet/src/types-external.js | 32 +-- packages/SwingSet/src/types-internal.js | 54 +++- .../change-parameters.test.js | 31 ++- packages/SwingSet/test/clist.test.js | 8 +- packages/SwingSet/test/controller.test.js | 14 ++ packages/SwingSet/test/kernel.test.js | 194 ++++++++++++++- .../SwingSet/test/snapshots/state.test.js.md | 4 +- .../test/snapshots/state.test.js.snap | Bin 276 -> 276 bytes packages/SwingSet/test/state.test.js | 230 +++++++++++++++++- .../SwingSet/test/upgrade-swingset.test.js | 202 +++++++++++++++ packages/SwingSet/test/vat-admin/bootstrap.js | 8 + .../test/vat-admin/create-vat.test.js | 37 +++ 22 files changed, 1306 insertions(+), 138 deletions(-) create mode 100644 packages/SwingSet/src/controller/upgradeSwingset.js create mode 100644 packages/SwingSet/test/upgrade-swingset.test.js diff --git a/packages/SwingSet/src/controller/initializeKernel.js b/packages/SwingSet/src/controller/initializeKernel.js index 3378a28c39f..3ffe291ada3 100644 --- a/packages/SwingSet/src/controller/initializeKernel.js +++ b/packages/SwingSet/src/controller/initializeKernel.js @@ -9,14 +9,32 @@ import { insistVatID } from '../lib/id.js'; import { makeVatSlot } from '../lib/parseVatSlots.js'; import { insistStorageAPI } from '../lib/storageAPI.js'; import { makeVatOptionRecorder } from '../lib/recordVatOptions.js'; -import makeKernelKeeper from '../kernel/state/kernelKeeper.js'; +import makeKernelKeeper, { + DEFAULT_DELIVERIES_PER_BOYD, + DEFAULT_GC_KREFS_PER_BOYD, +} from '../kernel/state/kernelKeeper.js'; import { exportRootObject } from '../kernel/kernel.js'; import { makeKernelQueueHandler } from '../kernel/kernelQueue.js'; +/** + * @typedef { import('../types-external.js').SwingSetKernelConfig } SwingSetKernelConfig + * @typedef { import('../types-external.js').SwingStoreKernelStorage } SwingStoreKernelStorage + * @typedef { import('../types-internal.js').InternalKernelOptions } InternalKernelOptions + * @typedef { import('../types-internal.js').ReapDirtThreshold } ReapDirtThreshold + */ + function makeVatRootObjectSlot() { return makeVatSlot('object', true, 0); } +/* + * @param {SwingSetKernelConfig} config + * @param {SwingStoreKernelStorage} kernelStorage + * @param {*} [options] + * @returns {Promise} KPID of the bootstrap message + * result promise + */ + export async function initializeKernel(config, kernelStorage, options = {}) { const { verbose = false, @@ -25,6 +43,9 @@ export async function initializeKernel(config, kernelStorage, options = {}) { const logStartup = verbose ? console.debug : () => 0; insistStorageAPI(kernelStorage.kvStore); + const CURRENT_VERSION = 1; + kernelStorage.kvStore.set('version', `${CURRENT_VERSION}`); + const kernelSlog = null; const kernelKeeper = makeKernelKeeper(kernelStorage, kernelSlog); const optionRecorder = makeVatOptionRecorder(kernelKeeper, bundleHandler); @@ -33,14 +54,22 @@ export async function initializeKernel(config, kernelStorage, options = {}) { assert(!wasInitialized); const { defaultManagerType, - defaultReapInterval, + defaultReapInterval = DEFAULT_DELIVERIES_PER_BOYD, + defaultReapGCKrefs = DEFAULT_GC_KREFS_PER_BOYD, relaxDurabilityRules, snapshotInitial, snapshotInterval, } = config; + /** @type { ReapDirtThreshold } */ + const defaultReapDirtThreshold = { + deliveries: defaultReapInterval, + gcKrefs: defaultReapGCKrefs, + computrons: 'never', // TODO no knob? + }; + /** @type { InternalKernelOptions } */ const kernelOptions = { defaultManagerType, - defaultReapInterval, + defaultReapDirtThreshold, relaxDurabilityRules, snapshotInitial, snapshotInterval, @@ -86,6 +115,8 @@ export async function initializeKernel(config, kernelStorage, options = {}) { 'useTranscript', 'critical', 'reapInterval', + 'reapGCKrefs', + 'neverReap', 'nodeOptions', ]); const vatID = kernelKeeper.allocateVatIDForNameIfNeeded(name); diff --git a/packages/SwingSet/src/controller/initializeSwingset.js b/packages/SwingSet/src/controller/initializeSwingset.js index e3f48c3950a..8f99f58899d 100644 --- a/packages/SwingSet/src/controller/initializeSwingset.js +++ b/packages/SwingSet/src/controller/initializeSwingset.js @@ -397,7 +397,7 @@ export async function initializeSwingset( enableSetup: true, managerType: 'local', useTranscript: false, - reapInterval: 'never', + neverReap: true, }, }; } diff --git a/packages/SwingSet/src/controller/upgradeSwingset.js b/packages/SwingSet/src/controller/upgradeSwingset.js new file mode 100644 index 00000000000..3c194b5ffa5 --- /dev/null +++ b/packages/SwingSet/src/controller/upgradeSwingset.js @@ -0,0 +1,193 @@ +import { + DEFAULT_REAP_DIRT_THRESHOLD_KEY, + DEFAULT_GC_KREFS_PER_BOYD, + getAllDynamicVats, + getAllStaticVats, +} from '../kernel/state/kernelKeeper.js'; + +const upgradeVatV0toV1 = (kvStore, defaultReapDirtThreshold, vatID) => { + // This is called, once per vat, when upgradeSwingset migrates from + // v0 to v1 + + // schema v0: + // Each vat has a `vNN.reapInterval` and `vNN.reapCountdown`. + // vNN.options has a `.reapInterval` property (however it was not + // updated by processChangeVatOptions). Either all are numbers, or + // all are 'never'. + + const oldReapIntervalKey = `${vatID}.reapInterval`; + const oldReapCountdownKey = `${vatID}.reapCountdown`; + const vatOptionsKey = `${vatID}.options`; + + // schema v1: + // Each vat has a `vNN.reapDirt`, and vNN.options has a + // `.reapDirtThreshold` property (which overrides kernel-wide + // `defaultReapDirtThreshold`) + + const reapDirtKey = `${vatID}.reapDirt`; + + assert(kvStore.has(oldReapIntervalKey), oldReapIntervalKey); + assert(kvStore.has(oldReapCountdownKey), oldReapCountdownKey); + assert(!kvStore.has(reapDirtKey), reapDirtKey); + + // initialize or upgrade state + const reapDirt = {}; // all missing keys are treated as zero + const threshold = {}; + + const reapIntervalString = kvStore.get(oldReapIntervalKey); + assert(reapIntervalString !== undefined); + const reapCountdownString = kvStore.get(oldReapCountdownKey); + assert(reapCountdownString !== undefined); + const intervalIsNever = reapIntervalString === 'never'; + const countdownIsNever = reapCountdownString === 'never'; + assert( + (intervalIsNever && countdownIsNever) || + (!intervalIsNever && !countdownIsNever), + `reapInterval=${reapIntervalString}, reapCountdown=${reapCountdownString}`, + ); + + if (!intervalIsNever && !countdownIsNever) { + // deduce delivery count from old countdown values + const reapInterval = Number.parseInt(reapIntervalString, 10); + const reapCountdown = Number.parseInt(reapCountdownString, 10); + const deliveries = reapInterval - reapCountdown; + reapDirt.deliveries = Math.max(deliveries, 0); // just in case + if (reapInterval !== defaultReapDirtThreshold.deliveries) { + threshold.deliveries = reapInterval; + } + } + + // old vats that were never reaped (eg comms) used + // reapInterval='never', so respect that and set the other + // threshold values to never as well + if (intervalIsNever) { + threshold.never = true; + } + kvStore.delete(oldReapIntervalKey); + kvStore.delete(oldReapCountdownKey); + kvStore.set(reapDirtKey, JSON.stringify(reapDirt)); + + // remove .reapInterval from options, replace with .reapDirtThreshold + const options = JSON.parse(kvStore.get(vatOptionsKey)); + delete options.reapInterval; + options.reapDirtThreshold = threshold; + kvStore.set(vatOptionsKey, JSON.stringify(options)); +}; + +/** + * (maybe) upgrade the kernel state to the current schema + * + * This function is responsible for bringing the kernel's portion of + * swing-store (kernelStorage) up to the current version. The host app + * must call this each time it launches with a new version of + * swingset, before using makeSwingsetController() to build the + * kernel/controller (which will throw an error if handed an old + * database). It is ok to call it only on those reboots, but it is + * also safe to call on every reboot, because upgradeSwingset() is a + * no-op if the DB is already up-to-date. + * + * If an upgrade is needed, this function will modify the DB state, so + * the host app must be prepared for export-data callbacks being + * called during the upgrade, and it is responsible for doing a + * `hostStorage.commit()` afterwards. + * + * @param { SwingStoreKernelStorage } kernelStorage + * @returns { boolean } true if any changes were made + */ +export const upgradeSwingset = kernelStorage => { + const { kvStore } = kernelStorage; + let modified = false; + let vstring = kvStore.get('version'); + if (vstring === undefined) { + vstring = '0'; + } + let version = Number(vstring); + + /** + * @param {string} key + * @returns {string} + */ + function getRequired(key) { + if (!kvStore.has(key)) { + throw Error(`storage lacks required key ${key}`); + } + // @ts-expect-error already checked .has() + return kvStore.get(key); + } + + // kernelKeeper.js has a large comment that defines our current + // kvStore schema, with a section describing the deltas. The upgrade + // steps applied here must match. + + // schema v0: + // The kernel overall has `kernel.defaultReapInterval`. + // Each vat has a `vNN.reapInterval` and `vNN.reapCountdown`. + // vNN.options has a `.reapInterval` property (however it was not + // updated by processChangeVatOptions, so do not rely upon its + // value). Either all are numbers, or all are 'never'. + + if (version < 1) { + // schema v1: + // The kernel overall has `kernel.defaultReapDirtThreshold`. + // Each vat has a `vNN.reapDirt`, and vNN.options has a + // `.reapDirtThreshold` property + + // So: + // * replace `kernel.defaultReapInterval` with + // `kernel.defaultReapDirtThreshold` + // * replace vat's `vNN.reapInterval`/`vNN.reapCountdown` with + // `vNN.reapDirt` and a `vNN.reapDirtThreshold` in `vNN.options` + // * then do per-vat upgrades with upgradeVatV0toV1 + + // upgrade from old kernel.defaultReapInterval + + const oldDefaultReapIntervalKey = 'kernel.defaultReapInterval'; + assert(kvStore.has(oldDefaultReapIntervalKey)); + assert(!kvStore.has(DEFAULT_REAP_DIRT_THRESHOLD_KEY)); + + /** + * @typedef { import('../types-internal.js').ReapDirtThreshold } ReapDirtThreshold + */ + + /** @type ReapDirtThreshold */ + const threshold = { + deliveries: 'never', + gcKrefs: 'never', + computrons: 'never', + }; + + const oldValue = getRequired(oldDefaultReapIntervalKey); + if (oldValue !== 'never') { + const value = Number.parseInt(oldValue, 10); + assert.typeof(value, 'number'); + threshold.deliveries = value; + // if BOYD wasn't turned off entirely (eg + // defaultReapInterval='never', which only happens in unit + // tests), then pretend we wanted a gcKrefs= threshold all + // along, so all vats get a retroactive gcKrefs threshold, which + // we need for the upcoming slow-vat-deletion to not trigger + // gigantic BOYD and break the kernel + threshold.gcKrefs = DEFAULT_GC_KREFS_PER_BOYD; + } + harden(threshold); + kvStore.set(DEFAULT_REAP_DIRT_THRESHOLD_KEY, JSON.stringify(threshold)); + kvStore.delete(oldDefaultReapIntervalKey); + + // now upgrade all vats + for (const [_name, vatID] of getAllStaticVats(kvStore)) { + upgradeVatV0toV1(kvStore, threshold, vatID); + } + for (const vatID of getAllDynamicVats(getRequired)) { + upgradeVatV0toV1(kvStore, threshold, vatID); + } + + modified = true; + version = 1; + } + + if (modified) { + kvStore.set('version', `${version}`); + } + return modified; +}; +harden(upgradeSwingset); diff --git a/packages/SwingSet/src/index.js b/packages/SwingSet/src/index.js index 3ba4767ef54..af694b5a215 100644 --- a/packages/SwingSet/src/index.js +++ b/packages/SwingSet/src/index.js @@ -9,7 +9,7 @@ export { loadBasedir, loadSwingsetConfigFile, } from './controller/initializeSwingset.js'; - +export { upgradeSwingset } from './controller/upgradeSwingset.js'; export { buildMailboxStateMap, buildMailbox, diff --git a/packages/SwingSet/src/kernel/kernel.js b/packages/SwingSet/src/kernel/kernel.js index 4dd202f4771..99dfee62f11 100644 --- a/packages/SwingSet/src/kernel/kernel.js +++ b/packages/SwingSet/src/kernel/kernel.js @@ -364,6 +364,7 @@ export default function buildKernel( * * @typedef { import('@agoric/swingset-liveslots').MeterConsumption } MeterConsumption * @typedef { import('../types-internal.js').MeterID } MeterID + * @typedef { import('../types-internal.js').Dirt } Dirt * * Any delivery crank (send, notify, start-vat.. anything which is allowed * to make vat delivery) emits one of these status events if a delivery @@ -382,7 +383,7 @@ export default function buildKernel( * didDelivery?: VatID, // we made a delivery to a vat, for run policy and save-snapshot * computrons?: BigInt, // computron count for run policy * meterID?: string, // deduct those computrons from a meter - * decrementReapCount?: { vatID: VatID }, // the reap counter should decrement + * measureDirt?: { vatID: VatID, dirt: Dirt }, // the dirt counter should increment * terminate?: { vatID: VatID, reject: boolean, info: SwingSetCapData }, // terminate vat, notify vat-admin * vatAdminMethargs?: RawMethargs, // methargs to notify vat-admin about create/upgrade results * } } CrankResults @@ -449,16 +450,17 @@ export default function buildKernel( * event handler. * * Two flags influence this: - * `decrementReapCount` is used for deliveries that run userspace code + * `measureDirt` is used for non-BOYD deliveries * `meterID` means we should check a meter * * @param {VatID} vatID * @param {DeliveryStatus} status - * @param {boolean} decrementReapCount + * @param {boolean} measureDirt * @param {MeterID} [meterID] + * @param {number} [gcKrefs] * @returns {CrankResults} */ - function deliveryCrankResults(vatID, status, decrementReapCount, meterID) { + function deliveryCrankResults(vatID, status, measureDirt, meterID, gcKrefs) { let meterUnderrun = false; let computrons; if (status.metering?.compute) { @@ -502,8 +504,16 @@ export default function buildKernel( results.terminate = { vatID, ...status.vatRequestedTermination }; } - if (decrementReapCount && !(results.abort || results.terminate)) { - results.decrementReapCount = { vatID }; + if (measureDirt && !(results.abort || results.terminate)) { + const dirt = { deliveries: 1 }; + if (computrons) { + // this is BigInt, but we use plain Number in Dirt records + dirt.computrons = Number(computrons); + } + if (gcKrefs) { + dirt.gcKrefs = gcKrefs; + } + results.measureDirt = { vatID, dirt }; } // We leave results.consumeMessage up to the caller. Send failures @@ -542,7 +552,8 @@ export default function buildKernel( } const status = await deliverAndLogToVat(vatID, kd, vd); - return deliveryCrankResults(vatID, status, true, meterID); + const gcKrefs = undefined; // TODO maybe increase by number of vrefs in args? + return deliveryCrankResults(vatID, status, true, meterID, gcKrefs); } /** @@ -588,7 +599,8 @@ export default function buildKernel( const vd = vatWarehouse.kernelDeliveryToVatDelivery(vatID, kd); vatKeeper.deleteCListEntriesForKernelSlots(targets); const status = await deliverAndLogToVat(vatID, kd, vd); - return deliveryCrankResults(vatID, status, true, meterID); + const gcKrefs = undefined; // TODO maybe increase by number of vrefs in args? + return deliveryCrankResults(vatID, status, true, meterID, gcKrefs); } /** @@ -616,7 +628,9 @@ export default function buildKernel( } const vd = vatWarehouse.kernelDeliveryToVatDelivery(vatID, kd); const status = await deliverAndLogToVat(vatID, kd, vd); - return deliveryCrankResults(vatID, status, false); // no meterID + const meterID = undefined; // no meterID + const gcKrefs = krefs.length; + return deliveryCrankResults(vatID, status, true, meterID, gcKrefs); } /** @@ -631,11 +645,14 @@ export default function buildKernel( if (!vatWarehouse.lookup(vatID)) { return NO_DELIVERY_CRANK_RESULTS; // can't collect from the dead } + const vatKeeper = kernelKeeper.provideVatKeeper(vatID); /** @type { KernelDeliveryBringOutYourDead } */ const kd = harden([type]); const vd = vatWarehouse.kernelDeliveryToVatDelivery(vatID, kd); const status = await deliverAndLogToVat(vatID, kd, vd); - return deliveryCrankResults(vatID, status, false); // no meter + vatKeeper.clearReapDirt(); // BOYD zeros out the when-to-BOYD counters + // no gcKrefs, BOYD clears them anyways + return deliveryCrankResults(vatID, status, false); // no meter, BOYD clears dirt } /** @@ -676,8 +693,9 @@ export default function buildKernel( const status = await deliverAndLogToVat(vatID, kd, vd); // note: if deliveryCrankResults() learns to suspend vats, // startVat errors should still terminate them + const gcKrefs = undefined; // TODO maybe increase by number of vrefs in args? const results = harden({ - ...deliveryCrankResults(vatID, status, true, meterID), + ...deliveryCrankResults(vatID, status, true, meterID, gcKrefs), consumeMessage: true, }); return results; @@ -742,9 +760,17 @@ export default function buildKernel( function setKernelVatOption(vatID, option, value) { switch (option) { case 'reapInterval': { + // This still controls reapDirtThreshold.deliveries, and we do not + // yet offer controls for the other limits (gcKrefs or computrons). if (value === 'never' || isNat(value)) { const vatKeeper = kernelKeeper.provideVatKeeper(vatID); - vatKeeper.updateReapInterval(value); + const threshold = { ...vatKeeper.getReapDirtThreshold() }; + if (value === 'never') { + threshold.deliveries = value; + } else { + threshold.deliveries = Number(value); + } + vatKeeper.setReapDirtThreshold(threshold); } else { console.log(`WARNING: invalid reapInterval value`, value); } @@ -884,6 +910,7 @@ export default function buildKernel( const boydVD = vatWarehouse.kernelDeliveryToVatDelivery(vatID, boydKD); const boydStatus = await deliverAndLogToVat(vatID, boydKD, boydVD); const boydResults = deliveryCrankResults(vatID, boydStatus, false); + vatKeeper.clearReapDirt(); // we don't meter bringOutYourDead since no user code is running, but we // still report computrons to the runPolicy @@ -958,7 +985,14 @@ export default function buildKernel( startVatKD, startVatVD, ); - const startVatResults = deliveryCrankResults(vatID, startVatStatus, false); + const gcKrefs = undefined; // TODO maybe increase by number of vrefs in args? + const startVatResults = deliveryCrankResults( + vatID, + startVatStatus, + true, + meterID, + gcKrefs, + ); computrons = addComputrons(computrons, startVatResults.computrons); if (startVatResults.terminate) { @@ -1299,13 +1333,11 @@ export default function buildKernel( } } } - if (crankResults.decrementReapCount) { + if (crankResults.measureDirt) { // deliveries cause garbage, garbage needs collection - const { vatID } = crankResults.decrementReapCount; + const { vatID, dirt } = crankResults.measureDirt; const vatKeeper = kernelKeeper.provideVatKeeper(vatID); - if (vatKeeper.countdownToReap()) { - kernelKeeper.scheduleReap(vatID); - } + vatKeeper.addDirt(dirt); // might schedule a reap for that vat } // Vat termination (during delivery) is triggered by an illegal @@ -1579,10 +1611,14 @@ export default function buildKernel( 'bundleID', 'enablePipelining', 'reapInterval', + 'reapGCKrefs', + 'neverReap', ]); const { bundleID = 'b1-00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', reapInterval = 'never', + reapGCKrefs = 'never', + neverReap = false, enablePipelining, } = creationOptions; const vatID = kernelKeeper.allocateVatIDForNameIfNeeded(name); @@ -1594,6 +1630,8 @@ export default function buildKernel( const options = { name, reapInterval, + reapGCKrefs, + neverReap, enablePipelining, managerType, }; @@ -1740,14 +1778,38 @@ export default function buildKernel( } function changeKernelOptions(options) { - assertKnownOptions(options, ['defaultReapInterval', 'snapshotInterval']); + assertKnownOptions(options, [ + 'defaultReapInterval', + 'defaultReapGCKrefs', + 'snapshotInterval', + ]); kernelKeeper.startCrank(); try { for (const option of Object.getOwnPropertyNames(options)) { const value = options[option]; switch (option) { case 'defaultReapInterval': { - kernelKeeper.setDefaultReapInterval(value); + if (typeof value === 'number') { + assert(value > 0, `defaultReapInterval = ${value}`); + } else { + assert.equal(value, 'never', `defaultReapInterval = ${value}`); + } + kernelKeeper.setDefaultReapDirtThreshold({ + ...kernelKeeper.getDefaultReapDirtThreshold(), + deliveries: value, + }); + break; + } + case 'defaultReapGCKrefs': { + if (typeof value === 'number') { + assert(value > 0, `defaultReapGCKrefs = ${value}`); + } else { + assert.equal(value, 'never', `defaultReapGCKrefs = ${value}`); + } + kernelKeeper.setDefaultReapDirtThreshold({ + ...kernelKeeper.getDefaultReapDirtThreshold(), + gcKrefs: value, + }); break; } case 'snapshotInterval': { diff --git a/packages/SwingSet/src/kernel/state/kernelKeeper.js b/packages/SwingSet/src/kernel/state/kernelKeeper.js index 7b24e6c9196..70f1642934b 100644 --- a/packages/SwingSet/src/kernel/state/kernelKeeper.js +++ b/packages/SwingSet/src/kernel/state/kernelKeeper.js @@ -1,6 +1,11 @@ +/* eslint-disable no-use-before-define */ import { Nat, isNat } from '@endo/nat'; import { assert, Fail } from '@agoric/assert'; -import { initializeVatState, makeVatKeeper } from './vatKeeper.js'; +import { + initializeVatState, + makeVatKeeper, + DEFAULT_REAP_DIRT_THRESHOLD_KEY, +} from './vatKeeper.js'; import { initializeDeviceState, makeDeviceKeeper } from './deviceKeeper.js'; import { parseReachableAndVatSlot } from './reachable.js'; import { insistStorageAPI } from '../../lib/storageAPI.js'; @@ -33,14 +38,17 @@ const enableKernelGC = true; * @typedef { import('../../types-external.js').BundleCap } BundleCap * @typedef { import('../../types-external.js').BundleID } BundleID * @typedef { import('../../types-external.js').EndoZipBase64Bundle } EndoZipBase64Bundle - * @typedef { import('../../types-external.js').KernelOptions } KernelOptions * @typedef { import('../../types-external.js').KernelSlog } KernelSlog * @typedef { import('../../types-external.js').ManagerType } ManagerType * @typedef { import('../../types-external.js').SnapStore } SnapStore * @typedef { import('../../types-external.js').TranscriptStore } TranscriptStore * @typedef { import('../../types-external.js').VatKeeper } VatKeeper + * @typedef { import('../../types-internal.js').InternalKernelOptions } InternalKernelOptions + * @typedef { import('../../types-internal.js').ReapDirtThreshold } ReapDirtThreshold */ +export { DEFAULT_REAP_DIRT_THRESHOLD_KEY }; + // Kernel state lives in a key-value store supporting key retrieval by // lexicographic range. All keys and values are strings. // We simulate a tree by concatenating path-name components with ".". When we @@ -52,8 +60,9 @@ const enableKernelGC = true; // allowed to vary between instances in a consensus machine. Everything else // is required to be deterministic. // -// The schema is: +// The current ("v1") schema is: // +// version = '1' // vat.names = JSON([names..]) // vat.dynamicIDs = JSON([vatIDs..]) // vat.name.$NAME = $vatID = v$NN @@ -68,13 +77,22 @@ const enableKernelGC = true; // bundle.$BUNDLEID = JSON(bundle) // // kernel.defaultManagerType = managerType -// kernel.defaultReapInterval = $NN +// (old) kernel.defaultReapInterval = $NN +// kernel.defaultReapDirtThreshold = JSON({ thresholds }) +// thresholds (all optional) +// deliveries: number or 'never' (default) +// gcKrefs: number or 'never' (default) +// computrons: number or 'never' (default) +// never: boolean (default false) // kernel.relaxDurabilityRules = missing | 'true' // kernel.snapshotInitial = $NN // kernel.snapshotInterval = $NN // v$NN.source = JSON({ bundle }) or JSON({ bundleName }) -// v$NN.options = JSON +// v$NN.options = JSON , options include: +// .reapDirtThreshold = JSON({ thresholds }) +// thresholds (all optional, default to kernel-wide defaultReapDirtThreshold) +// (leave room for .snapshotDirtThreshold for #6786) // v$NN.o.nextID = $NN // v$NN.p.nextID = $NN // v$NN.d.nextID = $NN @@ -84,8 +102,10 @@ const enableKernelGC = true; // v$NN.c.$vatSlot = $kernelSlot = ko$NN/kp$NN/kd$NN // v$NN.vs.$key = string // v$NN.meter = m$NN // XXX does this exist? -// v$NN.reapInterval = $NN or 'never' -// v$NN.reapCountdown = $NN or 'never' +// old (v0): v$NN.reapInterval = $NN or 'never' +// old (v0): v$NN.reapCountdown = $NN or 'never' +// v$NN.reapDirt = JSON({ deliveries, gcKrefs, computrons }) // missing keys treated as zero +// (leave room for v$NN.snapshotDirt and options.snapshotDirtThreshold for #6786) // exclude from consensus // local.* @@ -132,6 +152,14 @@ const enableKernelGC = true; // Prefix reserved for host written data: // host. +// Kernel state schemas. The 'version' key records the state of the +// database, and is only modified by a call to upgradeSwingset(). +// +// v0: the original +// v1: replace `kernel.defaultReapInterval` with `kernel.defaultReapDirtThreshold` +// replace vat's `vNN.reapInterval`/`vNN.reapCountdown` with `vNN.reapDirt` +// and a `vNN.reapDirtThreshold` in `vNN.options` + export function commaSplit(s) { if (s === '') { return []; @@ -145,6 +173,20 @@ function insistMeterID(m) { Nat(BigInt(m.slice(1))); } +export const getAllStaticVats = kvStore => { + const result = []; + for (const k of enumeratePrefixedKeys(kvStore, 'vat.name.')) { + const name = k.slice(9); + const vatID = kvStore.get(k) || Fail`getNextKey ensures get`; + result.push([name, vatID]); + } + return result; +}; + +export const getAllDynamicVats = getRequired => { + return JSON.parse(getRequired('vat.dynamicIDs')); +}; + // we use different starting index values for the various vNN/koNN/kdNN/kpNN // slots, to reduce confusing overlap when looking at debug messages (e.g. // seeing both kp1 and ko1, which are completely unrelated despite having the @@ -163,6 +205,23 @@ const FIRST_PROMISE_ID = 40n; const FIRST_CRANK_NUMBER = 0n; const FIRST_METER_ID = 1n; +// this default "reap interval" is low for the benefit of tests: +// applications should set it to something higher (perhaps 200) based +// on their expected usage + +export const DEFAULT_DELIVERIES_PER_BOYD = 1; + +// "20" will trigger a BOYD after 10 krefs are dropped and retired +// (drops and retires are delivered in separate messages, so +// 10+10=20). The worst case work-expansion we've seen is in #8401, +// where one drop breaks one cycle, and each cycle's cleanup causes 50 +// syscalls in the next v9-zoe BOYD. So this should limit each BOYD +// to cleaning 10 cycles, in 500 syscalls. + +export const DEFAULT_GC_KREFS_PER_BOYD = 20; + +const EXPECTED_VERSION = 1; + /** * @param {SwingStoreKernelStorage} kernelStorage * @param {KernelSlog|null} kernelSlog @@ -182,6 +241,16 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { return kvStore.get(key); } + if ( + !kvStore.has('version') || + Number(getRequired('version')) !== EXPECTED_VERSION + ) { + const have = kvStore.get('version') || 'undefined'; + throw Error( + `kernelStorage is too old (have ${have}, need ${EXPECTED_VERSION}), please upgradeSwingset()`, + ); + } + const { incStat, decStat, @@ -290,12 +359,17 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { } /** - * @param {KernelOptions} kernelOptions + * @param {InternalKernelOptions} kernelOptions */ function createStartingKernelState(kernelOptions) { + // this should probably be a standalone function, not a method const { defaultManagerType = 'local', - defaultReapInterval = 1, + defaultReapDirtThreshold = { + deliveries: DEFAULT_DELIVERIES_PER_BOYD, + gcKrefs: DEFAULT_GC_KREFS_PER_BOYD, + computrons: 'never', + }, relaxDurabilityRules = false, snapshotInitial = 3, snapshotInterval = 200, @@ -317,7 +391,10 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { initQueue('acceptanceQueue'); kvStore.set('crankNumber', `${FIRST_CRANK_NUMBER}`); kvStore.set('kernel.defaultManagerType', defaultManagerType); - kvStore.set('kernel.defaultReapInterval', `${defaultReapInterval}`); + kvStore.set( + DEFAULT_REAP_DIRT_THRESHOLD_KEY, + JSON.stringify(defaultReapDirtThreshold), + ); kvStore.set('kernel.snapshotInitial', `${snapshotInitial}`); kvStore.set('kernel.snapshotInterval', `${snapshotInterval}`); if (relaxDurabilityRules) { @@ -352,21 +429,26 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { /** * - * @returns {number | 'never'} + * @returns { ReapDirtThreshold } */ - function getDefaultReapInterval() { - const r = getRequired('kernel.defaultReapInterval'); - const ri = r === 'never' ? r : Number.parseInt(r, 10); - assert(ri === 'never' || typeof ri === 'number', `k.dri is '${ri}'`); - return ri; + function getDefaultReapDirtThreshold() { + return JSON.parse(getRequired(DEFAULT_REAP_DIRT_THRESHOLD_KEY)); } - function setDefaultReapInterval(interval) { - assert( - interval === 'never' || isNat(interval), - 'invalid defaultReapInterval value', - ); - kvStore.set('kernel.defaultReapInterval', `${interval}`); + /** + * @param { ReapDirtThreshold } threshold + */ + function setDefaultReapDirtThreshold(threshold) { + assert.typeof(threshold, 'object'); + assert(threshold); + for (const [key, value] of Object.entries(threshold)) { + if (typeof value === 'number') { + assert(value > 0, `threshold[${key}] = ${value}`); + } else { + assert.equal(value, 'never', `threshold[${key}] = ${value}`); + } + } + kvStore.set(DEFAULT_REAP_DIRT_THRESHOLD_KEY, JSON.stringify(threshold)); } function getNat(key) { @@ -764,7 +846,6 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { let idx = 0; for (const dataSlot of capdata.slots) { - // eslint-disable-next-line no-use-before-define incrementRefCount(dataSlot, `resolve|${kernelSlot}|s${idx}`); idx += 1; } @@ -787,7 +868,6 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { function cleanupAfterTerminatedVat(vatID) { insistVatID(vatID); - // eslint-disable-next-line no-use-before-define const vatKeeper = provideVatKeeper(vatID); const exportPrefix = `${vatID}.c.o+`; const importPrefix = `${vatID}.c.o-`; @@ -1102,15 +1182,7 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { kvStore.set(KEY, JSON.stringify(dynamicVatIDs)); } - function getStaticVats() { - const result = []; - for (const k of enumeratePrefixedKeys(kvStore, 'vat.name.')) { - const name = k.slice(9); - const vatID = kvStore.get(k) || Fail`getNextKey ensures get`; - result.push([name, vatID]); - } - return result; - } + const getStaticVats = () => getAllStaticVats(kvStore); function getDevices() { const result = []; @@ -1122,9 +1194,7 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { return result; } - function getDynamicVats() { - return JSON.parse(getRequired('vat.dynamicIDs')); - } + const getDynamicVats = () => getAllDynamicVats(getRequired); function allocateUpgradeID() { const nextID = Nat(BigInt(getRequired(`vat.nextUpgradeID`))); @@ -1262,7 +1332,6 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { if (reachable === 0) { const ownerVatID = ownerOfKernelObject(kref); if (ownerVatID) { - // eslint-disable-next-line no-use-before-define const vatKeeper = provideVatKeeper(ownerVatID); const isReachable = vatKeeper.getReachableFlag(kref); if (isReachable) { @@ -1316,6 +1385,7 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { incStat, decStat, getCrankNumber, + scheduleReap, snapStore, ); ephemeral.vatKeepers.set(vatID, vk); @@ -1536,9 +1606,10 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { setInitialized, createStartingKernelState, getDefaultManagerType, - getDefaultReapInterval, getRelaxDurabilityRules, - setDefaultReapInterval, + getDefaultReapDirtThreshold, + setDefaultReapDirtThreshold, + getSnapshotInitial, getSnapshotInterval, setSnapshotInterval, diff --git a/packages/SwingSet/src/kernel/state/vatKeeper.js b/packages/SwingSet/src/kernel/state/vatKeeper.js index 33769b87300..a64daaeb04e 100644 --- a/packages/SwingSet/src/kernel/state/vatKeeper.js +++ b/packages/SwingSet/src/kernel/state/vatKeeper.js @@ -1,7 +1,7 @@ /** * Kernel's keeper of persistent state for a vat. */ -import { Nat, isNat } from '@endo/nat'; +import { Nat } from '@endo/nat'; import { assert, q, Fail } from '@agoric/assert'; import { parseKernelSlot } from '../parseKernelSlots.js'; import { makeVatSlot, parseVatSlot } from '../../lib/parseVatSlots.js'; @@ -18,7 +18,9 @@ import { enumeratePrefixedKeys } from './storageHelper.js'; * @typedef { import('../../types-external.js').SnapStore } SnapStore * @typedef { import('../../types-external.js').SourceOfBundle } SourceOfBundle * @typedef { import('../../types-external.js').TranscriptStore } TranscriptStore + * @typedef { import('../../types-internal.js').Dirt } Dirt * @typedef { import('../../types-internal.js').VatManager } VatManager + * @typedef { import('../../types-internal.js').ReapDirtThreshold } ReapDirtThreshold * @typedef { import('../../types-internal.js').RecordedVatOptions } RecordedVatOptions * @typedef { import('../../types-internal.js').TranscriptEntry } TranscriptEntry * @import {TranscriptDeliverySaveSnapshot} from '../../types-internal.js' @@ -32,6 +34,13 @@ const FIRST_OBJECT_ID = 50n; const FIRST_PROMISE_ID = 60n; const FIRST_DEVICE_ID = 70n; +// TODO: we export this from vatKeeper.js, and import it from +// kernelKeeper.js, because both files need it, and we want to avoid +// an import cycle (kernelKeeper imports other things from vatKeeper), +// but it really wants to live in kernelKeeper not vatKeeper +export const DEFAULT_REAP_DIRT_THRESHOLD_KEY = + 'kernel.defaultReapDirtThreshold'; + /** * Establish a vat's state. * @@ -49,20 +58,18 @@ export function initializeVatState( source, options, ) { - assert(options.workerOptions, `vat ${vatID} options missing workerOptions`); assert(source); assert('bundle' in source || 'bundleName' in source || 'bundleID' in source); + assert(options); assert.typeof(options, 'object'); - const count = options.reapInterval; - assert(count === 'never' || isNat(count), `bad reapCountdown ${count}`); + assert(options.workerOptions, `vat ${vatID} options missing workerOptions`); kvStore.set(`${vatID}.o.nextID`, `${FIRST_OBJECT_ID}`); kvStore.set(`${vatID}.p.nextID`, `${FIRST_PROMISE_ID}`); kvStore.set(`${vatID}.d.nextID`, `${FIRST_DEVICE_ID}`); + kvStore.set(`${vatID}.reapDirt`, JSON.stringify({})); kvStore.set(`${vatID}.source`, JSON.stringify(source)); kvStore.set(`${vatID}.options`, JSON.stringify(options)); - kvStore.set(`${vatID}.reapInterval`, `${count}`); - kvStore.set(`${vatID}.reapCountdown`, `${count}`); transcriptStore.initTranscript(vatID); } @@ -87,6 +94,7 @@ export function initializeVatState( * @param {*} incStat * @param {*} decStat * @param {*} getCrankNumber + * @param {*} scheduleReap * @param {SnapStore} [snapStore] * returns an object to hold and access the kernel's state for the given vat */ @@ -107,10 +115,16 @@ export function makeVatKeeper( incStat, decStat, getCrankNumber, + scheduleReap, snapStore = undefined, ) { insistVatID(vatID); + // note: calling makeVatKeeper() does not change the DB. Any + // initialization or upgrade must be complete before it is + // called. Only the methods returned by makeVatKeeper() will change + // the DB. + function getRequired(key) { const value = kvStore.get(key); if (value === undefined) { @@ -119,6 +133,8 @@ export function makeVatKeeper( return value; } + const reapDirtKey = `${vatID}.reapDirt`; + /** * @param {SourceOfBundle} source * @param {RecordedVatOptions} options @@ -148,35 +164,76 @@ export function makeVatKeeper( return harden(options); } - function updateReapInterval(reapInterval) { - reapInterval === 'never' || - isNat(reapInterval) || - Fail`bad reapInterval ${reapInterval}`; - kvStore.set(`${vatID}.reapInterval`, `${reapInterval}`); - if (reapInterval === 'never') { - kvStore.set(`${vatID}.reapCountdown`, 'never'); - } - } + // This is named "addDirt" because it should increment all dirt + // counters (both for reap/BOYD and for heap snapshotting). We don't + // have `heapSnapshotDirt` yet, but when we do, it should get + // incremented here. - function countdownToReap() { - const rawCount = getRequired(`${vatID}.reapCountdown`); - if (rawCount === 'never') { - return false; - } else { - const count = Number.parseInt(rawCount, 10); - if (count === 1) { - kvStore.set( - `${vatID}.reapCountdown`, - getRequired(`${vatID}.reapInterval`), - ); - return true; - } else { - kvStore.set(`${vatID}.reapCountdown`, `${count - 1}`); - return false; + /** + * Add some "dirt" to the vat, possibly triggering a reap/BOYD. + * + * @param {Dirt} moreDirt + */ + function addDirt(moreDirt) { + const reapDirt = JSON.parse(getRequired(reapDirtKey)); + const thresholds = { + ...JSON.parse(getRequired(DEFAULT_REAP_DIRT_THRESHOLD_KEY)), + ...JSON.parse(getRequired(`${vatID}.options`)).reapDirtThreshold, + }; + assert.typeof(moreDirt, 'object'); + let reap = false; + for (const key of Object.keys(moreDirt)) { + const threshold = thresholds[key]; + // Don't accumulate dirt if it can't eventually trigger a + // BOYD. This is mainly to keep comms from counting upwards + // forever. TODO revisit this when we add heapSnapshotDirt, + // maybe check both thresholds and accumulate the dirt if either + // one is non-'never'. + if (threshold && threshold !== 'never') { + const oldDirt = reapDirt[key] || 0; + // The 'moreDirt' value might be Number or BigInt (eg + // .computrons). We coerce to Number so we can JSON-stringify. + const newDirt = oldDirt + Number(moreDirt[key]); + reapDirt[key] = newDirt; + if (newDirt >= threshold) { + reap = true; + } + } + } + if (!thresholds.never) { + kvStore.set(reapDirtKey, JSON.stringify(reapDirt)); + if (reap) { + scheduleReap(vatID); } } } + function getReapDirt() { + return JSON.parse(getRequired(reapDirtKey)); + } + + function clearReapDirt() { + // This is only called after a BOYD, so it should only clear the + // reap/BOYD counters. If/when we add heap-snapshot counters, + // those should get cleared in a separate clearHeapSnapshotDirt() + // function. + const reapDirt = {}; + kvStore.set(reapDirtKey, JSON.stringify(reapDirt)); + } + + function getReapDirtThreshold() { + return getOptions().reapDirtThreshold; + } + + /** + * @param {ReapDirtThreshold} reapDirtThreshold + */ + function setReapDirtThreshold(reapDirtThreshold) { + assert.typeof(reapDirtThreshold, 'object'); + const options = { ...getOptions(), reapDirtThreshold }; + kvStore.set(`${vatID}.options`, JSON.stringify(options)); + } + function nextDeliveryNum() { const { endPos } = transcriptStore.getCurrentSpanBounds(vatID); return Nat(endPos); @@ -669,8 +726,11 @@ export function makeVatKeeper( setSourceAndOptions, getSourceAndOptions, getOptions, - countdownToReap, - updateReapInterval, + addDirt, + getReapDirt, + clearReapDirt, + getReapDirtThreshold, + setReapDirtThreshold, nextDeliveryNum, getIncarnationNumber, importsKernelSlot, diff --git a/packages/SwingSet/src/kernel/vat-loader/manager-factory.js b/packages/SwingSet/src/kernel/vat-loader/manager-factory.js index 56160da9254..571cb868af5 100644 --- a/packages/SwingSet/src/kernel/vat-loader/manager-factory.js +++ b/packages/SwingSet/src/kernel/vat-loader/manager-factory.js @@ -45,7 +45,6 @@ export function makeVatManagerFactory({ 'enableSetup', 'useTranscript', 'critical', - 'reapInterval', 'sourcedConsole', 'name', ]); diff --git a/packages/SwingSet/src/kernel/vat-loader/vat-loader.js b/packages/SwingSet/src/kernel/vat-loader/vat-loader.js index a83022673d1..04e3e7a4150 100644 --- a/packages/SwingSet/src/kernel/vat-loader/vat-loader.js +++ b/packages/SwingSet/src/kernel/vat-loader/vat-loader.js @@ -26,7 +26,7 @@ export function makeVatLoader(stuff) { 'enablePipelining', 'useTranscript', 'critical', - 'reapInterval', + 'reapDirtThreshold', ]; /** diff --git a/packages/SwingSet/src/lib/recordVatOptions.js b/packages/SwingSet/src/lib/recordVatOptions.js index 9e87fca1200..991eace5e22 100644 --- a/packages/SwingSet/src/lib/recordVatOptions.js +++ b/packages/SwingSet/src/lib/recordVatOptions.js @@ -10,7 +10,9 @@ export const makeVatOptionRecorder = (kernelKeeper, bundleHandler) => { enablePipelining = false, enableDisavow = false, useTranscript = true, - reapInterval = kernelKeeper.getDefaultReapInterval(), + reapInterval, + reapGCKrefs, + neverReap = false, critical = false, meterID = undefined, managerType = kernelKeeper.getDefaultManagerType(), @@ -21,6 +23,17 @@ export const makeVatOptionRecorder = (kernelKeeper, bundleHandler) => { if (unused.length) { Fail`OptionRecorder: ${vatID} unused options ${unused.join(',')}`; } + const reapDirtThreshold = {}; + if (reapInterval !== undefined) { + reapDirtThreshold.deliveries = reapInterval; + } + if (reapGCKrefs !== undefined) { + reapDirtThreshold.gcKrefs = reapGCKrefs; + } + if (neverReap) { + reapDirtThreshold.never = true; + } + // TODO no computrons knob? const workerOptions = await makeWorkerOptions( managerType, bundleHandler, @@ -35,10 +48,11 @@ export const makeVatOptionRecorder = (kernelKeeper, bundleHandler) => { enablePipelining, enableDisavow, useTranscript, - reapInterval, + reapDirtThreshold, critical, meterID, }); + // want vNN.options to be in place before provideVatKeeper, so it can cache reapDirtThreshold in RAM, so: kernelKeeper.createVatState(vatID, source, vatOptions); }; diff --git a/packages/SwingSet/src/types-external.js b/packages/SwingSet/src/types-external.js index 7b45acfa86c..983055b1fe3 100644 --- a/packages/SwingSet/src/types-external.js +++ b/packages/SwingSet/src/types-external.js @@ -28,14 +28,14 @@ export {}; */ /** - * @typedef {{ - * defaultManagerType?: ManagerType, - * defaultReapInterval?: number | 'never', - * relaxDurabilityRules?: boolean, - * snapshotInitial?: number, - * snapshotInterval?: number, - * pinBootstrapRoot?: boolean, - * }} KernelOptions + * @typedef { object } KernelOptions + * @property { ManagerType } [defaultManagerType] + * @property { number | 'never' } [defaultReapGCKrefs] + * @property { number | 'never' } [defaultReapInterval] + * @property { boolean } [relaxDurabilityRules] + * @property { number } [snapshotInitial] + * @property { number } [snapshotInterval] + * @property { boolean } [pinBootstrapRoot] */ /** @@ -292,13 +292,15 @@ export {}; * reconstructed via replay. If false, no such record is kept. * Defaults to true. * @property { number | 'never' } [reapInterval] - * The interval (measured in number of deliveries to the vat) - * after which the kernel will deliver the 'bringOutYourDead' - * directive to the vat. If the value is 'never', - * 'bringOutYourDead' will never be delivered and the vat will - * be responsible for internally managing (in a deterministic - * manner) any visible effects of garbage collection. Defaults - * to the kernel's configured 'defaultReapInterval' value. + * Trigger a bringOutYourDead after the vat has received + * this many deliveries. If the value is 'never', + * 'bringOutYourDead' will not be triggered by a delivery + * count (but might be triggered for other reasons). + * @property { number | 'never' } [reapGCKrefs] + * Trigger a bringOutYourDead when the vat has been given + * this many krefs in GC deliveries (dropImports, + * retireImports, retireExports). If the value is 'never', + * GC deliveries and their krefs are not treated specially. * @property { boolean } [critical] */ diff --git a/packages/SwingSet/src/types-internal.js b/packages/SwingSet/src/types-internal.js index 9140db5b5ec..6d63425efaa 100644 --- a/packages/SwingSet/src/types-internal.js +++ b/packages/SwingSet/src/types-internal.js @@ -1,6 +1,19 @@ export {}; /** + * The host provides (external) KernelOptions as part of the + * SwingSetConfig record it passes to initializeSwingset(). This + * internal type represents the modified form passed to + * initializeKernel() and kernelKeeper.createStartingKernelState . + * + * @typedef { object } InternalKernelOptions + * @property { ManagerType } [defaultManagerType] + * @property { ReapDirtThreshold } [defaultReapDirtThreshold] + * @property { boolean } [relaxDurabilityRules] + * @property { number } [snapshotInitial] + * @property { number } [snapshotInterval] + * + * * The internal data that controls which worker we use (and how we use it) is * stored in a WorkerOptions record, which comes in "local", "node-subprocess", * and "xsnap" flavors. @@ -35,11 +48,48 @@ export {}; * @property { boolean } enableSetup * @property { boolean } enablePipelining * @property { boolean } useTranscript - * @property { number | 'never' } reapInterval + * @property { ReapDirtThreshold } reapDirtThreshold * @property { boolean } critical * @property { MeterID } [meterID] // property must be present, but can be undefined * @property { WorkerOptions } workerOptions * @property { boolean } enableDisavow + * + * @typedef ChangeVatOptions + * @property { number } [reapInterval] + */ + +/** + * Reap/BringOutYourDead/BOYD Scheduling + * + * We trigger a BringOutYourDead delivery (which "reaps" all dead + * objects from the vat) after a certain threshold of "dirt" has + * accumulated. This type is used to define the thresholds for three + * counters: 'deliveries', 'gcKrefs', and 'computrons'. If a property + * is a number, we trigger BOYD when the counter for that property + * exceeds the threshold value. If a property is the string 'never' or + * missing we do not use that counter to trigger BOYD. + * + * Each vat has a .reapDirtThreshold in their vNN.options record, + * which overrides the kernel-wide settings in + * 'kernel.defaultReapDirtThreshold' + * + * @typedef { object } ReapDirtThreshold + * @property { number | 'never' } [deliveries] + * @property { number | 'never' } [gcKrefs] + * @property { number | 'never' } [computrons] + * @property { boolean } [never] + */ + +/** + * Each counter in Dirt matches a threshold in + * ReapDirtThreshold. Missing values are treated as zero, so vats + * start with {} and accumulate dirt as deliveries are made, until a + * BOYD clears them. + * + * @typedef { object } Dirt + * @property { number } [deliveries] + * @property { number } [gcKrefs] + * @property { number } [computrons] */ /** @@ -86,7 +136,7 @@ export {}; * @typedef { { type: 'upgrade-vat', vatID: VatID, upgradeID: string, * bundleID: BundleID, vatParameters: SwingSetCapData, * upgradeMessage: string } } RunQueueEventUpgradeVat - * @typedef { { type: 'changeVatOptions', vatID: VatID, options: Record } } RunQueueEventChangeVatOptions + * @typedef { { type: 'changeVatOptions', vatID: VatID, options: ChangeVatOptions } } RunQueueEventChangeVatOptions * @typedef { { type: 'startVat', vatID: VatID, vatParameters: SwingSetCapData } } RunQueueEventStartVat * @typedef { { type: 'dropExports', vatID: VatID, krefs: string[] } } RunQueueEventDropExports * @typedef { { type: 'retireExports', vatID: VatID, krefs: string[] } } RunQueueEventRetireExports diff --git a/packages/SwingSet/test/change-parameters/change-parameters.test.js b/packages/SwingSet/test/change-parameters/change-parameters.test.js index 290cba11ca1..75319bb6443 100644 --- a/packages/SwingSet/test/change-parameters/change-parameters.test.js +++ b/packages/SwingSet/test/change-parameters/change-parameters.test.js @@ -29,14 +29,22 @@ async function testChangeParameters(t) { t.teardown(c.shutdown); c.pinVatRoot('bootstrap'); await c.run(); - t.is(kvStore.get('kernel.defaultReapInterval'), '1'); + t.deepEqual(JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')), { + deliveries: 1, + gcKrefs: 20, + computrons: 'never', + }); c.changeKernelOptions({ snapshotInterval: 1000, defaultReapInterval: 10, }); - t.is(kvStore.get('kernel.defaultReapInterval'), '10'); + t.deepEqual(JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')), { + deliveries: 10, + gcKrefs: 20, + computrons: 'never', + }); t.throws(() => c.changeKernelOptions({ defaultReapInterval: 'banana' }), { - message: 'invalid defaultReapInterval value', + message: 'defaultReapInterval = banana', }); t.throws(() => c.changeKernelOptions({ snapshotInterval: 'elephant' }), { message: 'invalid heap snapshotInterval value', @@ -44,6 +52,14 @@ async function testChangeParameters(t) { t.throws(() => c.changeKernelOptions({ baz: 'howdy' }), { message: 'unknown option "baz"', }); + c.changeKernelOptions({ + defaultReapGCKrefs: 77, + }); + t.deepEqual(JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')), { + deliveries: 10, + gcKrefs: 77, + computrons: 'never', + }); async function run(method, args = []) { assert(Array.isArray(args)); @@ -57,7 +73,10 @@ async function testChangeParameters(t) { // setup target vat const [prepStatus] = await run('prepare', []); t.is(prepStatus, 'fulfilled'); - t.is(kvStore.get('v6.reapInterval'), '10'); + // the vat was created without option overrides, so + // reapDirtThreshold will be empty (everything defaults to the + // kernel-wide values) + t.deepEqual(JSON.parse(kvStore.get('v6.options')).reapDirtThreshold, {}); // now fiddle with stuff const [c1Status, c1Result] = await run('change', [{ foo: 47 }]); @@ -71,7 +90,9 @@ async function testChangeParameters(t) { const [c4Status, c4Result] = await run('change', [{ reapInterval: 20 }]); t.is(c4Status, 'fulfilled'); t.is(c4Result, 'ok'); - t.is(kvStore.get('v6.reapInterval'), '20'); + t.deepEqual(JSON.parse(kvStore.get('v6.options')).reapDirtThreshold, { + deliveries: 20, + }); } test('change vat options', async t => { diff --git a/packages/SwingSet/test/clist.test.js b/packages/SwingSet/test/clist.test.js index 10237c76322..1e347f409cf 100644 --- a/packages/SwingSet/test/clist.test.js +++ b/packages/SwingSet/test/clist.test.js @@ -6,15 +6,18 @@ import { initSwingStore } from '@agoric/swing-store'; import { makeDummySlogger } from '../src/kernel/slogger.js'; import makeKernelKeeper from '../src/kernel/state/kernelKeeper.js'; +const CURRENT_VERSION = 1; + test(`clist reachability`, async t => { const slog = makeDummySlogger({}); const kernelStorage = initSwingStore(null).kernelStorage; + kernelStorage.kvStore.set('version', `${CURRENT_VERSION}`); const kk = makeKernelKeeper(kernelStorage, slog); const s = kk.kvStore; kk.createStartingKernelState({ defaultManagerType: 'local' }); const vatID = kk.allocateUnusedVatID(); const source = { bundleID: 'foo' }; - const options = { workerOptions: 'foo', reapInterval: 1 }; + const options = { workerOptions: 'foo', reapDirtThreshold: {} }; kk.createVatState(vatID, source, options); const vk = kk.provideVatKeeper(vatID); @@ -97,12 +100,13 @@ test(`clist reachability`, async t => { test('getImporters', async t => { const slog = makeDummySlogger({}); const kernelStorage = initSwingStore(null).kernelStorage; + kernelStorage.kvStore.set('version', `${CURRENT_VERSION}`); const kk = makeKernelKeeper(kernelStorage, slog); kk.createStartingKernelState({ defaultManagerType: 'local' }); const vatID1 = kk.allocateUnusedVatID(); const source = { bundleID: 'foo' }; - const options = { workerOptions: 'foo', reapInterval: 1 }; + const options = { workerOptions: 'foo', reapDirtThreshold: {} }; kk.createVatState(vatID1, source, options); kk.addDynamicVatID(vatID1); const vk1 = kk.provideVatKeeper(vatID1); diff --git a/packages/SwingSet/test/controller.test.js b/packages/SwingSet/test/controller.test.js index e731182d6fc..6994081b4ea 100644 --- a/packages/SwingSet/test/controller.test.js +++ b/packages/SwingSet/test/controller.test.js @@ -11,6 +11,7 @@ import { initializeSwingset, makeSwingsetController, } from '../src/index.js'; +import makeKernelKeeper from '../src/kernel/state/kernelKeeper.js'; import { checkKT } from './util.js'; const emptyVP = kser({}); @@ -487,3 +488,16 @@ test.serial('bootstrap export', async t => { removeTriple(kt, vattp0, vatTPVatID, 'o+0'); checkKT(t, c, kt); }); + +test('comms vat does not BOYD', async t => { + const config = {}; + const kernelStorage = initSwingStore().kernelStorage; + const controller = await buildVatController(config, [], { kernelStorage }); + t.teardown(controller.shutdown); + const k = makeKernelKeeper(kernelStorage, null); + const commsVatID = k.getVatIDForName('comms'); + t.deepEqual( + JSON.parse(k.kvStore.get(`${commsVatID}.options`)).reapDirtThreshold, + { never: true }, + ); +}); diff --git a/packages/SwingSet/test/kernel.test.js b/packages/SwingSet/test/kernel.test.js index 81e734fd66e..3047154f938 100644 --- a/packages/SwingSet/test/kernel.test.js +++ b/packages/SwingSet/test/kernel.test.js @@ -3,7 +3,7 @@ import { test } from '../tools/prepare-test-env-ava.js'; import { Fail } from '@agoric/assert'; -import { kser, kslot } from '@agoric/kmarshal'; +import { kser, kunser, kslot } from '@agoric/kmarshal'; import buildKernel from '../src/kernel/index.js'; import { initializeKernel } from '../src/controller/initializeKernel.js'; import { makeVatSlot } from '../src/lib/parseVatSlots.js'; @@ -1567,10 +1567,14 @@ test('xs-worker default manager type', async t => { ); }); -async function reapTest(t, freq) { - const kernel = await makeKernel(); +async function reapTest(t, freq, overrideNever) { + const endowments = makeKernelEndowments(); + await initializeKernel({}, endowments.kernelStorage); + const kernel = buildKernel(endowments, {}, {}); await kernel.start(); + const { kernelStorage } = endowments; const log = []; + function setup() { function dispatch(vatDeliverObject) { if (vatDeliverObject[0] === 'startVat') { @@ -1583,6 +1587,20 @@ async function reapTest(t, freq) { await kernel.createTestVat('vat1', setup, {}, { reapInterval: freq }); const vat1 = kernel.vatNameToID('vat1'); t.deepEqual(log, []); + const options = JSON.parse(kernelStorage.kvStore.get(`${vat1}.options`)); + t.deepEqual(options.reapDirtThreshold, { + deliveries: freq, + gcKrefs: 'never', // createTestVat minimizes BOYD + }); + + if (overrideNever) { + // when upgradeSwingset v0->v1 encounters a non-reaping vat (like + // comms), it sets the .options reapDirtThreshold to `{ never: + // true }`, so verify that this inhibits BOYD + options.reapDirtThreshold = { never: true }; + kernelStorage.kvStore.set(`${vat1}.options`, JSON.stringify(options)); + freq = 'never'; + } const vatRoot = kernel.addExport(vat1, 'o+1'); function deliverMessage(ordinal) { @@ -1602,11 +1620,28 @@ async function reapTest(t, freq) { return ['bringOutYourDead']; } - for (let i = 0; i < 100; i += 1) { - deliverMessage(i); - } + t.deepEqual(JSON.parse(kernelStorage.kvStore.get(`${vat1}.reapDirt`)), {}); + deliverMessage(0); // enqueues only t.deepEqual(log, []); await kernel.run(); + + // The first delivery increments dirt.deliveries . If freq=1 that + // will trigger an immediate BOYD and resets the counter, but for + // the slower-interval cases the counter will be left at 1. + + const expected1 = {}; + if (freq !== 'never' && freq > 1) { + expected1.deliveries = 1; + } + t.deepEqual( + JSON.parse(kernelStorage.kvStore.get(`${vat1}.reapDirt`)), + expected1, + ); + + for (let i = 1; i < 100; i += 1) { + deliverMessage(i); // enqueues only + } + await kernel.run(); for (let i = 0; i < 100; i += 1) { t.deepEqual(log.shift(), matchMsg(i)); if (freq !== 'never' && (i + 1) % freq === 0) { @@ -1635,3 +1670,150 @@ test('reap interval 17', async t => { test('reap interval never', async t => { await reapTest(t, 'never'); }); + +test('reap interval override never', async t => { + await reapTest(t, 5, true); +}); + +// Set up two vats, one to export vrefs, the other to import/drop +// them. The first will get a reapDirtThreshold.gcKrefs, and will log +// when the kernel sends it BOYD. + +async function reapGCKrefsTest(t, freq, overrideNever) { + const endowments = makeKernelEndowments(); + await initializeKernel({}, endowments.kernelStorage); + const kernel = buildKernel(endowments, {}, {}); + await kernel.start(); + const { kernelStorage } = endowments; + // note: worker=local, otherwise snapshotInitial/Interval would interfere + + let boyds = 0; + let rxGCkrefs = 0; + let lastExported = 2; + + // vat-under-test, export vrefs on request, watch for BOYDs + function setup1(syscall) { + function dispatch(vatDeliverObject) { + if (vatDeliverObject[0] === 'startVat') { + return; // skip startVat + } + if (vatDeliverObject[0] === 'message') { + // export vrefs, one per message + const target = vatDeliverObject[2].methargs.slots[0]; + const vref = `o+${lastExported}`; + lastExported += 1; + syscall.send(target, kser(['hold', [kslot(vref)]])); + return; + } + if (vatDeliverObject[0] === 'bringOutYourDead') { + boyds += 1; + } + if (vatDeliverObject[0] === 'dropExports') { + rxGCkrefs += vatDeliverObject[1].length; + } + if (vatDeliverObject[0] === 'retireExports') { + rxGCkrefs += vatDeliverObject[1].length; + } + if (vatDeliverObject[0] === 'retireImports') { + rxGCkrefs += vatDeliverObject[1].length; + } + } + return dispatch; + } + const vat1 = await kernel.createTestVat( + 'vat1', + setup1, + {}, + { reapInterval: 'never', reapGCKrefs: freq }, + ); + const v1root = kernel.getRootObject(vat1); + kernel.pinObject(v1root); + + if (overrideNever) { + // when upgradeSwingset v0->v1 encounters a non-reaping vat (like + // comms), it sets the .options reapDirtThreshold to `{ never: + // true }`, so verify that this inhibits BOYD. It is especially + // important that this works against gcKrefs, otherwise we'd be + // BOYDing vat-comms all the time, which is pointless. + const options = JSON.parse(kernelStorage.kvStore.get(`${vat1}.options`)); + options.reapDirtThreshold = { never: true }; + kernelStorage.kvStore.set(`${vat1}.options`, JSON.stringify(options)); + freq = 'never'; + } + + // helper vat, imports vrefs, drops on request + function setup2(syscall) { + const hold = []; + function dispatch(vatDeliverObject) { + if (vatDeliverObject[0] === 'startVat') { + return; // skip startVat + } + if (vatDeliverObject[0] === 'message') { + const [meth, args] = kunser(vatDeliverObject[2].methargs); + if (meth === 'hold') { + for (const vref of vatDeliverObject[2].methargs.slots) { + hold.push(vref); + } + } else { + const [count] = args; + syscall.dropImports(hold.slice(0, count)); + syscall.retireImports(hold.slice(0, count)); + hold.splice(0, count); + } + } + } + return dispatch; + } + const vat2 = await kernel.createTestVat('vat2', setup2, {}); + const v2root = kernel.getRootObject(vat2); + kernel.pinObject(v2root); + + await kernel.run(); + t.is(boyds, 0); + + async function addExport() { + kernel.queueToKref(v1root, `pleaseExport`, [kslot(v2root)], 'none'); + await kernel.run(); + } + + async function doDrop(count) { + kernel.queueToKref(v2root, `drop`, [count], 'none'); + await kernel.run(); + } + + await addExport(); + await addExport(); + t.is(boyds, 0); + // c-list should currently have two krefs exported by the vat + + // now we drop one for every new one we add, and every 'interval'/2 + // we should see a BOYD + + let krefs = 0; + for (let i = 0; i < 10; i += 1) { + await addExport(); + await doDrop(1); + krefs += 2; + t.is(rxGCkrefs, krefs); + if (freq === 'never' || krefs < freq) { + t.is(boyds, 0); + } else { + t.is(boyds, 1); + boyds = 0; + krefs = 0; + rxGCkrefs = 0; + } + } +} + +test('reap gc-krefs 10', async t => { + await reapGCKrefsTest(t, 10); +}); + +test('reap gc-krefs 12', async t => { + await reapGCKrefsTest(t, 12); +}); + +test('reap gc-krefs overrideNever', async t => { + await reapGCKrefsTest(t, 12, true); +}); diff --git a/packages/SwingSet/test/snapshots/state.test.js.md b/packages/SwingSet/test/snapshots/state.test.js.md index 3ab81fd5cd5..5a7cae18e2a 100644 --- a/packages/SwingSet/test/snapshots/state.test.js.md +++ b/packages/SwingSet/test/snapshots/state.test.js.md @@ -8,8 +8,8 @@ Generated by [AVA](https://avajs.dev). > initial state - 'a5d302e6743578ccda03ea386abd49de0a3bf4d7dedda2f69585c663806c30bc' + '2cc47b69a725bb4a2bfca1e2ba2b8625e3a62261acac60e37be95ebc09b1e02e' > expected activityhash - 'f5f1f643f6242a73c79b0437dbab222d34642ea5d047f15aaf5551d5903711d3' + 'c7edd8883ba896276247c1de6391d1cdac3fcc6bfbd1599098dbd367e454b41f' diff --git a/packages/SwingSet/test/snapshots/state.test.js.snap b/packages/SwingSet/test/snapshots/state.test.js.snap index 632a77941e6c1d9757416e97e33d0d9d0ca7357c..0a7a22b266fec1f45e928a1857a0de7f00ee0290 100644 GIT binary patch literal 276 zcmV+v0qg!jRzV7L*p4dF`mug=oPe74e_3nA@kZF7P{K{f zN!f@BiK#}@d^7W%u&i`!pI7O*7r8vrE7aoxZ5a<^uDNF@W9!gmgvui@Z_!#?ZWYdL zS$q4sGDC-#=eyy4xM{Z^9(GTf$a-`Uyq*y)XiNl3G;S_9J&OUSwNpSqZW7~B;w(s0 zL?x3JDS%Q=e=7V>A}`D#1cMNw)81+8NwGK+6)R#OlZYH9WHrY)#lXnMd9kxatI{v& aO)h7lF55=x`91fK+X+7>P-~hX0RRAo7b=y(L@=6+UD>~ z3(<}@JRge)00000000ARkUdTVF%U*KLWr6hxT8Yydi=B7aRO@E&z`YjWwRUM0fiE7 zLQaYzDkRY-uUHRz^u2jBZEf1Hn66MUSlNKNY1`J``ifKE zZ`!{u%uu{M-wb!dmEG-dzk5PpvQA?16w*`*!;*leh|GRwv7lT%Cl+S78s{*D5+hAE z7P7g>f-KANPs0DE8tPc1ry8AiNL13Ct*68p**VALqj!S9)~7lKpoS1e4zr~+j!eI% aH#wiAsIWoN_TKd4I^hSg1)BXJ0RR94fPa+$ diff --git a/packages/SwingSet/test/state.test.js b/packages/SwingSet/test/state.test.js index d1ab7b83d50..3a06071b84e 100644 --- a/packages/SwingSet/test/state.test.js +++ b/packages/SwingSet/test/state.test.js @@ -7,6 +7,7 @@ import { createHash } from 'crypto'; import { kser, kslot } from '@agoric/kmarshal'; import { initSwingStore } from '@agoric/swing-store'; import makeKernelKeeper from '../src/kernel/state/kernelKeeper.js'; +import { upgradeSwingset } from '../src/controller/upgradeSwingset.js'; import { makeKernelStats } from '../src/kernel/state/stats.js'; import { KERNEL_STATS_METRICS } from '../src/kernel/metrics.js'; import { @@ -143,8 +144,11 @@ test('storage helpers', t => { ]); }); +const CURRENT_VERSION = 1; + function buildKeeperStorageInMemory() { const { kernelStorage, debug } = initSwingStore(null); + kernelStorage.kvStore.set('version', `${CURRENT_VERSION}`); return { ...debug, // serialize, dump ...kernelStorage, @@ -181,6 +185,7 @@ test('kernel state', async t => { k.emitCrankHashes(); checkState(t, store.dump, [ + ['version', '1'], ['crankNumber', '0'], ['initialized', 'true'], ['gcActions', '[]'], @@ -197,7 +202,10 @@ test('kernel state', async t => { ['kd.nextID', '30'], ['kp.nextID', '40'], ['kernel.defaultManagerType', 'local'], - ['kernel.defaultReapInterval', '1'], + [ + 'kernel.defaultReapDirtThreshold', + JSON.stringify({ deliveries: 1, gcKrefs: 20, computrons: 'never' }), + ], ['kernel.snapshotInitial', '3'], ['kernel.snapshotInterval', '200'], ['meter.nextID', '1'], @@ -216,6 +224,7 @@ test('kernelKeeper vat names', async t => { k.emitCrankHashes(); checkState(t, store.dump, [ + ['version', '1'], ['crankNumber', '0'], ['gcActions', '[]'], ['runQueue', '[1,1]'], @@ -233,7 +242,10 @@ test('kernelKeeper vat names', async t => { ['vat.name.vatname5', 'v1'], ['vat.name.Frank', 'v2'], ['kernel.defaultManagerType', 'local'], - ['kernel.defaultReapInterval', '1'], + [ + 'kernel.defaultReapDirtThreshold', + JSON.stringify({ deliveries: 1, gcKrefs: 20, computrons: 'never' }), + ], ['kernel.snapshotInitial', '3'], ['kernel.snapshotInterval', '200'], ['meter.nextID', '1'], @@ -266,6 +278,7 @@ test('kernelKeeper device names', async t => { k.emitCrankHashes(); checkState(t, store.dump, [ + ['version', '1'], ['crankNumber', '0'], ['gcActions', '[]'], ['runQueue', '[1,1]'], @@ -283,7 +296,10 @@ test('kernelKeeper device names', async t => { ['device.name.devicename5', 'd7'], ['device.name.Frank', 'd8'], ['kernel.defaultManagerType', 'local'], - ['kernel.defaultReapInterval', '1'], + [ + 'kernel.defaultReapDirtThreshold', + JSON.stringify({ deliveries: 1, gcKrefs: 20, computrons: 'never' }), + ], ['kernel.snapshotInitial', '3'], ['kernel.snapshotInterval', '200'], ['meter.nextID', '1'], @@ -442,6 +458,7 @@ test('kernelKeeper promises', async t => { k.emitCrankHashes(); checkState(t, store.dump, [ + ['version', '1'], ['crankNumber', '0'], ['device.nextID', '7'], ['vat.nextID', '1'], @@ -465,7 +482,10 @@ test('kernelKeeper promises', async t => { [`${ko}.owner`, 'v1'], [`${ko}.refCount`, '1,1'], ['kernel.defaultManagerType', 'local'], - ['kernel.defaultReapInterval', '1'], + [ + 'kernel.defaultReapDirtThreshold', + JSON.stringify({ deliveries: 1, gcKrefs: 20, computrons: 'never' }), + ], ['kernel.snapshotInitial', '3'], ['kernel.snapshotInterval', '200'], ['meter.nextID', '1'], @@ -513,7 +533,7 @@ test('vatKeeper', async t => { k.createStartingKernelState({ defaultManagerType: 'local' }); const v1 = k.allocateVatIDForNameIfNeeded('name1'); const source = { bundleID: 'foo' }; - const options = { workerOptions: 'foo', reapInterval: 1 }; + const options = { workerOptions: 'foo', reapDirtThreshold: {} }; k.createVatState(v1, source, options); const vk = k.provideVatKeeper(v1); @@ -555,7 +575,7 @@ test('vatKeeper.getOptions', async t => { 'b1-00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'; const source = { bundleID }; const workerOptions = { type: 'local' }; - const options = { workerOptions, name: 'fred', reapInterval: 1 }; + const options = { workerOptions, name: 'fred', reapDirtThreshold: {} }; k.createVatState(v1, source, options); const vk = k.provideVatKeeper(v1); @@ -925,3 +945,201 @@ test('stats - can load and save existing stats', t => { t.deepEqual(JSON.parse(getSerializedStats().consensusStats), consensusStats); t.deepEqual(JSON.parse(getSerializedStats().localStats), localStats); }); + +test('vatKeeper dirt counters', async t => { + const store = buildKeeperStorageInMemory(); + const k = makeKernelKeeper(store, null); + k.createStartingKernelState({ + defaultManagerType: 'local', + }); + k.saveStats(); + + // the defaults are designed for testing + t.deepEqual(JSON.parse(k.kvStore.get(`kernel.defaultReapDirtThreshold`)), { + deliveries: 1, + gcKrefs: 20, + computrons: 'never', + }); + + const reapDirtThreshold = { deliveries: 10, gcKrefs: 20, computrons: 100 }; + const never = { deliveries: 'never', gcKrefs: 'never', computrons: 'never' }; + + // a new DB will have empty dirt entries for each vat created + const source = { bundleID: 'foo' }; + const v1 = k.allocateVatIDForNameIfNeeded('name1'); + k.createVatState(v1, source, { workerOptions: 'foo', reapDirtThreshold }); + const vk1 = k.provideVatKeeper(v1); + + const v2 = k.allocateVatIDForNameIfNeeded('name2'); + k.createVatState(v2, source, { workerOptions: 'foo', reapDirtThreshold }); + const vk2 = k.provideVatKeeper(v2); + + const v3 = k.allocateVatIDForNameIfNeeded('name3'); + k.createVatState(v3, source, { + workerOptions: 'foo', + reapDirtThreshold: never, + }); + const vk3 = k.provideVatKeeper(v3); + + // the nominal "all clean" entry is { deliveries: 0, gcKrefs: 0, + // computrons: 0 }, but we only store the non-zero keys, so it's + // really {} + t.deepEqual(vk1.getReapDirt(), {}); + t.deepEqual(vk2.getReapDirt(), {}); + + // our write-through cache should store the initial value in the DB + t.true(store.kvStore.has(`${v1}.reapDirt`)); + t.deepEqual(JSON.parse(store.kvStore.get(`${v1}.reapDirt`)), {}); + + // changing one entry doesn't change any others + vk1.addDirt({ deliveries: 1, gcKrefs: 0, computrons: 12 }); + t.deepEqual(vk1.getReapDirt(), { deliveries: 1, gcKrefs: 0, computrons: 12 }); + t.deepEqual(vk2.getReapDirt(), {}); + t.not(vk1.getReapDirt(), vk2.getReapDirt()); + // and writes through the cache + t.deepEqual(JSON.parse(store.kvStore.get(`${v1}.reapDirt`)), { + deliveries: 1, + gcKrefs: 0, + computrons: 12, + }); + + // clearing the dirt will zero out the entries + vk1.clearReapDirt(); + t.deepEqual(vk1.getReapDirt(), {}); + t.deepEqual(JSON.parse(store.kvStore.get(`${v1}.reapDirt`)), {}); + + // nothing has reached the threshold yet + t.is(k.nextReapAction(), undefined); + + vk1.addDirt({ deliveries: 4 }); + t.deepEqual(vk1.getReapDirt(), { deliveries: 4 }); + t.is(k.nextReapAction(), undefined); + vk1.addDirt({ deliveries: 5 }); + t.deepEqual(vk1.getReapDirt(), { deliveries: 9 }); + t.is(k.nextReapAction(), undefined); + vk1.addDirt({ deliveries: 6 }); + t.deepEqual(vk1.getReapDirt(), { deliveries: 15 }); + t.deepEqual(k.nextReapAction(), { type: 'bringOutYourDead', vatID: v1 }); + t.is(k.nextReapAction(), undefined); + + // dirt is ignored when the threshold is 'never' + vk3.addDirt({ deliveries: 4 }); + t.deepEqual(vk3.getReapDirt(), {}); +}); + +test('dirt upgrade', async t => { + const store = buildKeeperStorageInMemory(); + const k = makeKernelKeeper(store, null); + k.createStartingKernelState({ + defaultManagerType: 'local', + }); + k.saveStats(); + const v1 = k.allocateVatIDForNameIfNeeded('name1'); + const source = { bundleID: 'foo' }; + // actual vats get options.reapDirtThreshold ; we install + // options.reapInterval to simulate the old version, and we use + // nonsense values because .reapInterval was not updated by + // changeVatOptions so the upgrade process should ignore it + const options = { workerOptions: 'foo', reapInterval: 666 }; + k.createVatState(v1, source, options); + // "v2" is like v1 but with the default reapInterval + const v2 = k.allocateVatIDForNameIfNeeded('name2'); + const options2 = { ...options, reapInterval: 667 }; + k.createVatState(v2, source, options2); + // "v3" is like comms: no BOYD + const v3 = k.allocateVatIDForNameIfNeeded('comms'); + const options3 = { ...options, reapInterval: 'never' }; + k.createVatState(v3, source, options3); + + // Test that upgrade from an older version of the DB will populate + // the right keys. We simulate the old version by modifying a + // serialized copy. The old version (on mainnet) had things like: + // * kernel.defaultReapInterval: 1000 + // * v1.options: { ... reapInterval: 1000 } + // * v1.reapCountdown: 123 + // * v1.reapInterval: 1000 + // * v2.options: { ... reapInterval: 300 } + // * v2.reapCountdown: 123 + // * v2.reapInterval: 300 + // * v3.options: { ... reapInterval: 'never' } + // * v3.reapCountdown: 'never' + // * v3.reapInterval: 'never' + + t.is(k.kvStore.get('version'), '1'); + k.kvStore.delete(`kernel.defaultReapDirtThreshold`); + k.kvStore.set(`kernel.defaultReapInterval`, '1000'); + + // v1 uses the default reapInterval + k.kvStore.delete(`${v1}.reapDirt`); + k.kvStore.delete(`${v1}.reapDirtThreshold`); + k.kvStore.set(`${v1}.reapInterval`, '1000'); + k.kvStore.set(`${v1}.reapCountdown`, '700'); + + // v2 uses a custom reapCountdown + k.kvStore.delete(`${v2}.reapDirt`); + k.kvStore.delete(`${v2}.reapDirtThreshold`); + k.kvStore.set(`${v2}.reapInterval`, '300'); + k.kvStore.set(`${v2}.reapCountdown`, '70'); + + // v3 is like comms and never reaps + k.kvStore.delete(`${v3}.reapDirt`); + k.kvStore.delete(`${v3}.reapDirtThreshold`); + k.kvStore.set(`${v3}.reapInterval`, 'never'); + k.kvStore.set(`${v3}.reapCountdown`, 'never'); + + k.kvStore.delete(`version`); + + // kernelKeeper refuses to work with an old state + t.throws(() => duplicateKeeper(store.serialize)); + + // it requires a manual upgrade + let k2; + { + const serialized = store.serialize(); + const { kernelStorage } = initSwingStore(null, { serialized }); + upgradeSwingset(kernelStorage); + k2 = makeKernelKeeper(kernelStorage, null); // works this time + k2.loadStats(); + } + + t.true(k2.kvStore.has(`kernel.defaultReapDirtThreshold`)); + // threshold.deliveries is converted from defaultReapInterval + t.deepEqual(JSON.parse(k2.kvStore.get(`kernel.defaultReapDirtThreshold`)), { + deliveries: 1000, + gcKrefs: 20, + computrons: 'never', + }); + + t.true(k2.kvStore.has(`${v1}.reapDirt`)); + // reapDirt.deliveries computed from old reapInterval-reapCountdown + t.deepEqual(JSON.parse(k2.kvStore.get(`${v1}.reapDirt`)), { + deliveries: 300, + }); + // reapDirtThreshold.deliveries computed from old reapInterval, and + // because it matches the kernel-wide default, the .options record + // is left empty + t.deepEqual( + JSON.parse(k2.kvStore.get(`${v1}.options`)).reapDirtThreshold, + {}, + ); + const vk1New = k2.provideVatKeeper(v1); + t.deepEqual(vk1New.getReapDirt(), { deliveries: 300 }); + + // v2 reapDirt is computed the same way + t.true(k2.kvStore.has(`${v2}.reapDirt`)); + t.deepEqual(JSON.parse(k2.kvStore.get(`${v2}.reapDirt`)), { + deliveries: 230, + }); + // the custom reapInterval is transformed into an .options override + t.deepEqual(JSON.parse(k2.kvStore.get(`${v2}.options`)).reapDirtThreshold, { + deliveries: 300, + }); + const vk2New = k2.provideVatKeeper(v2); + t.deepEqual(vk2New.getReapDirt(), { deliveries: 230 }); + + t.true(k2.kvStore.has(`${v3}.reapDirt`)); + t.deepEqual(JSON.parse(k2.kvStore.get(`${v3}.reapDirt`)), {}); + t.deepEqual(JSON.parse(k2.kvStore.get(`${v3}.options`)).reapDirtThreshold, { + never: true, + }); +}); diff --git a/packages/SwingSet/test/upgrade-swingset.test.js b/packages/SwingSet/test/upgrade-swingset.test.js new file mode 100644 index 00000000000..1a468543bf5 --- /dev/null +++ b/packages/SwingSet/test/upgrade-swingset.test.js @@ -0,0 +1,202 @@ +/* eslint-disable no-underscore-dangle */ +// @ts-nocheck + +import { initSwingStore } from '@agoric/swing-store'; +import { test } from '../tools/prepare-test-env-ava.js'; + +import { + initializeSwingset, + makeSwingsetController, + upgradeSwingset, + buildKernelBundles, +} from '../src/index.js'; + +test.before(async t => { + const kernelBundles = await buildKernelBundles(); + t.context.data = { kernelBundles }; +}); + +test('kernel refuses to run with out-of-date DB', async t => { + const { hostStorage, kernelStorage } = initSwingStore(); + const { commit } = hostStorage; + const { kvStore } = kernelStorage; + const config = {}; + await initializeSwingset(config, [], kernelStorage, t.context.data); + await commit(); + + // now doctor the initial state to make it look like the + // kernelkeeper v0 schema, just deleting the version key + + t.is(kvStore.get('version'), '1'); + kvStore.delete(`version`); + await commit(); + + // Now build a controller around this modified state, which should fail. + await t.throwsAsync(() => makeSwingsetController(kernelStorage), { + message: /kernelStorage is too old/, + }); +}); + +test('upgrade kernel state', async t => { + const { hostStorage, kernelStorage } = initSwingStore(); + const { commit } = hostStorage; + const { kvStore } = kernelStorage; + const config = { + vats: { + one: { + sourceSpec: new URL( + 'files-vattp/bootstrap-test-vattp.js', + import.meta.url, + ).pathname, + }, + }, + }; + await initializeSwingset(config, [], kernelStorage, t.context.data); + await commit(); + + // now doctor the initial state to make it look like the + // kernelkeeper v0 schema, with 'kernel.defaultReapInterval' instead + // of 'kernel.defaultReapDirtThreshold', and + // 'v1.reapCountdown`/`.reapInterval` . This is cribbed from "dirt + // upgrade" in test-state.js. + // + // our mainnet vats have data like: + // v5.options|{"workerOptions":{"type":"xsnap","bundleIDs":["b0-5c790a966210b78de758fb442af542714ed96da09db76e0b31d6a237e555fd62","b0-e0d2dafc7e981947b42118e8c950837109683bae56f7b4f5bffa1b67e5c1e768"]},"name":"timer","enableSetup":false,"enablePipelining":false,"enableDisavow":false,"useTranscript":true,"reapInterval":1000,"critical":false} + // v5.reapCountdown|181 + // v5.reapInterval|1000 + // + // This is a bit fragile.. there are probably ways to refactor + // kernelKeeper to make this better, or at least put all the + // manipulation/simulation code in the same place. + + t.true(kvStore.has('kernel.defaultReapDirtThreshold')); + + t.is(kvStore.get('version'), '1'); + kvStore.delete('version'); // i.e. revert to v0 + kvStore.delete(`kernel.defaultReapDirtThreshold`); + kvStore.set(`kernel.defaultReapInterval`, '300'); + + const vatIDs = {}; + for (const name of JSON.parse(kvStore.get('vat.names'))) { + const vatID = kvStore.get(`vat.name.${name}`); + t.truthy(vatID, name); + vatIDs[name] = vatID; + t.true(kvStore.has(`${vatID}.reapDirt`)); + kvStore.delete(`${vatID}.reapDirt`); + const options = JSON.parse(kvStore.get(`${vatID}.options`)); + t.truthy(options); + t.truthy(options.reapDirtThreshold); + delete options.reapDirtThreshold; + options.reapInterval = 55; // ignored by upgrader, so make it bogus + kvStore.set(`${vatID}.options`, JSON.stringify(options)); + if (name === 'comms') { + kvStore.set(`${vatID}.reapInterval`, 'never'); + kvStore.set(`${vatID}.reapCountdown`, 'never'); + } else { + kvStore.set(`${vatID}.reapInterval`, '100'); + kvStore.set(`${vatID}.reapCountdown`, '70'); + // 100-70 means the new state's dirt.deliveries should be 30 + } + } + + await commit(); + + // confirm that this state is too old for the kernel to use + await t.throwsAsync(() => makeSwingsetController(kernelStorage), { + message: /kernelStorage is too old/, + }); + + // upgrade it + upgradeSwingset(kernelStorage); + + // now we should be good to go + const _controller = await makeSwingsetController(kernelStorage); + + t.true(kvStore.has('kernel.defaultReapDirtThreshold')); + // the kernel-wide threshold gets a .gcKrefs (to meet our upcoming + // slow-deletion goals) + t.deepEqual(JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')), { + computrons: 'never', + deliveries: 300, + gcKrefs: 20, + }); + + // normal vat has some (computed) accumulated dirt + t.deepEqual(JSON.parse(kvStore.get(`${vatIDs.one}.reapDirt`)), { + deliveries: 30, + }); + // anywhere the vat's upgraded threshold differs from the + // kernel-wide threshold, .options gets an override value, in this + // case on deliveries (since 100 !== 300) + t.deepEqual( + JSON.parse(kvStore.get(`${vatIDs.one}.options`)).reapDirtThreshold, + { deliveries: 100 }, + ); + + // comms doesn't reap, and doesn't count dirt, and gets a special + // 'never' marker + t.deepEqual(JSON.parse(kvStore.get(`${vatIDs.comms}.reapDirt`)), {}); + t.deepEqual( + JSON.parse(kvStore.get(`${vatIDs.comms}.options`)).reapDirtThreshold, + { never: true }, + ); + + // TODO examine the state, use it + + // TODO check the export-data callbacks +}); + +test('upgrade non-reaping kernel state', async t => { + const { hostStorage, kernelStorage } = initSwingStore(); + const { commit } = hostStorage; + const { kvStore } = kernelStorage; + const config = {}; + await initializeSwingset(config, [], kernelStorage, t.context.data); + await commit(); + + // now doctor the initial state to make it look like the + // kernelkeeper v0 schema, with 'kernel.defaultReapInterval' of 'never' + + t.true(kvStore.has('kernel.defaultReapDirtThreshold')); + + t.is(kvStore.get('version'), '1'); + kvStore.delete('version'); // i.e. revert to v0 + kvStore.delete(`kernel.defaultReapDirtThreshold`); + kvStore.set(`kernel.defaultReapInterval`, 'never'); + + const vatIDs = {}; + for (const name of JSON.parse(kvStore.get('vat.names'))) { + const vatID = kvStore.get(`vat.name.${name}`); + t.truthy(vatID, name); + vatIDs[name] = vatID; + t.true(kvStore.has(`${vatID}.reapDirt`)); + kvStore.delete(`${vatID}.reapDirt`); + const options = JSON.parse(kvStore.get(`${vatID}.options`)); + t.truthy(options); + t.truthy(options.reapDirtThreshold); + delete options.reapDirtThreshold; + options.reapInterval = 'never'; + kvStore.set(`${vatID}.options`, JSON.stringify(options)); + kvStore.set(`${vatID}.reapInterval`, 'never'); + kvStore.set(`${vatID}.reapCountdown`, 'never'); + } + await commit(); + + // confirm that this state is too old for the kernel to use + await t.throwsAsync(() => makeSwingsetController(kernelStorage), { + message: /kernelStorage is too old/, + }); + + // upgrade it + upgradeSwingset(kernelStorage); + + // now we should be good to go + const _controller = await makeSwingsetController(kernelStorage); + + t.true(kvStore.has('kernel.defaultReapDirtThreshold')); + t.deepEqual(JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')), { + computrons: 'never', + deliveries: 'never', + gcKrefs: 'never', + }); +}); diff --git a/packages/SwingSet/test/vat-admin/bootstrap.js b/packages/SwingSet/test/vat-admin/bootstrap.js index 5b1bcaa15ee..74bf050d9ac 100644 --- a/packages/SwingSet/test/vat-admin/bootstrap.js +++ b/packages/SwingSet/test/vat-admin/bootstrap.js @@ -25,6 +25,14 @@ export function buildRootObject() { return n; }, + async byNameWithOptions(bundleName, opts) { + const { root } = await E(admin).createVatByName(bundleName, { + ...options, + ...opts, + }); + return root; + }, + async byNamedBundleCap(name) { const bcap = await E(admin).getNamedBundleCap(name); const { root } = await E(admin).createVat(bcap, options); diff --git a/packages/SwingSet/test/vat-admin/create-vat.test.js b/packages/SwingSet/test/vat-admin/create-vat.test.js index a23a686752e..40b3325cc58 100644 --- a/packages/SwingSet/test/vat-admin/create-vat.test.js +++ b/packages/SwingSet/test/vat-admin/create-vat.test.js @@ -444,3 +444,40 @@ test('createVat holds refcount', async t => { await stepUntil(() => false); t.deepEqual(kunser(c.kpResolution(kpid)), 0); }); + +test('createVat without options', async t => { + const printSlog = false; + const { c, kernelStorage } = await doTestSetup(t, false, printSlog); + const { kvStore } = kernelStorage; + const threshold = JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')); + t.deepEqual(threshold, { deliveries: 1, gcKrefs: 20, computrons: 'never' }); + + const kpid = c.queueToVatRoot('bootstrap', 'byNameWithOptions', [ + 'new13', + {}, + ]); + await c.run(); + const kref = kunser(c.kpResolution(kpid)).getKref(); + const vatID = kvStore.get(`${kref}.owner`); + const options = JSON.parse(kvStore.get(`${vatID}.options`)); + t.deepEqual(options.reapDirtThreshold, {}); +}); + +test('createVat with options', async t => { + const printSlog = false; + const { c, kernelStorage } = await doTestSetup(t, false, printSlog); + const { kvStore } = kernelStorage; + const threshold = JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')); + t.deepEqual(threshold, { deliveries: 1, gcKrefs: 20, computrons: 'never' }); + + const opts = { reapInterval: 123 }; + const kpid = c.queueToVatRoot('bootstrap', 'byNameWithOptions', [ + 'new13', + opts, + ]); + await c.run(); + const kref = kunser(c.kpResolution(kpid)).getKref(); + const vatID = kvStore.get(`${kref}.owner`); + const options = JSON.parse(kvStore.get(`${vatID}.options`)); + t.deepEqual(options.reapDirtThreshold, { deliveries: 123 }); +});