From 0fe9f39ee24ebd525ac9015d7a608d6faf48a714 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Wed, 27 Mar 2024 10:43:15 -0700 Subject: [PATCH] fix(swingset): use "dirt" to schedule vat reap/bringOutYourDead `dispatch.bringOutYourDead()`, aka "reap", triggers garbage collection inside a vat, and gives it a chance to drop imported c-list vrefs that are no longer referenced by anything inside the vat. Previously, each vat has a configurable parameter named `reapInterval`, which defaults to a kernel-wide `defaultReapInterval` (but can be set separately for each vat). This defaults to 1, mainly for unit testing, but real applications set it to something like 200. This caused BOYD to happen once every 200 deliveries, plus an extra BOYD just before we save an XS heap-state snapshot. This commit switches to a "dirt"-based BOYD scheduler, wherein we consider the vat to get more and more dirty as it does work, and eventually it reaches a `reapDirtThreshold` that triggers the BOYD (which resets the dirt counter). We continue to track `dirt.deliveries` as before, with the same defaults. But we add a new `dirt.gcKrefs` counter, which is incremented by the krefs we submit to the vat in GC deliveries. For example, calling `dispatch.dropImports([kref1, kref2])` would increase `dirt.gcKrefs` by two. The `reapDirtThreshold.gcKrefs` limit defaults to 20. For normal use patterns, this will trigger a BOYD after ten krefs have been dropped and retired. We choose this value to allow the #8928 slow vat termination process to trigger BOYD frequently enough to keep the BOYD cranks small: since these will be happening constantly (in the "background"), we don't want them to take more than 500ms or so. Given the current size of the large vats that #8928 seeks to terminate, 10 krefs seems like a reasonable limit. And of course we don't want to perform too many BOYDs, so `gcKrefs: 20` is about the smallest threshold we'd want to use. External APIs continue to accept `reapInterval`, and now also accept `reapGCKrefs`. * kernel config record * takes `config.defaultReapInterval` and `defaultReapGCKrefs` * takes `vat.NAME.creationOptions.reapInterval` and `.reapGCKrefs` * `controller.changeKernelOptions()` still takes `defaultReapInterval` but now also accepts `defaultReapGCKrefs` The APIs available to userspace code (through `vatAdminSvc`) are unchanged (partially due to upgrade/backwards-compatibility limitations), and continue to only support setting `reapInterval`. Internally, this just modifies `reapDirtThreshold.deliveries`. * `E(vatAdminSvc).createVat(bcap, { reapInterval })` * `E(adminNode).upgrade(bcap, { reapInterval })` * `E(adminNode).changeOptions({ reapInterval })` Internally, the kernel-wide state records `defaultReapDirtThreshold` instead of `defaultReapInterval`, and each vat records `.reapDirtThreshold` in their `vNN.options` key instead of `vNN.reapInterval`. The current dirt level is recorded in `vNN.reapDirt`. The kernel will automatically upgrade both the kernel-wide and the per-vat state upon the first reboot with the new kernel code. The old `reapCountdown` value is used to initialize the vat's `reapDirt.deliveries` counter, so the upgrade shouldn't disrupt the existing schedule. Vats which used `reapInterval = 'never'` (eg comms) will get a `reapDirtThreshold` of all 'never' values, so they continue to inhibit BOYD. Otherwise, all vats get a `threshold.gcKrefs` of 20. We do not track dirt when the corresponding threshold is 'never', to avoid incrementing the comms dirt counters forever. This design leaves room for adding `.computrons` to the dirt record, as well as tracking a separate `snapshotDirt` counter (to trigger XS heap snapshots, ala #6786). We add `reapDirtThreshold.computrons`, but do not yet expose an API to set it. Future work includes: * upgrade vat-vat-admin to let userspace set `reapDirtThreshold` New tests were added to exercise the upgrade process, and other tests were updated to match the new internal initialization pattern. We now reset the dirt counter upon any BOYD, so this also happens to help with #8665 (doing a `reapAllVats()` resets the delivery counters, so future BOYDs will be delayed, which is what we want). But we should still change `controller.reapAllVats()` to avoid BOYDs on vats which haven't received any deliveries. closes #8980 --- .../src/controller/initializeKernel.js | 33 +++- .../src/controller/initializeSwingset.js | 1 + packages/SwingSet/src/kernel/kernel.js | 80 ++++++-- .../SwingSet/src/kernel/state/kernelKeeper.js | 151 ++++++++++++--- .../SwingSet/src/kernel/state/vatKeeper.js | 182 +++++++++++++++--- .../src/kernel/vat-loader/manager-factory.js | 1 - .../src/kernel/vat-loader/vat-loader.js | 2 +- packages/SwingSet/src/lib/recordVatOptions.js | 14 +- packages/SwingSet/src/types-external.js | 32 +-- packages/SwingSet/src/types-internal.js | 49 ++++- .../test-change-parameters.js | 34 +++- .../SwingSet/test/snapshots/test-state.js.md | 4 +- .../test/snapshots/test-state.js.snap | Bin 276 -> 278 bytes packages/SwingSet/test/test-clist.js | 4 +- packages/SwingSet/test/test-controller.js | 14 ++ packages/SwingSet/test/test-kernel.js | 122 +++++++++++- packages/SwingSet/test/test-state.js | 179 ++++++++++++++++- 17 files changed, 789 insertions(+), 113 deletions(-) diff --git a/packages/SwingSet/src/controller/initializeKernel.js b/packages/SwingSet/src/controller/initializeKernel.js index 67552acf6299..d7c81675efa5 100644 --- a/packages/SwingSet/src/controller/initializeKernel.js +++ b/packages/SwingSet/src/controller/initializeKernel.js @@ -9,14 +9,32 @@ import { insistVatID } from '../lib/id.js'; import { makeVatSlot } from '../lib/parseVatSlots.js'; import { insistStorageAPI } from '../lib/storageAPI.js'; import { makeVatOptionRecorder } from '../lib/recordVatOptions.js'; -import makeKernelKeeper from '../kernel/state/kernelKeeper.js'; +import makeKernelKeeper, { + DEFAULT_DELIVERIES_PER_BOYD, + DEFAULT_GC_KREFS_PER_BOYD, +} from '../kernel/state/kernelKeeper.js'; import { exportRootObject } from '../kernel/kernel.js'; import { makeKernelQueueHandler } from '../kernel/kernelQueue.js'; +/** + * @typedef { import('../types-external.js').SwingSetKernelConfig } SwingSetKernelConfig + * @typedef { import('../types-external.js').SwingStoreKernelStorage } SwingStoreKernelStorage + * @typedef { import('../types-internal.js').InternalKernelOptions } InternalKernelOptions + * @typedef { import('../types-internal.js').ReapDirtThreshold } ReapDirtThreshold + */ + function makeVatRootObjectSlot() { return makeVatSlot('object', true, 0); } +/* + * @param {SwingSetKernelConfig} config + * @param {SwingStoreKernelStorage} kernelStorage + * @param {*} [options] + * @returns {Promise} KPID of the bootstrap message + * result promise + */ + export async function initializeKernel(config, kernelStorage, options = {}) { const { verbose = false, @@ -33,14 +51,22 @@ export async function initializeKernel(config, kernelStorage, options = {}) { assert(!wasInitialized); const { defaultManagerType, - defaultReapInterval, + defaultReapInterval = DEFAULT_DELIVERIES_PER_BOYD, + defaultReapGCKrefs = DEFAULT_GC_KREFS_PER_BOYD, relaxDurabilityRules, snapshotInitial, snapshotInterval, } = config; + /** @type { ReapDirtThreshold } */ + const defaultReapDirtThreshold = { + deliveries: defaultReapInterval, + gcKrefs: defaultReapGCKrefs, + computrons: 'never', // TODO no knob? + }; + /** @type { InternalKernelOptions } */ const kernelOptions = { defaultManagerType, - defaultReapInterval, + defaultReapDirtThreshold, relaxDurabilityRules, snapshotInitial, snapshotInterval, @@ -86,6 +112,7 @@ export async function initializeKernel(config, kernelStorage, options = {}) { 'useTranscript', 'critical', 'reapInterval', + 'reapGCKrefs', 'nodeOptions', ]); const vatID = kernelKeeper.allocateVatIDForNameIfNeeded(name); diff --git a/packages/SwingSet/src/controller/initializeSwingset.js b/packages/SwingSet/src/controller/initializeSwingset.js index e3f48c3950ad..49078ec1f416 100644 --- a/packages/SwingSet/src/controller/initializeSwingset.js +++ b/packages/SwingSet/src/controller/initializeSwingset.js @@ -398,6 +398,7 @@ export async function initializeSwingset( managerType: 'local', useTranscript: false, reapInterval: 'never', + reapGCKrefs: 'never', }, }; } diff --git a/packages/SwingSet/src/kernel/kernel.js b/packages/SwingSet/src/kernel/kernel.js index d0f3b65d06b6..79b78a342e02 100644 --- a/packages/SwingSet/src/kernel/kernel.js +++ b/packages/SwingSet/src/kernel/kernel.js @@ -361,6 +361,7 @@ export default function buildKernel( * * @typedef { import('@agoric/swingset-liveslots').MeterConsumption } MeterConsumption * @typedef { import('../types-internal.js').MeterID } MeterID + * @typedef { import('../types-internal.js').Dirt } Dirt * * Any delivery crank (send, notify, start-vat.. anything which is allowed * to make vat delivery) emits one of these status events if a delivery @@ -379,7 +380,7 @@ export default function buildKernel( * didDelivery?: VatID, // we made a delivery to a vat, for run policy and save-snapshot * computrons?: BigInt, // computron count for run policy * meterID?: string, // deduct those computrons from a meter - * decrementReapCount?: { vatID: VatID }, // the reap counter should decrement + * measureDirt?: [ VatID, Dirt ], // the dirt counter should increment * terminate?: { vatID: VatID, reject: boolean, info: SwingSetCapData }, // terminate vat, notify vat-admin * vatAdminMethargs?: RawMethargs, // methargs to notify vat-admin about create/upgrade results * } } CrankResults @@ -446,16 +447,16 @@ export default function buildKernel( * event handler. * * Two flags influence this: - * `decrementReapCount` is used for deliveries that run userspace code + * `measureDirt` is used for non-BOYD deliveries * `meterID` means we should check a meter * * @param {VatID} vatID * @param {DeliveryStatus} status - * @param {boolean} decrementReapCount + * @param {boolean} measureDirt * @param {MeterID} [meterID] * @returns {CrankResults} */ - function deliveryCrankResults(vatID, status, decrementReapCount, meterID) { + function deliveryCrankResults(vatID, status, measureDirt, meterID) { let meterUnderrun = false; let computrons; if (status.metering?.compute) { @@ -499,8 +500,13 @@ export default function buildKernel( results.terminate = { vatID, ...status.vatRequestedTermination }; } - if (decrementReapCount && !(results.abort || results.terminate)) { - results.decrementReapCount = { vatID }; + if (measureDirt && !(results.abort || results.terminate)) { + const dirt = { deliveries: 1 }; + if (computrons) { + // this is BigInt, but we use plain Number in Dirt records + dirt.computrons = Number(computrons); + } + results.measureDirt = [vatID, dirt]; } // We leave results.consumeMessage up to the caller. Send failures @@ -601,6 +607,8 @@ export default function buildKernel( if (!vatWarehouse.lookup(vatID)) { return NO_DELIVERY_CRANK_RESULTS; // can't collect from the dead } + const vatKeeper = kernelKeeper.provideVatKeeper(vatID); + vatKeeper.addDirt({ gcKrefs: krefs.length }); /** @type { KernelDeliveryDropExports | KernelDeliveryRetireExports | KernelDeliveryRetireImports } */ const kd = harden([type, krefs]); if (type === 'retireExports') { @@ -613,7 +621,7 @@ export default function buildKernel( } const vd = vatWarehouse.kernelDeliveryToVatDelivery(vatID, kd); const status = await deliverAndLogToVat(vatID, kd, vd); - return deliveryCrankResults(vatID, status, false); // no meterID + return deliveryCrankResults(vatID, status, true); // no meterID } /** @@ -628,11 +636,13 @@ export default function buildKernel( if (!vatWarehouse.lookup(vatID)) { return NO_DELIVERY_CRANK_RESULTS; // can't collect from the dead } + const vatKeeper = kernelKeeper.provideVatKeeper(vatID); /** @type { KernelDeliveryBringOutYourDead } */ const kd = harden([type]); const vd = vatWarehouse.kernelDeliveryToVatDelivery(vatID, kd); const status = await deliverAndLogToVat(vatID, kd, vd); - return deliveryCrankResults(vatID, status, false); // no meter + vatKeeper.clearReapDirt(); // BOYD zeros out the when-to-BOYD counters + return deliveryCrankResults(vatID, status, false); // no meter, BOYD clears dirt } /** @@ -739,9 +749,17 @@ export default function buildKernel( function setKernelVatOption(vatID, option, value) { switch (option) { case 'reapInterval': { + // This still controls reapDirtThreshold.deliveries, and we do not + // yet offer controls for the other limits (gcKrefs or computrons). if (value === 'never' || isNat(value)) { const vatKeeper = kernelKeeper.provideVatKeeper(vatID); - vatKeeper.updateReapInterval(value); + const threshold = { ...vatKeeper.getReapDirtThreshold() }; + if (value === 'never') { + threshold.deliveries = value; + } else { + threshold.deliveries = Number(value); + } + vatKeeper.setReapDirtThreshold(threshold); } else { console.log(`WARNING: invalid reapInterval value`, value); } @@ -877,6 +895,7 @@ export default function buildKernel( const boydVD = vatWarehouse.kernelDeliveryToVatDelivery(vatID, boydKD); const boydStatus = await deliverAndLogToVat(vatID, boydKD, boydVD); const boydResults = deliveryCrankResults(vatID, boydStatus, false); + vatKeeper.clearReapDirt(); // we don't meter bringOutYourDead since no user code is running, but we // still report computrons to the runPolicy @@ -951,7 +970,7 @@ export default function buildKernel( startVatKD, startVatVD, ); - const startVatResults = deliveryCrankResults(vatID, startVatStatus, false); + const startVatResults = deliveryCrankResults(vatID, startVatStatus, true); computrons = addComputrons(computrons, startVatResults.computrons); if (startVatResults.terminate) { @@ -1292,13 +1311,11 @@ export default function buildKernel( } } } - if (crankResults.decrementReapCount) { + if (crankResults.measureDirt) { // deliveries cause garbage, garbage needs collection - const { vatID } = crankResults.decrementReapCount; + const [vatID, dirt] = crankResults.measureDirt; const vatKeeper = kernelKeeper.provideVatKeeper(vatID); - if (vatKeeper.countdownToReap()) { - kernelKeeper.scheduleReap(vatID); - } + vatKeeper.addDirt(dirt); // might schedule a reap for that vat } // Vat termination (during delivery) is triggered by an illegal @@ -1572,10 +1589,12 @@ export default function buildKernel( 'bundleID', 'enablePipelining', 'reapInterval', + 'reapGCKrefs', ]); const { bundleID = 'b1-00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000', reapInterval = 'never', + reapGCKrefs = 'never', enablePipelining, } = creationOptions; const vatID = kernelKeeper.allocateVatIDForNameIfNeeded(name); @@ -1587,6 +1606,7 @@ export default function buildKernel( const options = { name, reapInterval, + reapGCKrefs, enablePipelining, managerType, }; @@ -1728,14 +1748,38 @@ export default function buildKernel( } function changeKernelOptions(options) { - assertKnownOptions(options, ['defaultReapInterval', 'snapshotInterval']); + assertKnownOptions(options, [ + 'defaultReapInterval', + 'defaultReapGCKrefs', + 'snapshotInterval', + ]); kernelKeeper.startCrank(); try { for (const option of Object.getOwnPropertyNames(options)) { const value = options[option]; switch (option) { case 'defaultReapInterval': { - kernelKeeper.setDefaultReapInterval(value); + if (typeof value === 'number') { + assert(value > 0, `defaultReapInterval = ${value}`); + } else { + assert.equal(value, 'never', `defaultReapInterval = ${value}`); + } + kernelKeeper.setDefaultReapDirtThreshold({ + ...kernelKeeper.getDefaultReapDirtThreshold(), + deliveries: value, + }); + break; + } + case 'defaultReapGCKrefs': { + if (typeof value === 'number') { + assert(value > 0, `defaultReapGCKrefs = ${value}`); + } else { + assert.equal(value, 'never', `defaultReapGCKrefs = ${value}`); + } + kernelKeeper.setDefaultReapDirtThreshold({ + ...kernelKeeper.getDefaultReapDirtThreshold(), + gcKrefs: value, + }); break; } case 'snapshotInterval': { @@ -1768,6 +1812,7 @@ export default function buildKernel( if (!started) { throw Error('must do kernel.start() before step()'); } + kernelKeeper.maybeUpgradeKernelState(); kernelKeeper.startCrank(); await null; try { @@ -1805,6 +1850,7 @@ export default function buildKernel( let count = 0; await null; for (;;) { + kernelKeeper.maybeUpgradeKernelState(); kernelKeeper.startCrank(); try { kernelKeeper.establishCrankSavepoint('start'); diff --git a/packages/SwingSet/src/kernel/state/kernelKeeper.js b/packages/SwingSet/src/kernel/state/kernelKeeper.js index 7b24e6c91967..fd6d79533f70 100644 --- a/packages/SwingSet/src/kernel/state/kernelKeeper.js +++ b/packages/SwingSet/src/kernel/state/kernelKeeper.js @@ -1,6 +1,11 @@ +/* eslint-disable no-use-before-define */ import { Nat, isNat } from '@endo/nat'; import { assert, Fail } from '@agoric/assert'; -import { initializeVatState, makeVatKeeper } from './vatKeeper.js'; +import { + initializeVatState, + maybeUpgradeVatState, + makeVatKeeper, +} from './vatKeeper.js'; import { initializeDeviceState, makeDeviceKeeper } from './deviceKeeper.js'; import { parseReachableAndVatSlot } from './reachable.js'; import { insistStorageAPI } from '../../lib/storageAPI.js'; @@ -33,12 +38,13 @@ const enableKernelGC = true; * @typedef { import('../../types-external.js').BundleCap } BundleCap * @typedef { import('../../types-external.js').BundleID } BundleID * @typedef { import('../../types-external.js').EndoZipBase64Bundle } EndoZipBase64Bundle - * @typedef { import('../../types-external.js').KernelOptions } KernelOptions * @typedef { import('../../types-external.js').KernelSlog } KernelSlog * @typedef { import('../../types-external.js').ManagerType } ManagerType * @typedef { import('../../types-external.js').SnapStore } SnapStore * @typedef { import('../../types-external.js').TranscriptStore } TranscriptStore * @typedef { import('../../types-external.js').VatKeeper } VatKeeper + * @typedef { import('../../types-internal.js').InternalKernelOptions } InternalKernelOptions + * @typedef { import('../../types-internal.js').ReapDirtThreshold } ReapDirtThreshold */ // Kernel state lives in a key-value store supporting key retrieval by @@ -68,7 +74,8 @@ const enableKernelGC = true; // bundle.$BUNDLEID = JSON(bundle) // // kernel.defaultManagerType = managerType -// kernel.defaultReapInterval = $NN +// (old) kernel.defaultReapInterval = $NN +// kernel.defaultReapDirtThreshold = JSON({ deliveries, gcKrefs, computrons }) // number or 'never' // kernel.relaxDurabilityRules = missing | 'true' // kernel.snapshotInitial = $NN // kernel.snapshotInterval = $NN @@ -84,8 +91,11 @@ const enableKernelGC = true; // v$NN.c.$vatSlot = $kernelSlot = ko$NN/kp$NN/kd$NN // v$NN.vs.$key = string // v$NN.meter = m$NN // XXX does this exist? -// v$NN.reapInterval = $NN or 'never' -// v$NN.reapCountdown = $NN or 'never' +// old: v$NN.reapInterval = $NN or 'never' +// old: v$NN.reapCountdown = $NN or 'never' +// v$NN.reapDirt = JSON({ deliveries, gcKrefs, computrons }) // missing keys treated as zero +// v$NN.reapDirtThreshold = JSON({ deliveries, gcKrefs, computrons }) // number or 'never' +// (leave room for v$NN.snapshotDirt and .snapshotDirtThreshold for #6786) // exclude from consensus // local.* @@ -132,6 +142,21 @@ const enableKernelGC = true; // Prefix reserved for host written data: // host. +// Kernel state schemas. These are nominal versions: we don't actually +// record these version numbers in the kvStore anywhere, but if we +// were more strict/organized, it would be in kvStore.get('version'). +// We call maybeUpgradeKernelState() on every step, once per reboot it +// will perform an upgrade by looking at the individual keys and +// making whatever adjustments seem necessary. That code also calls +// maybeUpgradeVatState(), to do the same thing for each vat's state. +// +// v0: the original +// v1: replace `kernel.defaultReapInterval` with `kernel.defaultReapDirtThreshold` +// replace vat's `vNN.reapInterval`/`vNN.reapCountdown` with `vNN.reapDirt` +// and a `vNN.reapDirtThreshold` in `vNN.options` + +const defaultReapDirtThresholdKey = 'kernel.defaultReapDirtThreshold'; + export function commaSplit(s) { if (s === '') { return []; @@ -163,6 +188,21 @@ const FIRST_PROMISE_ID = 40n; const FIRST_CRANK_NUMBER = 0n; const FIRST_METER_ID = 1n; +// this default "reap interval" is low for the benefit of tests: +// applications should set it to something higher (perhaps 200) based +// on their expected usage + +export const DEFAULT_DELIVERIES_PER_BOYD = 1; + +// "20" will trigger a BOYD after 10 krefs are dropped and retired +// (drops and retires are delivered in separate messages, so +// 10+10=20). The worst case work-expansion we've seen is in #8401, +// where one drop breaks one cycle, and each cycle's cleanup causes 50 +// syscalls in the next v9-zoe BOYD. So this should limit each BOYD +// to cleaning 10 cycles, in 500 syscalls. + +export const DEFAULT_GC_KREFS_PER_BOYD = 20; + /** * @param {SwingStoreKernelStorage} kernelStorage * @param {KernelSlog|null} kernelSlog @@ -290,12 +330,16 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { } /** - * @param {KernelOptions} kernelOptions + * @param {InternalKernelOptions} kernelOptions */ function createStartingKernelState(kernelOptions) { const { defaultManagerType = 'local', - defaultReapInterval = 1, + defaultReapDirtThreshold = { + deliveries: DEFAULT_DELIVERIES_PER_BOYD, + gcKrefs: DEFAULT_GC_KREFS_PER_BOYD, + computrons: 'never', + }, relaxDurabilityRules = false, snapshotInitial = 3, snapshotInterval = 200, @@ -317,7 +361,10 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { initQueue('acceptanceQueue'); kvStore.set('crankNumber', `${FIRST_CRANK_NUMBER}`); kvStore.set('kernel.defaultManagerType', defaultManagerType); - kvStore.set('kernel.defaultReapInterval', `${defaultReapInterval}`); + kvStore.set( + defaultReapDirtThresholdKey, + JSON.stringify(defaultReapDirtThreshold), + ); kvStore.set('kernel.snapshotInitial', `${snapshotInitial}`); kvStore.set('kernel.snapshotInterval', `${snapshotInterval}`); if (relaxDurabilityRules) { @@ -327,6 +374,55 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { initializeStats(); } + // maybeUpgradeKernelState() can be called multiple times; this flag + // helps avoid extra "are we upgraded" DB queries + let upgraded = false; + + // the caller is responsible for ensuring that any changes made here + // get committed at a useful point + function maybeUpgradeKernelState() { + if (upgraded) { + return; + } + // upgrade from old kernel.defaultReapInterval + + const oldDefaultReapIntervalKey = 'kernel.defaultReapInterval'; + if (!kvStore.has(defaultReapDirtThresholdKey)) { + // We pretend that we wanted this kref/BOYD trigger all + // along. By upgrading the kernel's recorded default here, the + // subsequent maybeUpgradeVatState() calls will upgrade each vat + // to this new value. If we waited for the host app to call + // controller.setKernelOptions(), the vats would be upgraded + // without a "gcKrefs", and would not trigger BOYDs the way we + // want. + /** @type ReapDirtThreshold */ + const threshold = { + deliveries: 'never', + gcKrefs: DEFAULT_GC_KREFS_PER_BOYD, + computrons: 'never', + }; + + const oldValue = getRequired(oldDefaultReapIntervalKey); + if (oldValue !== 'never') { + const value = Number.parseInt(oldValue, 10); + assert.typeof(value, 'number'); + threshold.deliveries = value; + } + kvStore.set(defaultReapDirtThresholdKey, JSON.stringify(threshold)); + kvStore.delete(oldDefaultReapIntervalKey); + } + + // upgrade all vats + for (const [_name, vatID] of getStaticVats()) { + maybeUpgradeVatState(vatID, kvStore, getDefaultReapDirtThreshold); + } + for (const vatID of getDynamicVats()) { + maybeUpgradeVatState(vatID, kvStore, getDefaultReapDirtThreshold); + } + + upgraded = true; // inhibit more work until next reboot + } + /** * * @param {string} mt @@ -352,21 +448,26 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { /** * - * @returns {number | 'never'} + * @returns { ReapDirtThreshold } */ - function getDefaultReapInterval() { - const r = getRequired('kernel.defaultReapInterval'); - const ri = r === 'never' ? r : Number.parseInt(r, 10); - assert(ri === 'never' || typeof ri === 'number', `k.dri is '${ri}'`); - return ri; + function getDefaultReapDirtThreshold() { + return JSON.parse(getRequired(defaultReapDirtThresholdKey)); } - function setDefaultReapInterval(interval) { - assert( - interval === 'never' || isNat(interval), - 'invalid defaultReapInterval value', - ); - kvStore.set('kernel.defaultReapInterval', `${interval}`); + /** + * @param { ReapDirtThreshold } threshold + */ + function setDefaultReapDirtThreshold(threshold) { + assert.typeof(threshold, 'object'); + assert(threshold); + for (const [key, value] of Object.entries(threshold)) { + if (typeof value === 'number') { + assert(value > 0, `threshold[${key}] = ${value}`); + } else { + assert.equal(value, 'never', `threshold[${key}] = ${value}`); + } + } + kvStore.set(defaultReapDirtThresholdKey, JSON.stringify(threshold)); } function getNat(key) { @@ -764,7 +865,6 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { let idx = 0; for (const dataSlot of capdata.slots) { - // eslint-disable-next-line no-use-before-define incrementRefCount(dataSlot, `resolve|${kernelSlot}|s${idx}`); idx += 1; } @@ -787,7 +887,6 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { function cleanupAfterTerminatedVat(vatID) { insistVatID(vatID); - // eslint-disable-next-line no-use-before-define const vatKeeper = provideVatKeeper(vatID); const exportPrefix = `${vatID}.c.o+`; const importPrefix = `${vatID}.c.o-`; @@ -1262,7 +1361,6 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { if (reachable === 0) { const ownerVatID = ownerOfKernelObject(kref); if (ownerVatID) { - // eslint-disable-next-line no-use-before-define const vatKeeper = provideVatKeeper(ownerVatID); const isReachable = vatKeeper.getReachableFlag(kref); if (isReachable) { @@ -1316,6 +1414,7 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { incStat, decStat, getCrankNumber, + scheduleReap, snapStore, ); ephemeral.vatKeepers.set(vatID, vk); @@ -1535,10 +1634,12 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) { getInitialized, setInitialized, createStartingKernelState, + maybeUpgradeKernelState, getDefaultManagerType, - getDefaultReapInterval, getRelaxDurabilityRules, - setDefaultReapInterval, + getDefaultReapDirtThreshold, + setDefaultReapDirtThreshold, + getSnapshotInitial, getSnapshotInterval, setSnapshotInterval, diff --git a/packages/SwingSet/src/kernel/state/vatKeeper.js b/packages/SwingSet/src/kernel/state/vatKeeper.js index 33769b873004..7cd7ff51ef58 100644 --- a/packages/SwingSet/src/kernel/state/vatKeeper.js +++ b/packages/SwingSet/src/kernel/state/vatKeeper.js @@ -1,7 +1,9 @@ +/* eslint-disable no-use-before-define */ + /** * Kernel's keeper of persistent state for a vat. */ -import { Nat, isNat } from '@endo/nat'; +import { Nat } from '@endo/nat'; import { assert, q, Fail } from '@agoric/assert'; import { parseKernelSlot } from '../parseKernelSlots.js'; import { makeVatSlot, parseVatSlot } from '../../lib/parseVatSlots.js'; @@ -18,7 +20,9 @@ import { enumeratePrefixedKeys } from './storageHelper.js'; * @typedef { import('../../types-external.js').SnapStore } SnapStore * @typedef { import('../../types-external.js').SourceOfBundle } SourceOfBundle * @typedef { import('../../types-external.js').TranscriptStore } TranscriptStore + * @typedef { import('../../types-internal.js').Dirt } Dirt * @typedef { import('../../types-internal.js').VatManager } VatManager + * @typedef { import('../../types-internal.js').ReapDirtThreshold } ReapDirtThreshold * @typedef { import('../../types-internal.js').RecordedVatOptions } RecordedVatOptions * @typedef { import('../../types-internal.js').TranscriptEntry } TranscriptEntry * @import {TranscriptDeliverySaveSnapshot} from '../../types-internal.js' @@ -53,19 +57,82 @@ export function initializeVatState( assert(source); assert('bundle' in source || 'bundleName' in source || 'bundleID' in source); assert.typeof(options, 'object'); - const count = options.reapInterval; - assert(count === 'never' || isNat(count), `bad reapCountdown ${count}`); kvStore.set(`${vatID}.o.nextID`, `${FIRST_OBJECT_ID}`); kvStore.set(`${vatID}.p.nextID`, `${FIRST_PROMISE_ID}`); kvStore.set(`${vatID}.d.nextID`, `${FIRST_DEVICE_ID}`); + kvStore.set(`${vatID}.reapDirt`, JSON.stringify({})); kvStore.set(`${vatID}.source`, JSON.stringify(source)); kvStore.set(`${vatID}.options`, JSON.stringify(options)); - kvStore.set(`${vatID}.reapInterval`, `${count}`); - kvStore.set(`${vatID}.reapCountdown`, `${count}`); transcriptStore.initTranscript(vatID); } +export function maybeUpgradeVatState( + vatID, + kvStore, + getDefaultReapDirtThreshold, +) { + // This is called, once per vat, by maybeUpgradeKernelState. + + // schema v0->v1: if the vat state has `reapInterval` and + // `reapCountdown` (and a .reapInterval in `options`) , replace them + // with `reapDirt` (and a .reapDirtThreshold in `options`) + + const reapDirtKey = `${vatID}.reapDirt`; + + if (!kvStore.has(reapDirtKey)) { + // initialize or upgrade state + const reapDirt = {}; // all missing keys are treated as zero + const threshold = { ...getDefaultReapDirtThreshold() }; + + // the old version also had vNN.options = { reapInterval } , but + // it was not updated by processChangeVatOptions, so only read the + // one from vNN.reapInterval + + const vatOptionsKey = `${vatID}.options`; + const oldReapIntervalKey = `${vatID}.reapInterval`; + const oldReapCountdownKey = `${vatID}.reapCountdown`; + + const reapIntervalString = kvStore.get(oldReapIntervalKey); + const reapCountdownString = kvStore.get(oldReapCountdownKey); + if ( + reapIntervalString && + reapIntervalString !== 'never' && + reapCountdownString && + reapCountdownString !== 'never' + ) { + // deduce delivery count from old countdown values + const reapInterval = Number.parseInt(reapIntervalString, 10); + const reapCountdown = Number.parseInt(reapCountdownString, 10); + const deliveries = reapInterval - reapCountdown; + reapDirt.deliveries = Math.max(deliveries, 0); // just in case + threshold.deliveries = reapInterval; + } + + // old vats that were never reaped (eg comms) used + // reapInterval='never', so respect that and set the other + // threshold values to never as well + if (reapIntervalString === 'never') { + threshold.deliveries = 'never'; + threshold.gcKrefs = 'never'; + threshold.computrons = 'never'; + } + if (reapIntervalString) { + kvStore.delete(oldReapIntervalKey); + } + if (reapCountdownString) { + kvStore.delete(oldReapCountdownKey); + } + kvStore.set(reapDirtKey, JSON.stringify(reapDirt)); + + // remove .reapInterval from options, replace with .reapDirtThreshold + const options = JSON.parse(kvStore.get(vatOptionsKey)); + delete options.reapInterval; + options.reapDirtThreshold = threshold; + kvStore.set(vatOptionsKey, JSON.stringify(options)); + } +} + /** * Produce a vat keeper for a vat. * @@ -87,6 +154,7 @@ export function initializeVatState( * @param {*} incStat * @param {*} decStat * @param {*} getCrankNumber + * @param {*} scheduleReap * @param {SnapStore} [snapStore] * returns an object to hold and access the kernel's state for the given vat */ @@ -107,10 +175,16 @@ export function makeVatKeeper( incStat, decStat, getCrankNumber, + scheduleReap, snapStore = undefined, ) { insistVatID(vatID); + // note: calling makeVatKeeper() does not change the DB. Any + // initialization or upgrade must be complete before it is + // called. Only the methods returned by makeVatKeeper() will change + // the DB. + function getRequired(key) { const value = kvStore.get(key); if (value === undefined) { @@ -119,6 +193,12 @@ export function makeVatKeeper( return value; } + // We keep the vat's reap-dirt state (and threshold) in a + // write-through cache, and populate the cache at vatKeeper startup. + const reapDirtKey = `${vatID}.reapDirt`; + let reapDirt = JSON.parse(getRequired(reapDirtKey)); + let reapDirtThreshold = getOptions().reapDirtThreshold; + /** * @param {SourceOfBundle} source * @param {RecordedVatOptions} options @@ -148,33 +228,70 @@ export function makeVatKeeper( return harden(options); } - function updateReapInterval(reapInterval) { - reapInterval === 'never' || - isNat(reapInterval) || - Fail`bad reapInterval ${reapInterval}`; - kvStore.set(`${vatID}.reapInterval`, `${reapInterval}`); - if (reapInterval === 'never') { - kvStore.set(`${vatID}.reapCountdown`, 'never'); - } - } + // This is named "addDirt" because it should increment all dirt + // counters (both for reap/BOYD and for heap snapshotting). We don't + // have `heapSnapshotDirt` yet, but when we do, it should get + // incremented here. - function countdownToReap() { - const rawCount = getRequired(`${vatID}.reapCountdown`); - if (rawCount === 'never') { - return false; - } else { - const count = Number.parseInt(rawCount, 10); - if (count === 1) { - kvStore.set( - `${vatID}.reapCountdown`, - getRequired(`${vatID}.reapInterval`), - ); - return true; - } else { - kvStore.set(`${vatID}.reapCountdown`, `${count - 1}`); - return false; + /** + * Add some "dirt" to the vat, possibly triggering a reap/BOYD. + * + * @param {Dirt} moreDirt + */ + function addDirt(moreDirt) { + // this references our cached 'reapDirt' and 'reapDirtThreshold' + assert.typeof(moreDirt, 'object'); + let reap = false; + for (const key of Object.keys(moreDirt)) { + const threshold = reapDirtThreshold[key]; + // Don't accumulate dirt if it can't eventually trigger a + // BOYD. This is mainly to keep comms from counting upwards + // forever. TODO revisit this when we add heapSnapshotDirt, + // maybe check both thresholds and accumulate the dirt if either + // one is non-'never'. + if (threshold && threshold !== 'never') { + const oldDirt = reapDirt[key] || 0; + // The 'moreDirt' value might be Number or BigInt (eg + // .computrons). We coerce to Number so we can JSON-stringify. + const newDirt = oldDirt + Number(moreDirt[key]); + reapDirt[key] = newDirt; + if (newDirt >= threshold) { + reap = true; + } } } + kvStore.set(reapDirtKey, JSON.stringify(reapDirt)); + if (reap) { + scheduleReap(vatID); + } + } + + function getReapDirt() { + return reapDirt; + } + + function clearReapDirt() { + // This is only called after a BOYD, so it should only clear the + // reap/BOYD counters. If/when we add heap-snapshot counters, + // those should get cleared in a separate clearHeapSnapshotDirt() + // function. + reapDirt = {}; + kvStore.set(reapDirtKey, JSON.stringify(reapDirt)); + } + + function getReapDirtThreshold() { + return reapDirtThreshold; + } + + /** + * @param {ReapDirtThreshold} threshold + */ + function setReapDirtThreshold(threshold) { + assert.typeof(threshold, 'object'); + reapDirtThreshold = harden({ ...threshold }); + const { ...options } = getOptions(); + options.reapDirtThreshold = threshold; + kvStore.set(`${vatID}.options`, JSON.stringify(options)); } function nextDeliveryNum() { @@ -669,8 +786,11 @@ export function makeVatKeeper( setSourceAndOptions, getSourceAndOptions, getOptions, - countdownToReap, - updateReapInterval, + addDirt, + getReapDirt, + clearReapDirt, + getReapDirtThreshold, + setReapDirtThreshold, nextDeliveryNum, getIncarnationNumber, importsKernelSlot, diff --git a/packages/SwingSet/src/kernel/vat-loader/manager-factory.js b/packages/SwingSet/src/kernel/vat-loader/manager-factory.js index 56160da9254e..571cb868af54 100644 --- a/packages/SwingSet/src/kernel/vat-loader/manager-factory.js +++ b/packages/SwingSet/src/kernel/vat-loader/manager-factory.js @@ -45,7 +45,6 @@ export function makeVatManagerFactory({ 'enableSetup', 'useTranscript', 'critical', - 'reapInterval', 'sourcedConsole', 'name', ]); diff --git a/packages/SwingSet/src/kernel/vat-loader/vat-loader.js b/packages/SwingSet/src/kernel/vat-loader/vat-loader.js index a83022673d11..04e3e7a41509 100644 --- a/packages/SwingSet/src/kernel/vat-loader/vat-loader.js +++ b/packages/SwingSet/src/kernel/vat-loader/vat-loader.js @@ -26,7 +26,7 @@ export function makeVatLoader(stuff) { 'enablePipelining', 'useTranscript', 'critical', - 'reapInterval', + 'reapDirtThreshold', ]; /** diff --git a/packages/SwingSet/src/lib/recordVatOptions.js b/packages/SwingSet/src/lib/recordVatOptions.js index 9e87fca1200e..151231db14e9 100644 --- a/packages/SwingSet/src/lib/recordVatOptions.js +++ b/packages/SwingSet/src/lib/recordVatOptions.js @@ -10,7 +10,8 @@ export const makeVatOptionRecorder = (kernelKeeper, bundleHandler) => { enablePipelining = false, enableDisavow = false, useTranscript = true, - reapInterval = kernelKeeper.getDefaultReapInterval(), + reapInterval, + reapGCKrefs, critical = false, meterID = undefined, managerType = kernelKeeper.getDefaultManagerType(), @@ -21,6 +22,14 @@ export const makeVatOptionRecorder = (kernelKeeper, bundleHandler) => { if (unused.length) { Fail`OptionRecorder: ${vatID} unused options ${unused.join(',')}`; } + const reapDirtThreshold = { ...kernelKeeper.getDefaultReapDirtThreshold() }; + if (reapInterval !== undefined) { + reapDirtThreshold.deliveries = reapInterval; + } + if (reapGCKrefs !== undefined) { + reapDirtThreshold.gcKrefs = reapGCKrefs; + } + // TODO no computrons knob? const workerOptions = await makeWorkerOptions( managerType, bundleHandler, @@ -35,10 +44,11 @@ export const makeVatOptionRecorder = (kernelKeeper, bundleHandler) => { enablePipelining, enableDisavow, useTranscript, - reapInterval, + reapDirtThreshold, critical, meterID, }); + // want vNN.options to be in place before provideVatKeeper, so it can cache reapDirtThreshold in RAM, so: kernelKeeper.createVatState(vatID, source, vatOptions); }; diff --git a/packages/SwingSet/src/types-external.js b/packages/SwingSet/src/types-external.js index b27d225eacb7..c8f2233fb83d 100644 --- a/packages/SwingSet/src/types-external.js +++ b/packages/SwingSet/src/types-external.js @@ -23,14 +23,14 @@ export {}; */ /** - * @typedef {{ - * defaultManagerType?: ManagerType, - * defaultReapInterval?: number | 'never', - * relaxDurabilityRules?: boolean, - * snapshotInitial?: number, - * snapshotInterval?: number, - * pinBootstrapRoot?: boolean, - * }} KernelOptions + * @typedef { object } KernelOptions + * @property { ManagerType } [defaultManagerType] + * @property { number | 'never' } [defaultReapGCKrefs] + * @property { number | 'never' } [defaultReapInterval] + * @property { boolean } [relaxDurabilityRules] + * @property { number } [snapshotInitial] + * @property { number } [snapshotInterval] + * @property { boolean } [pinBootstrapRoot] */ /** @@ -287,13 +287,15 @@ export {}; * reconstructed via replay. If false, no such record is kept. * Defaults to true. * @property { number | 'never' } [reapInterval] - * The interval (measured in number of deliveries to the vat) - * after which the kernel will deliver the 'bringOutYourDead' - * directive to the vat. If the value is 'never', - * 'bringOutYourDead' will never be delivered and the vat will - * be responsible for internally managing (in a deterministic - * manner) any visible effects of garbage collection. Defaults - * to the kernel's configured 'defaultReapInterval' value. + * Trigger a bringOutYourDead after the vat has received + * this many deliveries. If the value is 'never', + * 'bringOutYourDead' will not be triggered by a delivery + * count (but might be triggered for other reasons). + * @property { number | 'never' } [reapGCKrefs] + * Trigger a bringOutYourDead when the vat has been given + * this many krefs in GC deliveries (dropImports, + * retireImports, retireExports). If the value is 'never', + * GC deliveries and their krefs are not treated specially. * @property { boolean } [critical] */ diff --git a/packages/SwingSet/src/types-internal.js b/packages/SwingSet/src/types-internal.js index 9140db5b5ecf..e135ff848106 100644 --- a/packages/SwingSet/src/types-internal.js +++ b/packages/SwingSet/src/types-internal.js @@ -1,6 +1,19 @@ export {}; /** + * The host provides (external) KernelOptions as part of the + * SwingSetConfig record it passes to initializeSwingset(). This + * internal type represents the modified form passed to + * initializeKernel() and kernelKeeper.createStartingKernelState . + * + * @typedef { object } InternalKernelOptions + * @property { ManagerType } [defaultManagerType] + * @property { ReapDirtThreshold } [defaultReapDirtThreshold] + * @property { boolean } [relaxDurabilityRules] + * @property { number } [snapshotInitial] + * @property { number } [snapshotInterval] + * + * * The internal data that controls which worker we use (and how we use it) is * stored in a WorkerOptions record, which comes in "local", "node-subprocess", * and "xsnap" flavors. @@ -35,11 +48,43 @@ export {}; * @property { boolean } enableSetup * @property { boolean } enablePipelining * @property { boolean } useTranscript - * @property { number | 'never' } reapInterval + * @property { ReapDirtThreshold } reapDirtThreshold * @property { boolean } critical * @property { MeterID } [meterID] // property must be present, but can be undefined * @property { WorkerOptions } workerOptions * @property { boolean } enableDisavow + * + * @typedef ChangeVatOptions + * @property { number } [reapInterval] + */ + +/** + * Reap/BringOutYourDead/BOYD Scheduling + * + * We trigger a BringOutYourDead delivery (which "reaps" all dead + * objects from the vat) after a certain threshold of "dirt" has + * accumulated. This type is used to define the thresholds for three + * counters: 'deliveries', 'gcKrefs', and 'computrons'. If a property + * is a number, we trigger BOYD when the counter for that property + * exceeds the threshold value. If a property is the string 'never', + * we do not use that counter to trigger BOYD. + * + * @typedef { object } ReapDirtThreshold + * @property { number | 'never' } deliveries + * @property { number | 'never' } gcKrefs + * @property { number | 'never' } computrons + */ + +/** + * Each counter in Dirt matches a threshold in + * ReapDirtThreshold. Missing values are treated as zero, so vats + * start with {} and accumulate dirt as deliveries are made, until a + * BOYD clears them. + * + * @typedef { object } Dirt + * @property { number } [deliveries] + * @property { number } [gcKrefs] + * @property { number } [computrons] */ /** @@ -86,7 +131,7 @@ export {}; * @typedef { { type: 'upgrade-vat', vatID: VatID, upgradeID: string, * bundleID: BundleID, vatParameters: SwingSetCapData, * upgradeMessage: string } } RunQueueEventUpgradeVat - * @typedef { { type: 'changeVatOptions', vatID: VatID, options: Record } } RunQueueEventChangeVatOptions + * @typedef { { type: 'changeVatOptions', vatID: VatID, options: ChangeVatOptions } } RunQueueEventChangeVatOptions * @typedef { { type: 'startVat', vatID: VatID, vatParameters: SwingSetCapData } } RunQueueEventStartVat * @typedef { { type: 'dropExports', vatID: VatID, krefs: string[] } } RunQueueEventDropExports * @typedef { { type: 'retireExports', vatID: VatID, krefs: string[] } } RunQueueEventRetireExports diff --git a/packages/SwingSet/test/change-parameters/test-change-parameters.js b/packages/SwingSet/test/change-parameters/test-change-parameters.js index 290cba11ca13..3851d44b9a8e 100644 --- a/packages/SwingSet/test/change-parameters/test-change-parameters.js +++ b/packages/SwingSet/test/change-parameters/test-change-parameters.js @@ -29,14 +29,22 @@ async function testChangeParameters(t) { t.teardown(c.shutdown); c.pinVatRoot('bootstrap'); await c.run(); - t.is(kvStore.get('kernel.defaultReapInterval'), '1'); + t.deepEqual(JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')), { + deliveries: 1, + gcKrefs: 20, + computrons: 'never', + }); c.changeKernelOptions({ snapshotInterval: 1000, defaultReapInterval: 10, }); - t.is(kvStore.get('kernel.defaultReapInterval'), '10'); + t.deepEqual(JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')), { + deliveries: 10, + gcKrefs: 20, + computrons: 'never', + }); t.throws(() => c.changeKernelOptions({ defaultReapInterval: 'banana' }), { - message: 'invalid defaultReapInterval value', + message: 'defaultReapInterval = banana', }); t.throws(() => c.changeKernelOptions({ snapshotInterval: 'elephant' }), { message: 'invalid heap snapshotInterval value', @@ -44,6 +52,14 @@ async function testChangeParameters(t) { t.throws(() => c.changeKernelOptions({ baz: 'howdy' }), { message: 'unknown option "baz"', }); + c.changeKernelOptions({ + defaultReapGCKrefs: 77, + }); + t.deepEqual(JSON.parse(kvStore.get('kernel.defaultReapDirtThreshold')), { + deliveries: 10, + gcKrefs: 77, + computrons: 'never', + }); async function run(method, args = []) { assert(Array.isArray(args)); @@ -57,7 +73,11 @@ async function testChangeParameters(t) { // setup target vat const [prepStatus] = await run('prepare', []); t.is(prepStatus, 'fulfilled'); - t.is(kvStore.get('v6.reapInterval'), '10'); + t.deepEqual(JSON.parse(kvStore.get('v6.options')).reapDirtThreshold, { + deliveries: 10, + gcKrefs: 77, + computrons: 'never', + }); // now fiddle with stuff const [c1Status, c1Result] = await run('change', [{ foo: 47 }]); @@ -71,7 +91,11 @@ async function testChangeParameters(t) { const [c4Status, c4Result] = await run('change', [{ reapInterval: 20 }]); t.is(c4Status, 'fulfilled'); t.is(c4Result, 'ok'); - t.is(kvStore.get('v6.reapInterval'), '20'); + t.deepEqual(JSON.parse(kvStore.get('v6.options')).reapDirtThreshold, { + deliveries: 20, + gcKrefs: 77, + computrons: 'never', + }); } test('change vat options', async t => { diff --git a/packages/SwingSet/test/snapshots/test-state.js.md b/packages/SwingSet/test/snapshots/test-state.js.md index a3be9f040d62..48daa769554d 100644 --- a/packages/SwingSet/test/snapshots/test-state.js.md +++ b/packages/SwingSet/test/snapshots/test-state.js.md @@ -8,8 +8,8 @@ Generated by [AVA](https://avajs.dev). > initial state - 'a5d302e6743578ccda03ea386abd49de0a3bf4d7dedda2f69585c663806c30bc' + '09c3651da4f2f1fc6cd4e61170aaab3954ee1ace51e6028c69361f73b0ac7272' > expected activityhash - 'f5f1f643f6242a73c79b0437dbab222d34642ea5d047f15aaf5551d5903711d3' + '00a0e15b521f7c32726b0b2d4da425eba66fab98678975ea5b03cd07bfe3109a' diff --git a/packages/SwingSet/test/snapshots/test-state.js.snap b/packages/SwingSet/test/snapshots/test-state.js.snap index 632a77941e6c1d9757416e97e33d0d9d0ca7357c..223f3081f3568f64aae27207bcb07f96082f9d16 100644 GIT binary patch literal 278 zcmV+x0qOohRzVJg9%1vkj$q)75*V9DAA6b+45|+tS3b#4IA4yA$Xrbar9w~!3i8G%Si>wVzr9+ cCB4bzEZk(<$gRAW_HjGm2Sn8@2_OLg022vb=y(L@=6+UD>~ z3(<}@JRge)00000000ARkUdTVF%U*KLWr6hxT8Yydi=B7aRO@E&z`YjWwRUM0fiE7 zLQaYzDkRY-uUHRz^u2jBZEf1Hn66MUSlNKNY1`J``ifKE zZ`!{u%uu{M-wb!dmEG-dzk5PpvQA?16w*`*!;*leh|GRwv7lT%Cl+S78s{*D5+hAE z7P7g>f-KANPs0DE8tPc1ry8AiNL13Ct*68p**VALqj!S9)~7lKpoS1e4zr~+j!eI% aH#wiAsIWoN_TKd4I^hSg1)BXJ0RR94fPa+$ diff --git a/packages/SwingSet/test/test-clist.js b/packages/SwingSet/test/test-clist.js index 10237c763229..8c35bd9b60cd 100644 --- a/packages/SwingSet/test/test-clist.js +++ b/packages/SwingSet/test/test-clist.js @@ -14,7 +14,7 @@ test(`clist reachability`, async t => { kk.createStartingKernelState({ defaultManagerType: 'local' }); const vatID = kk.allocateUnusedVatID(); const source = { bundleID: 'foo' }; - const options = { workerOptions: 'foo', reapInterval: 1 }; + const options = { workerOptions: 'foo', reapDirtThreshold: {} }; kk.createVatState(vatID, source, options); const vk = kk.provideVatKeeper(vatID); @@ -102,7 +102,7 @@ test('getImporters', async t => { kk.createStartingKernelState({ defaultManagerType: 'local' }); const vatID1 = kk.allocateUnusedVatID(); const source = { bundleID: 'foo' }; - const options = { workerOptions: 'foo', reapInterval: 1 }; + const options = { workerOptions: 'foo', reapDirtThreshold: {} }; kk.createVatState(vatID1, source, options); kk.addDynamicVatID(vatID1); const vk1 = kk.provideVatKeeper(vatID1); diff --git a/packages/SwingSet/test/test-controller.js b/packages/SwingSet/test/test-controller.js index e731182d6fcd..113932168737 100644 --- a/packages/SwingSet/test/test-controller.js +++ b/packages/SwingSet/test/test-controller.js @@ -11,6 +11,7 @@ import { initializeSwingset, makeSwingsetController, } from '../src/index.js'; +import makeKernelKeeper from '../src/kernel/state/kernelKeeper.js'; import { checkKT } from './util.js'; const emptyVP = kser({}); @@ -487,3 +488,16 @@ test.serial('bootstrap export', async t => { removeTriple(kt, vattp0, vatTPVatID, 'o+0'); checkKT(t, c, kt); }); + +test('comms vat does not BOYD', async t => { + const config = {}; + const kernelStorage = initSwingStore().kernelStorage; + const controller = await buildVatController(config, [], { kernelStorage }); + t.teardown(controller.shutdown); + const k = makeKernelKeeper(kernelStorage, null); + const commsVatID = k.getVatIDForName('comms'); + t.deepEqual( + JSON.parse(k.kvStore.get(`${commsVatID}.options`)).reapDirtThreshold, + { deliveries: 'never', gcKrefs: 'never', computrons: 'never' }, + ); +}); diff --git a/packages/SwingSet/test/test-kernel.js b/packages/SwingSet/test/test-kernel.js index 81e734fd66e4..f4c9b90a15c5 100644 --- a/packages/SwingSet/test/test-kernel.js +++ b/packages/SwingSet/test/test-kernel.js @@ -3,7 +3,7 @@ import { test } from '../tools/prepare-test-env-ava.js'; import { Fail } from '@agoric/assert'; -import { kser, kslot } from '@agoric/kmarshal'; +import { kser, kunser, kslot } from '@agoric/kmarshal'; import buildKernel from '../src/kernel/index.js'; import { initializeKernel } from '../src/controller/initializeKernel.js'; import { makeVatSlot } from '../src/lib/parseVatSlots.js'; @@ -1635,3 +1635,123 @@ test('reap interval 17', async t => { test('reap interval never', async t => { await reapTest(t, 'never'); }); + +// Set up two vats, one to export vrefs, the other to import/drop +// them. The first will get a reapDirtThreshold.gcKrefs, and will log +// when the kernel sends it BOYD. + +async function reapGCKrefsTest(t, freq) { + const kernel = await makeKernel(); + await kernel.start(); + // note: worker=local, otherwise snapshotInitial/Interval would interfere + + let boyds = 0; + let rxGCkrefs = 0; + let lastExported = 2; + + // vat-under-test, export vrefs on request, watch for BOYDs + function setup1(syscall) { + function dispatch(vatDeliverObject) { + if (vatDeliverObject[0] === 'startVat') { + return; // skip startVat + } + if (vatDeliverObject[0] === 'message') { + // export vrefs, one per message + const target = vatDeliverObject[2].methargs.slots[0]; + const vref = `o+${lastExported}`; + lastExported += 1; + syscall.send(target, kser(['hold', [kslot(vref)]])); + return; + } + if (vatDeliverObject[0] === 'bringOutYourDead') { + boyds += 1; + } + if (vatDeliverObject[0] === 'dropExports') { + rxGCkrefs += vatDeliverObject[1].length; + } + if (vatDeliverObject[0] === 'retireExports') { + rxGCkrefs += vatDeliverObject[1].length; + } + if (vatDeliverObject[0] === 'retireImports') { + rxGCkrefs += vatDeliverObject[1].length; + } + } + return dispatch; + } + const vat1 = await kernel.createTestVat( + 'vat1', + setup1, + {}, + { reapInterval: 'never', reapGCKrefs: freq }, + ); + const v1root = kernel.getRootObject(vat1); + kernel.pinObject(v1root); + + // helper vat, imports vrefs, drops on request + function setup2(syscall) { + const hold = []; + function dispatch(vatDeliverObject) { + if (vatDeliverObject[0] === 'startVat') { + return; // skip startVat + } + if (vatDeliverObject[0] === 'message') { + const [meth, args] = kunser(vatDeliverObject[2].methargs); + if (meth === 'hold') { + for (const vref of vatDeliverObject[2].methargs.slots) { + hold.push(vref); + } + } else { + const [count] = args; + syscall.dropImports(hold.slice(0, count)); + syscall.retireImports(hold.slice(0, count)); + hold.splice(0, count); + } + } + } + return dispatch; + } + const vat2 = await kernel.createTestVat('vat2', setup2, {}); + const v2root = kernel.getRootObject(vat2); + kernel.pinObject(v2root); + + await kernel.run(); + t.is(boyds, 0); + + async function addExport() { + kernel.queueToKref(v1root, `pleaseExport`, [kslot(v2root)], 'none'); + await kernel.run(); + } + + async function doDrop(count) { + kernel.queueToKref(v2root, `drop`, [count], 'none'); + await kernel.run(); + } + + await addExport(); + await addExport(); + t.is(boyds, 0); + // c-list should currently have two krefs exported by the vat + + // now we drop one for every new one we add, and every 'interval'/2 + // we should see a BOYD + + let krefs = 0; + for (let i = 0; i < 10; i += 1) { + await addExport(); + await doDrop(1); + krefs += 2; + t.is(rxGCkrefs, krefs); + if (krefs < freq) { + t.is(boyds, 0); + } else { + t.is(boyds, 1); + boyds = 0; + krefs = 0; + rxGCkrefs = 0; + } + } +} + +test('reap gc-krefs 10', async t => { + await reapGCKrefsTest(t, 10); +}); diff --git a/packages/SwingSet/test/test-state.js b/packages/SwingSet/test/test-state.js index c4f5ba78a731..98f0488a86de 100644 --- a/packages/SwingSet/test/test-state.js +++ b/packages/SwingSet/test/test-state.js @@ -197,7 +197,10 @@ test('kernel state', async t => { ['kd.nextID', '30'], ['kp.nextID', '40'], ['kernel.defaultManagerType', 'local'], - ['kernel.defaultReapInterval', '1'], + [ + 'kernel.defaultReapDirtThreshold', + JSON.stringify({ deliveries: 1, gcKrefs: 20, computrons: 'never' }), + ], ['kernel.snapshotInitial', '3'], ['kernel.snapshotInterval', '200'], ['meter.nextID', '1'], @@ -233,7 +236,10 @@ test('kernelKeeper vat names', async t => { ['vat.name.vatname5', 'v1'], ['vat.name.Frank', 'v2'], ['kernel.defaultManagerType', 'local'], - ['kernel.defaultReapInterval', '1'], + [ + 'kernel.defaultReapDirtThreshold', + JSON.stringify({ deliveries: 1, gcKrefs: 20, computrons: 'never' }), + ], ['kernel.snapshotInitial', '3'], ['kernel.snapshotInterval', '200'], ['meter.nextID', '1'], @@ -283,7 +289,10 @@ test('kernelKeeper device names', async t => { ['device.name.devicename5', 'd7'], ['device.name.Frank', 'd8'], ['kernel.defaultManagerType', 'local'], - ['kernel.defaultReapInterval', '1'], + [ + 'kernel.defaultReapDirtThreshold', + JSON.stringify({ deliveries: 1, gcKrefs: 20, computrons: 'never' }), + ], ['kernel.snapshotInitial', '3'], ['kernel.snapshotInterval', '200'], ['meter.nextID', '1'], @@ -465,7 +474,10 @@ test('kernelKeeper promises', async t => { [`${ko}.owner`, 'v1'], [`${ko}.refCount`, '1,1'], ['kernel.defaultManagerType', 'local'], - ['kernel.defaultReapInterval', '1'], + [ + 'kernel.defaultReapDirtThreshold', + JSON.stringify({ deliveries: 1, gcKrefs: 20, computrons: 'never' }), + ], ['kernel.snapshotInitial', '3'], ['kernel.snapshotInterval', '200'], ['meter.nextID', '1'], @@ -513,7 +525,7 @@ test('vatKeeper', async t => { k.createStartingKernelState({ defaultManagerType: 'local' }); const v1 = k.allocateVatIDForNameIfNeeded('name1'); const source = { bundleID: 'foo' }; - const options = { workerOptions: 'foo', reapInterval: 1 }; + const options = { workerOptions: 'foo', reapDirtThreshold: {} }; k.createVatState(v1, source, options); const vk = k.provideVatKeeper(v1); @@ -555,7 +567,7 @@ test('vatKeeper.getOptions', async t => { 'b1-00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'; const source = { bundleID }; const workerOptions = { type: 'local' }; - const options = { workerOptions, name: 'fred', reapInterval: 1 }; + const options = { workerOptions, name: 'fred', reapDirtThreshold: {} }; k.createVatState(v1, source, options); const vk = k.provideVatKeeper(v1); @@ -925,3 +937,158 @@ test('stats - can load and save existing stats', t => { t.deepEqual(JSON.parse(getSerializedStats().consensusStats), consensusStats); t.deepEqual(JSON.parse(getSerializedStats().localStats), localStats); }); + +test('vatKeeper dirt counters', async t => { + const store = buildKeeperStorageInMemory(); + const k = makeKernelKeeper(store, null); + k.createStartingKernelState({ + defaultManagerType: 'local', + }); + k.saveStats(); + + // the defaults are designed for testing + t.deepEqual(JSON.parse(k.kvStore.get(`kernel.defaultReapDirtThreshold`)), { + deliveries: 1, + gcKrefs: 20, + computrons: 'never', + }); + + const reapDirtThreshold = { deliveries: 10, gcKrefs: 20, computrons: 100 }; + const never = { deliveries: 'never', gcKrefs: 'never', computrons: 'never' }; + + // a new DB will have empty dirt entries for each vat created + const source = { bundleID: 'foo' }; + const v1 = k.allocateVatIDForNameIfNeeded('name1'); + k.createVatState(v1, source, { workerOptions: 'foo', reapDirtThreshold }); + const vk1 = k.provideVatKeeper(v1); + + const v2 = k.allocateVatIDForNameIfNeeded('name2'); + k.createVatState(v2, source, { workerOptions: 'foo', reapDirtThreshold }); + const vk2 = k.provideVatKeeper(v2); + + const v3 = k.allocateVatIDForNameIfNeeded('name3'); + k.createVatState(v3, source, { + workerOptions: 'foo', + reapDirtThreshold: never, + }); + const vk3 = k.provideVatKeeper(v3); + + // the nominal "all clean" entry is { deliveries: 0, gcKrefs: 0, + // computrons: 0 }, but we only store the non-zero keys, so it's + // really {} + t.deepEqual(vk1.getReapDirt(), {}); + t.deepEqual(vk2.getReapDirt(), {}); + + // our write-through cache should store the initial value in the DB + t.true(store.kvStore.has(`${v1}.reapDirt`)); + t.deepEqual(JSON.parse(store.kvStore.get(`${v1}.reapDirt`)), {}); + + // changing one entry doesn't change any others + vk1.addDirt({ deliveries: 1, gcKrefs: 0, computrons: 12 }); + t.deepEqual(vk1.getReapDirt(), { deliveries: 1, gcKrefs: 0, computrons: 12 }); + t.deepEqual(vk2.getReapDirt(), {}); + t.not(vk1.getReapDirt(), vk2.getReapDirt()); + // and writes through the cache + t.deepEqual(JSON.parse(store.kvStore.get(`${v1}.reapDirt`)), { + deliveries: 1, + gcKrefs: 0, + computrons: 12, + }); + + // clearing the dirt will zero out the entries + vk1.clearReapDirt(); + t.deepEqual(vk1.getReapDirt(), {}); + t.deepEqual(JSON.parse(store.kvStore.get(`${v1}.reapDirt`)), {}); + + // nothing has reached the threshold yet + t.is(k.nextReapAction(), undefined); + + vk1.addDirt({ deliveries: 4 }); + t.deepEqual(vk1.getReapDirt(), { deliveries: 4 }); + t.is(k.nextReapAction(), undefined); + vk1.addDirt({ deliveries: 5 }); + t.deepEqual(vk1.getReapDirt(), { deliveries: 9 }); + t.is(k.nextReapAction(), undefined); + vk1.addDirt({ deliveries: 6 }); + t.deepEqual(vk1.getReapDirt(), { deliveries: 15 }); + t.deepEqual(k.nextReapAction(), { type: 'bringOutYourDead', vatID: v1 }); + t.is(k.nextReapAction(), undefined); + + // dirt is ignored when the threshold is 'never' + vk3.addDirt({ deliveries: 4 }); + t.deepEqual(vk3.getReapDirt(), {}); +}); + +test('dirt upgrade', async t => { + const store = buildKeeperStorageInMemory(); + const k = makeKernelKeeper(store, null); + k.createStartingKernelState({ + defaultManagerType: 'local', + }); + k.saveStats(); + const v1 = k.allocateVatIDForNameIfNeeded('name1'); + const source = { bundleID: 'foo' }; + // actual vats get options.reapDirtThreshold ; we install + // options.reapInterval to simulate the old version + const options = { workerOptions: 'foo', reapInterval: 100 }; + k.createVatState(v1, source, options); + const v2 = k.allocateVatIDForNameIfNeeded('comms'); + const options2 = { ...options, reapInterval: 'never' }; + k.createVatState(v2, source, options2); + + // Test that upgrade from an older version of the DB will populate + // the right keys. We simulate the old version by modifying a + // serialized copy. The old version (on mainnet) had things like: + // * kernel.defaultReapInterval: 1000 + // * v1.options: { ... reapInterval: 1000 } + // * v1.reapCountdown: 123 + // * v1.reapInterval: 1000 + // * v3.options: { ... reapInterval: 'never' } + // * v3.reapCountdown: 'never' + // * v3.reapInterval: 'never' + + k.kvStore.delete(`kernel.defaultReapDirtThreshold`); + k.kvStore.set(`kernel.defaultReapInterval`, '300'); + + k.kvStore.delete(`${v1}.reapDirt`); + k.kvStore.delete(`${v1}.reapDirtThreshold`); + k.kvStore.set(`${v1}.reapInterval`, '100'); + k.kvStore.set(`${v1}.reapCountdown`, '70'); + + k.kvStore.delete(`${v2}.reapDirt`); + k.kvStore.delete(`${v2}.reapDirtThreshold`); + k.kvStore.set(`${v2}.reapInterval`, 'never'); + k.kvStore.set(`${v2}.reapCountdown`, 'never'); + + // upgrade is supposed to be the first thing a new kernel does + const k2 = duplicateKeeper(store.serialize); + k2.maybeUpgradeKernelState(); // also upgrades vats + + t.true(k2.kvStore.has(`kernel.defaultReapDirtThreshold`)); + // threshold.deliveries is converted from defaultReapInterval + t.deepEqual(JSON.parse(k2.kvStore.get(`kernel.defaultReapDirtThreshold`)), { + deliveries: 300, + gcKrefs: 20, + computrons: 'never', + }); + + t.true(k2.kvStore.has(`${v1}.reapDirt`)); + // reapDirt.deliveries computed from old reapInterval-reapCountdown + t.deepEqual(JSON.parse(k2.kvStore.get(`${v1}.reapDirt`)), { deliveries: 30 }); + // reapDirtThreshold.deliveries computed from old reapInterval + t.deepEqual(JSON.parse(k2.kvStore.get(`${v1}.options`)).reapDirtThreshold, { + deliveries: 100, + gcKrefs: 20, + computrons: 'never', + }); + const vk1New = k2.provideVatKeeper(v1); + t.deepEqual(vk1New.getReapDirt(), { deliveries: 30 }); + + t.true(k2.kvStore.has(`${v2}.reapDirt`)); + t.deepEqual(JSON.parse(k2.kvStore.get(`${v2}.reapDirt`)), {}); + t.deepEqual(JSON.parse(k2.kvStore.get(`${v2}.options`)).reapDirtThreshold, { + deliveries: 'never', + gcKrefs: 'never', // upgrade must not start BOYDing comms vat + computrons: 'never', + }); +});