From 5296f2cdfb8353a0f90573753ea46959e3499d6b Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Fri, 13 Dec 2024 07:16:37 -0500 Subject: [PATCH 01/18] chore(swing-store): Fix documentation typos --- packages/swing-store/src/assertComplete.js | 2 +- packages/swing-store/src/kvStore.js | 2 +- packages/swing-store/src/snapStore.js | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/swing-store/src/assertComplete.js b/packages/swing-store/src/assertComplete.js index 73a8d0f7292..145a17efc09 100644 --- a/packages/swing-store/src/assertComplete.js +++ b/packages/swing-store/src/assertComplete.js @@ -1,6 +1,6 @@ /** * @param {import('./internal.js').SwingStoreInternal} internal - * @param {Omit} checkMode + * @param {Exclude} checkMode * @returns {void} */ export function assertComplete(internal, checkMode) { diff --git a/packages/swing-store/src/kvStore.js b/packages/swing-store/src/kvStore.js index 4f8b58b5825..20fcda03f09 100644 --- a/packages/swing-store/src/kvStore.js +++ b/packages/swing-store/src/kvStore.js @@ -27,7 +27,7 @@ export function getKeyType(key) { /** * @param {object} db The SQLite database connection. * @param {() => void} ensureTxn Called before mutating methods to establish a DB transaction - * @param {(...args: string[]) => void} trace Called after sets/gets to record a debug log + * @param {(...args: string[]) => void} trace Called after set/delete to record a debug log * @returns { KVStore } */ diff --git a/packages/swing-store/src/snapStore.js b/packages/swing-store/src/snapStore.js index e8548c77fef..57545c2a4af 100644 --- a/packages/swing-store/src/snapStore.js +++ b/packages/swing-store/src/snapStore.js @@ -54,7 +54,8 @@ import { buffer } from './util.js'; * * @typedef {{ * hasHash: (vatID: string, hash: string) => boolean, - * dumpSnapshots: (includeHistorical?: boolean) => {}, + * listAllSnapshots: () => Iterable<{}>, + * dumpSnapshots: (includeHistorical?: boolean) => Record>, * deleteSnapshotByHash: (vatID: string, hash: string) => void, * }} SnapStoreDebug * @@ -677,7 +678,6 @@ export function makeSnapStore( /** * debug function to list all snapshots - * */ function* listAllSnapshots() { yield* sqlListAllSnapshots.iterate(); @@ -706,6 +706,7 @@ export function makeSnapStore( const sql = includeHistorical ? sqlDumpAllSnapshots : sqlDumpCurrentSnapshots; + /** @type {Record>} */ const dump = {}; for (const row of sql.iterate()) { const { vatID, snapPos, hash, compressedSnapshot, inUse } = row; From 25998fbcac1328f30d5b1eb12821d2edba193345 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Fri, 13 Dec 2024 07:31:54 -0500 Subject: [PATCH 02/18] chore(swing-store): Add type SwingStoreOptions --- packages/swing-store/src/snapStore.js | 19 ++++++++++------- packages/swing-store/src/swingStore.js | 23 ++++++++++++++------- packages/swing-store/src/transcriptStore.js | 13 +++++++----- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/packages/swing-store/src/snapStore.js b/packages/swing-store/src/snapStore.js index 57545c2a4af..db14b2550db 100644 --- a/packages/swing-store/src/snapStore.js +++ b/packages/swing-store/src/snapStore.js @@ -7,6 +7,11 @@ import { Fail, q } from '@endo/errors'; import { withDeferredCleanup } from '@agoric/internal'; import { buffer } from './util.js'; +/** + * @import { AnyIterableIterator, SwingStoreExporter } from './exporter.js'; + * @import { ArtifactMode } from './internal.js'; + */ + /** * @typedef {object} SnapshotResult * @property {string} hash sha256 hash of (uncompressed) snapshot @@ -26,13 +31,6 @@ import { buffer } from './util.js'; */ /** - * @import {AnyIterableIterator} from './exporter.js' - */ - -/** - * @typedef { import('./exporter.js').SwingStoreExporter } SwingStoreExporter - * @typedef { import('./internal.js').ArtifactMode } ArtifactMode - * * @typedef {{ * loadSnapshot: (vatID: string) => AsyncIterableIterator, * saveSnapshot: (vatID: string, snapPos: number, snapshotStream: AsyncIterable) => Promise, @@ -59,6 +57,11 @@ import { buffer } from './util.js'; * deleteSnapshotByHash: (vatID: string, hash: string) => void, * }} SnapStoreDebug * + * @callback SnapshotCallback + * Called with the gzipped contents of a new heap snapshot. + * @param {string} name an export key, e.g. `snapshot.${vatID}.${deliveryCount}` + * @param {Parameters[0]} compressedData + * @returns {Promise} */ const finished = promisify(finishedCallback); @@ -72,7 +75,7 @@ const finished = promisify(finishedCallback); * @param {(key: string, value: string | undefined) => void} noteExport * @param {object} [options] * @param {boolean | undefined} [options.keepSnapshots] - * @param {(name: string, compressedData: Parameters[0]) => Promise} [options.archiveSnapshot] + * @param {SnapshotCallback} [options.archiveSnapshot] * @returns {SnapStore & SnapStoreInternal & SnapStoreDebug} */ export function makeSnapStore( diff --git a/packages/swing-store/src/swingStore.js b/packages/swing-store/src/swingStore.js index 314a3eb73ab..8d64b75004c 100644 --- a/packages/swing-store/src/swingStore.js +++ b/packages/swing-store/src/swingStore.js @@ -119,6 +119,18 @@ import { doRepairMetadata } from './repairMetadata.js'; * relative the relevant transactional unit. */ +/** + * @typedef {object} SwingStoreOptions + * @property {Buffer} [serialized] Binary data to load in memory + * @property {boolean} [unsafeFastMode] Disable SQLite safeties for e.g. fast import + * @property {string} [traceFile] Path at which to record KVStore set/delete activity + * @property {boolean} [keepSnapshots] Retain old heap snapshots + * @property {boolean} [keepTranscripts] Retain old transcript span items + * @property {import('./snapStore.js').SnapshotCallback} [archiveSnapshot] Called after creation of a new heap snapshot + * @property {import('./transcriptStore.js').TranscriptCallback} [archiveTranscript] Called after a formerly-current transcript span is finalized + * @property {(pendingExports: Iterable<[key: string, value: string | null]>) => void} [exportCallback] + */ + /** * Do the work of `initSwingStore` and `openSwingStore`. * @@ -127,8 +139,7 @@ import { doRepairMetadata } from './repairMetadata.js'; * database that evaporates when the process exits, which is useful for testing. * @param {boolean} forceReset If true, initialize the database to an empty * state if it already exists - * @param {object} options Configuration options - * + * @param {SwingStoreOptions} [options] * @returns {SwingStore} */ export function makeSwingStore(dirPath, forceReset, options = {}) { @@ -616,13 +627,12 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { * undefined, a memory-only ephemeral store will be created that will evaporate * on program exit. * - * @param {string|null} dirPath Path to a directory in which database files may + * @param {string|null} [dirPath] Path to a directory in which database files may * be kept. This directory need not actually exist yet (if it doesn't it will * be created) but it is reserved (by the caller) for the exclusive use of * this swing store instance. If null, an ephemeral (memory only) store will * be created. - * @param {object?} options Optional configuration options - * + * @param {SwingStoreOptions} [options] * @returns {SwingStore} */ export function initSwingStore(dirPath = null, options = {}) { @@ -640,8 +650,7 @@ export function initSwingStore(dirPath = null, options = {}) { * This directory need not actually exist yet (if it doesn't it will be * created) but it is reserved (by the caller) for the exclusive use of this * swing store instance. - * @param {object?} options Optional configuration options - * + * @param {SwingStoreOptions} [options] * @returns {SwingStore} */ export function openSwingStore(dirPath, options = {}) { diff --git a/packages/swing-store/src/transcriptStore.js b/packages/swing-store/src/transcriptStore.js index 06be31aa9b0..57ee1bfed67 100644 --- a/packages/swing-store/src/transcriptStore.js +++ b/packages/swing-store/src/transcriptStore.js @@ -6,13 +6,11 @@ import BufferLineTransform from '@agoric/internal/src/node/buffer-line-transform import { createSHA256 } from './hasher.js'; /** - * @template T - * @typedef { IterableIterator | AsyncIterableIterator } AnyIterableIterator + * @import { AnyIterable, AnyIterableIterator } from './exporter.js'; + * @import { ArtifactMode } from './internal.js'; */ /** - * @typedef { import('./internal.js').ArtifactMode } ArtifactMode - * * @typedef {{ * initTranscript: (vatID: string) => void, * rolloverSpan: (vatID: string) => Promise, @@ -39,6 +37,11 @@ import { createSHA256 } from './hasher.js'; * dumpTranscripts: (includeHistorical?: boolean) => {[vatID: string]: {[position: number]: string}} * }} TranscriptStoreDebug * + * @callback TranscriptCallback + * Called with the entries of a newly-finalized transcript span. + * @param {string} spanName e.g., `transcript.${vatID}.${startPos}.${endPos}` + * @param {AnyIterable} entries as from `exportSpan` + * @returns {Promise} */ function* empty() { @@ -62,7 +65,7 @@ function insistTranscriptPosition(position) { * @param {(key: string, value: string | undefined ) => void} noteExport * @param {object} [options] * @param {boolean} [options.keepTranscripts] - * @param {(spanName: string, entries: ReturnType) => Promise} [options.archiveTranscript] + * @param {TranscriptCallback} [options.archiveTranscript] * @returns { TranscriptStore & TranscriptStoreInternal & TranscriptStoreDebug } */ export function makeTranscriptStore( From 06c6fc5a072498db496299df902bb4403958051c Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Fri, 13 Dec 2024 07:38:22 -0500 Subject: [PATCH 03/18] chore(swing-store): Narrow AnyIterableIterator to AnyIterable We never consume any affected value as an already-realized iterator --- packages/swing-store/src/exporter.js | 10 +++------- packages/swing-store/src/snapStore.js | 6 +++--- packages/swing-store/src/transcriptStore.js | 6 +++--- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/packages/swing-store/src/exporter.js b/packages/swing-store/src/exporter.js index e8a6cc9fb62..94debee3743 100644 --- a/packages/swing-store/src/exporter.js +++ b/packages/swing-store/src/exporter.js @@ -15,10 +15,6 @@ import { validateArtifactMode } from './internal.js'; * @template T * @typedef { Iterable | AsyncIterable } AnyIterable */ -/** - * @template T - * @typedef { IterableIterator | AsyncIterableIterator } AnyIterableIterator - */ /** * @@ -41,7 +37,7 @@ import { validateArtifactMode } from './internal.js'; * Retrieve a value from the "host" portion of the kvStore, just like * hostStorage.hostKVStore.get() would do. * - * @property {() => AnyIterableIterator} getExportData + * @property {() => AnyIterable} getExportData * * Get a full copy of the first-stage export data (key-value pairs) from the * swingStore. This represents both the contents of the KVStore (excluding host @@ -56,7 +52,7 @@ import { validateArtifactMode } from './internal.js'; * - transcript.${vatID}.${startPos} = ${{ vatID, startPos, endPos, hash }} * - transcript.${vatID}.current = ${{ vatID, startPos, endPos, hash }} * - * @property {() => AnyIterableIterator} getArtifactNames + * @property {() => AnyIterable} getArtifactNames * * Get a list of name of artifacts available from the swingStore. A name * returned by this method guarantees that a call to `getArtifact` on the same @@ -69,7 +65,7 @@ import { validateArtifactMode } from './internal.js'; * - snapshot.${vatID}.${snapPos} * - bundle.${bundleID} * - * @property {(name: string) => AnyIterableIterator} getArtifact + * @property {(name: string) => AnyIterable} getArtifact * * Retrieve an artifact by name as a sequence of binary chunks. May throw if * the artifact is not available, which can occur if the artifact is historical diff --git a/packages/swing-store/src/snapStore.js b/packages/swing-store/src/snapStore.js index db14b2550db..2d1fc975846 100644 --- a/packages/swing-store/src/snapStore.js +++ b/packages/swing-store/src/snapStore.js @@ -8,7 +8,7 @@ import { withDeferredCleanup } from '@agoric/internal'; import { buffer } from './util.js'; /** - * @import { AnyIterableIterator, SwingStoreExporter } from './exporter.js'; + * @import { AnyIterable, SwingStoreExporter } from './exporter.js'; * @import { ArtifactMode } from './internal.js'; */ @@ -45,7 +45,7 @@ import { buffer } from './util.js'; * getExportRecords: (includeHistorical: boolean) => IterableIterator, * getArtifactNames: (artifactMode: ArtifactMode) => AsyncIterableIterator, * importSnapshotRecord: (key: string, value: string) => void, - * populateSnapshot: (name: string, makeChunkIterator: () => AnyIterableIterator, options: { artifactMode: ArtifactMode }) => Promise, + * populateSnapshot: (name: string, makeChunkIterator: () => AnyIterable, options: { artifactMode: ArtifactMode }) => Promise, * assertComplete: (checkMode: Omit) => void, * repairSnapshotRecord: (key: string, value: string) => void, * }} SnapStoreInternal @@ -609,7 +609,7 @@ export function makeSnapStore( /** * @param {string} name Artifact name of the snapshot - * @param {() => AnyIterableIterator} makeChunkIterator get an iterator of snapshot byte chunks + * @param {() => AnyIterable} makeChunkIterator get an iterator of snapshot byte chunks * @param {object} options * @param {ArtifactMode} options.artifactMode * @returns {Promise} diff --git a/packages/swing-store/src/transcriptStore.js b/packages/swing-store/src/transcriptStore.js index 57ee1bfed67..25706f7ba73 100644 --- a/packages/swing-store/src/transcriptStore.js +++ b/packages/swing-store/src/transcriptStore.js @@ -6,7 +6,7 @@ import BufferLineTransform from '@agoric/internal/src/node/buffer-line-transform import { createSHA256 } from './hasher.js'; /** - * @import { AnyIterable, AnyIterableIterator } from './exporter.js'; + * @import { AnyIterable } from './exporter.js'; * @import { ArtifactMode } from './internal.js'; */ @@ -27,7 +27,7 @@ import { createSHA256 } from './hasher.js'; * getExportRecords: (includeHistorical: boolean) => IterableIterator, * getArtifactNames: (artifactMode: ArtifactMode) => AsyncIterableIterator, * importTranscriptSpanRecord: (key: string, value: string) => void, - * populateTranscriptSpan: (name: string, makeChunkIterator: () => AnyIterableIterator, options: { artifactMode: ArtifactMode }) => Promise, + * populateTranscriptSpan: (name: string, makeChunkIterator: () => AnyIterable, options: { artifactMode: ArtifactMode }) => Promise, * assertComplete: (checkMode: Omit) => void, * repairTranscriptSpanRecord: (key: string, value: string) => void, * readFullVatTranscript: (vatID: string) => Iterable<{position: number, item: string}> @@ -778,7 +778,7 @@ export function makeTranscriptStore( * Import a transcript span from another store. * * @param {string} name Artifact Name of the transcript span - * @param {() => AnyIterableIterator} makeChunkIterator get an iterator of transcript byte chunks + * @param {() => AnyIterable} makeChunkIterator get an iterator of transcript byte chunks * @param {object} options * @param {ArtifactMode} options.artifactMode * From 7396efd4ccc075461b7c5a7902fe595dfcb6d2e0 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Mon, 16 Dec 2024 12:30:28 -0500 Subject: [PATCH 04/18] feat(internal): Add `pick` utility --- packages/internal/src/index.js | 4 --- packages/internal/src/ses-utils.js | 29 ++++++++++++++++++ .../test/snapshots/exports.test.js.md | 1 + .../test/snapshots/exports.test.js.snap | Bin 693 -> 687 bytes 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/packages/internal/src/index.js b/packages/internal/src/index.js index 662e273110d..f24e084952b 100644 --- a/packages/internal/src/index.js +++ b/packages/internal/src/index.js @@ -14,7 +14,3 @@ export * from './typeGuards.js'; // eslint-disable-next-line import/export -- just types export * from './types-index.js'; - -export { objectMap } from '@endo/common/object-map.js'; -export { objectMetaMap } from '@endo/common/object-meta-map.js'; -export { fromUniqueEntries } from '@endo/common/from-unique-entries.js'; diff --git a/packages/internal/src/ses-utils.js b/packages/internal/src/ses-utils.js index 11b11699d4b..b6f27e6cdc3 100644 --- a/packages/internal/src/ses-utils.js +++ b/packages/internal/src/ses-utils.js @@ -5,12 +5,19 @@ * either directly or indirectly (e.g. by @endo imports). */ +import { objectMap } from '@endo/common/object-map.js'; +import { objectMetaMap } from '@endo/common/object-meta-map.js'; +import { fromUniqueEntries } from '@endo/common/from-unique-entries.js'; import { q, Fail, makeError, annotateError, X } from '@endo/errors'; import { deeplyFulfilled, isObject } from '@endo/marshal'; import { makePromiseKit } from '@endo/promise-kit'; import { makeQueue } from '@endo/stream'; +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore TS7016 The 'jessie.js' library may need to update its package.json or typings import { asyncGenerate } from 'jessie.js'; +export { objectMap, objectMetaMap, fromUniqueEntries }; + const { fromEntries, keys, values } = Object; /** @import {ERef} from '@endo/far' */ @@ -163,6 +170,28 @@ export const assertAllDefined = obj => { } }; +/** + * @template {Record} T + * @template {Partial<{ [K in keyof T]: true }>} U + * @param {T} target + * @param {U} [permits] + * @returns {keyof U extends keyof T ? Pick : never} + */ +export const pick = ( + target, + permits = /** @type {U} */ (objectMap(target, () => true)), +) => { + const attenuation = objectMap(permits, (permit, key) => { + permit === true || Fail`internal: ${q(key)} permit must be true`; + // eslint-disable-next-line no-restricted-syntax + key in target || Fail`internal: target is missing ${q(key)}`; + // eslint-disable-next-line no-restricted-syntax + return target[key]; + }); + // @ts-expect-error cast + return attenuation; +}; + /** @type {IteratorResult} */ const notDone = harden({ done: false, value: undefined }); diff --git a/packages/internal/test/snapshots/exports.test.js.md b/packages/internal/test/snapshots/exports.test.js.md index 06f69a7e5d7..d36f773467d 100644 --- a/packages/internal/test/snapshots/exports.test.js.md +++ b/packages/internal/test/snapshots/exports.test.js.md @@ -35,6 +35,7 @@ Generated by [AVA](https://avajs.dev). 'mustMatch', 'objectMap', 'objectMetaMap', + 'pick', 'pureDataMarshaller', 'synchronizedTee', 'untilTrue', diff --git a/packages/internal/test/snapshots/exports.test.js.snap b/packages/internal/test/snapshots/exports.test.js.snap index a952ead72687ef678a52dfe153b5c7b7158609cb..e3a78b366a094d39c7f13e0a2ce4c66a94fb2b41 100644 GIT binary patch literal 687 zcmV;g0#N-yRzV*N8&y&h%EB)8XsG+Rur6v zr)ssz_%!498J|I;#FluO@v$1Oty&o;uqu;=_U%+;aAt5Gl*v=25r?D{lKPr)L*p7+ z`WH}i%)eqeh|?{jTe(uwr6H??Y~iIfL+M~esV%!m(O8hWHt%3~VD96Tz6-SrvM)5=AE0KBSY>uq%`T-x5Pwdn#m$B1tSH!b8Uvn^L1$ z1Zg@*Q}}jAs2r&M73wCO1M6{v>(G}P=zL~^G1tQp!@LPTH8HI)8}dpd9Cesg6iwp4$x|B>tgYfnyRn4&Dx)^<8 zT{;>ns&gwiRafXJwMNT5h&w>{cAHA!h)UJ5IYoCU+D)Z`i_u-0(_QIP95JvK;xz?A V=hXUMkji^9`vb`Bjm_Z%001K3M6Un< literal 693 zcmV;m0!sZsRzV$csaU3Uv`{e;53ZB5ZDF)d=vJ*r(-aXzO zGM-sxW(iK04vCV2iXv2qA{`GwN8SSE0jQBlW|APbTIu_A=0BI?L0{@*-yVG-C!8b_ zDy4VADqh%JJ9~7TrrhyYmCrGzF#gyyd;Ar^34osf&HywR@Pq+f29yl=#DMP%IAy?Z z2Gk?qX#{j5Kt#ZY2>2EOrx9=#0dq018Ux)JIEaCdF>n$CKVsll3{)$?Y6aM?07(US zUm3qSS!WYXWB72yf=l1oXh!-t{vjAsp= zWri~DsAoo7LUAcaUYhmN43~0`%aTI+8zCFFmW*pkxj_S=D9oqojrqP%X;aEJxm>5A z!kp)ogxipLKGaQ#igjI#o>{GUUxws0Z_JI2HZOMiuPF%y7Cc@aV>#L?VD>2me#eoOTzix3P}HLy3GrZZinezAaeW)z7s+f zrJ)TzhQ{XO^9c{%^-j%>RFWI5#37|U3LCl?o-dUXvS-SG?gw;`3rSbegK&`t!sQz@ bAY&+9m&8>W@;&rYd?<=Pn-gc8#{>WX+-p8{ From 712f5a59499d516c48529327bb235e0386f4e97c Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Mon, 16 Dec 2024 12:55:13 -0500 Subject: [PATCH 05/18] refactor(swing-store): Use new `pick` utility --- packages/swing-store/src/swingStore.js | 77 +++++++++++++++----------- 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/packages/swing-store/src/swingStore.js b/packages/swing-store/src/swingStore.js index 8d64b75004c..83f816e54ca 100644 --- a/packages/swing-store/src/swingStore.js +++ b/packages/swing-store/src/swingStore.js @@ -7,6 +7,8 @@ import sqlite3 from 'better-sqlite3'; import { Fail, q } from '@endo/errors'; +import { pick } from '@agoric/internal'; + import { dbFileInDirectory } from './util.js'; import { makeKVStore, getKeyType } from './kvStore.js'; import { makeTranscriptStore } from './transcriptStore.js'; @@ -303,7 +305,7 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { const kvStore = makeKVStore(db, ensureTxn, trace); - const { dumpTranscripts, ...transcriptStore } = makeTranscriptStore( + const { dumpTranscripts, ...transcriptStoreInternal } = makeTranscriptStore( db, ensureTxn, noteExport, @@ -312,7 +314,7 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { archiveTranscript, }, ); - const { dumpSnapshots, ...snapStore } = makeSnapStore( + const { dumpSnapshots, ...snapStoreInternal } = makeSnapStore( db, ensureTxn, makeSnapStoreIO(), @@ -322,7 +324,7 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { archiveSnapshot, }, ); - const { dumpBundles, ...bundleStore } = makeBundleStore( + const { dumpBundles, ...bundleStoreInternal } = makeBundleStore( db, ensureTxn, noteExport, @@ -515,9 +517,9 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { /** @type {import('./internal.js').SwingStoreInternal} */ const internal = harden({ dirPath, - snapStore, - transcriptStore, - bundleStore, + snapStore: snapStoreInternal, + transcriptStore: transcriptStoreInternal, + bundleStore: bundleStoreInternal, }); async function repairMetadata(exporter) { @@ -560,38 +562,47 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { return db; } - const transcriptStorePublic = { - initTranscript: transcriptStore.initTranscript, - rolloverSpan: transcriptStore.rolloverSpan, - rolloverIncarnation: transcriptStore.rolloverIncarnation, - getCurrentSpanBounds: transcriptStore.getCurrentSpanBounds, - addItem: transcriptStore.addItem, - readSpan: transcriptStore.readSpan, - stopUsingTranscript: transcriptStore.stopUsingTranscript, - deleteVatTranscripts: transcriptStore.deleteVatTranscripts, - }; + const transcriptStore = pick( + transcriptStoreInternal, + /** @type {const} */ ({ + initTranscript: true, + rolloverSpan: true, + rolloverIncarnation: true, + getCurrentSpanBounds: true, + addItem: true, + readSpan: true, + stopUsingTranscript: true, + deleteVatTranscripts: true, + }), + ); - const snapStorePublic = { - loadSnapshot: snapStore.loadSnapshot, - saveSnapshot: snapStore.saveSnapshot, - deleteAllUnusedSnapshots: snapStore.deleteAllUnusedSnapshots, - deleteVatSnapshots: snapStore.deleteVatSnapshots, - stopUsingLastSnapshot: snapStore.stopUsingLastSnapshot, - getSnapshotInfo: snapStore.getSnapshotInfo, - }; + const snapStore = pick( + snapStoreInternal, + /** @type {const} */ ({ + loadSnapshot: true, + saveSnapshot: true, + deleteAllUnusedSnapshots: true, + deleteVatSnapshots: true, + stopUsingLastSnapshot: true, + getSnapshotInfo: true, + }), + ); - const bundleStorePublic = { - addBundle: bundleStore.addBundle, - hasBundle: bundleStore.hasBundle, - getBundle: bundleStore.getBundle, - deleteBundle: bundleStore.deleteBundle, - }; + const bundleStore = pick( + bundleStoreInternal, + /** @type {const} */ ({ + addBundle: true, + hasBundle: true, + getBundle: true, + deleteBundle: true, + }), + ); const kernelStorage = { kvStore: kernelKVStore, - transcriptStore: transcriptStorePublic, - snapStore: snapStorePublic, - bundleStore: bundleStorePublic, + transcriptStore, + snapStore, + bundleStore, startCrank, establishCrankSavepoint, rollbackCrank, From 70e0945e3fc11b3dc0924c732cef5dba3cb8922d Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Mon, 16 Dec 2024 12:35:32 -0500 Subject: [PATCH 06/18] feat(cosmic-swingset): Expose a controller and associated helpers from test-kit.js --- packages/SwingSet/tools/run-utils.js | 3 +- .../boot/test/upgrading/upgrade-vats.test.ts | 1 + packages/boot/tools/supports.ts | 1 + packages/cosmic-swingset/src/chain-main.js | 8 ++++-- packages/cosmic-swingset/src/launch-chain.js | 25 +++++++++++++++-- packages/cosmic-swingset/tools/test-kit.js | 28 +++++++++++++++++-- 6 files changed, 58 insertions(+), 8 deletions(-) diff --git a/packages/SwingSet/tools/run-utils.js b/packages/SwingSet/tools/run-utils.js index 229b89483c9..408c41228e5 100644 --- a/packages/SwingSet/tools/run-utils.js +++ b/packages/SwingSet/tools/run-utils.js @@ -15,9 +15,10 @@ import { makeQueue } from '@endo/stream'; */ export const makeRunUtils = (controller, harness) => { const mutex = makeQueue(); + mutex.put('dummy result'); // so the first `await mutex.get()` doesn't hang + const logRunFailure = reason => console.log('controller.run() failure', reason); - mutex.put(controller.run().catch(logRunFailure)); /** * Wait for exclusive access to the controller, then before relinquishing that access, diff --git a/packages/boot/test/upgrading/upgrade-vats.test.ts b/packages/boot/test/upgrading/upgrade-vats.test.ts index 9711c7dd121..09755b946a7 100644 --- a/packages/boot/test/upgrading/upgrade-vats.test.ts +++ b/packages/boot/test/upgrading/upgrade-vats.test.ts @@ -49,6 +49,7 @@ const makeScenario = async ( t.teardown(c.shutdown); c.pinVatRoot('bootstrap'); + await c.run(); const runUtils = makeRunUtils(c); return runUtils; }; diff --git a/packages/boot/tools/supports.ts b/packages/boot/tools/supports.ts index 35959613993..7df7baaa72e 100644 --- a/packages/boot/tools/supports.ts +++ b/packages/boot/tools/supports.ts @@ -587,6 +587,7 @@ export const makeSwingsetTestKit = async ( console.timeLog('makeBaseSwingsetTestKit', 'buildSwingset'); + await controller.run(); const runUtils = makeBootstrapRunUtils(controller, harness); const buildProposal = makeProposalExtractor({ diff --git a/packages/cosmic-swingset/src/chain-main.js b/packages/cosmic-swingset/src/chain-main.js index 18a34081e50..44684b56587 100644 --- a/packages/cosmic-swingset/src/chain-main.js +++ b/packages/cosmic-swingset/src/chain-main.js @@ -47,7 +47,7 @@ import { makeReadCachingStorage, } from './helpers/bufferedStorage.js'; import stringify from './helpers/json-stable-stringify.js'; -import { launch } from './launch-chain.js'; +import { launch, launchAndShareInternals } from './launch-chain.js'; import { makeProcessValue } from './helpers/process-value.js'; import { spawnSwingStoreExport, @@ -228,6 +228,7 @@ export const makeQueueStorage = (call, queuePath) => { * slogSender?: ERef>, * swingStore?: import('@agoric/swing-store').SwingStore, * vatconfig?: Parameters[0]['vatconfig'], + * withInternals?: boolean, * }} [options.testingOverrides] */ export const makeLaunchChain = ( @@ -523,7 +524,10 @@ export const makeLaunchChain = ( ? makeArchiveTranscript(vatTranscriptArchiveDir, fsPowers) : undefined; - const s = await launch({ + const launcher = testingOverrides.withInternals + ? launchAndShareInternals + : launch; + const s = await launcher({ actionQueueStorage, highPriorityQueueStorage, swingStore: testingOverrides.swingStore, diff --git a/packages/cosmic-swingset/src/launch-chain.js b/packages/cosmic-swingset/src/launch-chain.js index f8ef579a9da..4d182d29490 100644 --- a/packages/cosmic-swingset/src/launch-chain.js +++ b/packages/cosmic-swingset/src/launch-chain.js @@ -26,7 +26,7 @@ import { } from '@agoric/swingset-vat'; import { waitUntilQuiescent } from '@agoric/internal/src/lib-nodejs/waitUntilQuiescent.js'; import { openSwingStore } from '@agoric/swing-store'; -import { BridgeId as BRIDGE_ID } from '@agoric/internal'; +import { pick, BridgeId as BRIDGE_ID } from '@agoric/internal'; import { makeWithQueue } from '@agoric/internal/src/queue.js'; import * as ActionType from '@agoric/internal/src/action-types.js'; @@ -333,7 +333,7 @@ export async function buildSwingset( /** * @param {LaunchOptions} options */ -export async function launch({ +export async function launchAndShareInternals({ actionQueueStorage, highPriorityQueueStorage, kernelStateDBDir, @@ -1263,5 +1263,26 @@ export async function launch({ writeSlogObject, savedHeight, savedChainSends: JSON.parse(kvStore.get(getHostKey('chainSends')) || '[]'), + // NOTE: to be used only for testing purposes! + internals: { + controller, + bridgeInbound, + timer, + }, }; } + +/** + * @param {LaunchOptions} options + * @returns {Promise>, 'internals'>>} + */ +export async function launch(options) { + const launchResult = await launchAndShareInternals(options); + return pick(launchResult, { + blockingSend: true, + shutdown: true, + writeSlogObject: true, + savedHeight: true, + savedChainSends: true, + }); +} diff --git a/packages/cosmic-swingset/tools/test-kit.js b/packages/cosmic-swingset/tools/test-kit.js index 4905fa455f6..a7c0fc42a06 100644 --- a/packages/cosmic-swingset/tools/test-kit.js +++ b/packages/cosmic-swingset/tools/test-kit.js @@ -13,8 +13,8 @@ import { } from '@agoric/internal/src/action-types.js'; import * as STORAGE_PATH from '@agoric/internal/src/chain-storage-paths.js'; import { deepCopyJsonable } from '@agoric/internal/src/js-utils.js'; +import { makeRunUtils } from '@agoric/swingset-vat/tools/run-utils.js'; import { initSwingStore } from '@agoric/swing-store'; - import { extractPortNums, makeLaunchChain, @@ -23,6 +23,7 @@ import { import { DEFAULT_SIM_SWINGSET_PARAMS } from '../src/sim-params.js'; import { makeQueue } from '../src/helpers/make-queue.js'; +/** @import {EReturn} from '@endo/far'; */ /** @import { BlockInfo, InitMsg } from '@agoric/internal/src/chain-utils.js' */ /** @import { ManagerType, SwingSetConfig } from '@agoric/swingset-vat' */ /** @import { InboundQueue } from '../src/launch-chain.js'; */ @@ -252,13 +253,25 @@ export const makeCosmicSwingsetTestKit = async ( env, fs, path: nativePath, - testingOverrides: { debugName, slogSender, swingStore, vatconfig: config }, + testingOverrides: { + debugName, + slogSender, + swingStore, + vatconfig: config, + withInternals: true, + }, }); const launchResult = await launchChain({ ...initMessage, resolvedConfig: swingsetConfig, }); - const { blockingSend, shutdown: shutdownKernel } = launchResult; + const { + blockingSend, + shutdown: shutdownKernel, + internals, + } = /** @type {EReturn} */ ( + launchResult + ); /** @type {(options?: { kernelOnly?: boolean }) => Promise} */ const shutdown = async ({ kernelOnly = false } = {}) => { await shutdownKernel(); @@ -266,6 +279,8 @@ export const makeCosmicSwingsetTestKit = async ( await hostStorage.close(); await cleanupDB(); }; + const { controller, bridgeInbound, timer } = internals; + const { queueAndRun, EV } = makeRunUtils(controller); // Remember information about the current block, starting with the init // message. @@ -388,6 +403,13 @@ export const makeCosmicSwingsetTestKit = async ( shutdown, swingStore, + // Controller-oriented helpers. + controller, + bridgeInbound, + timer, + queueAndRun, + EV, + // Functions specific to this kit. getLastBlockInfo, pushQueueRecord, From dd54fd1ebd5fd464bdc37696f401e40668d5319e Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Mon, 16 Dec 2024 12:45:31 -0500 Subject: [PATCH 07/18] refactor(swing-store): Extract all makeSwingStore options up front --- packages/swing-store/src/swingStore.js | 32 ++++++++++++++------------ 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/packages/swing-store/src/swingStore.js b/packages/swing-store/src/swingStore.js index 83f816e54ca..1a12bb1fb9c 100644 --- a/packages/swing-store/src/swingStore.js +++ b/packages/swing-store/src/swingStore.js @@ -145,11 +145,26 @@ import { doRepairMetadata } from './repairMetadata.js'; * @returns {SwingStore} */ export function makeSwingStore(dirPath, forceReset, options = {}) { - const { serialized } = options; + const { + serialized, + unsafeFastMode, + + traceFile, + keepSnapshots, + keepTranscripts, + archiveSnapshot, + archiveTranscript, + exportCallback, + } = options; + if (serialized) { Buffer.isBuffer(serialized) || Fail`options.serialized must be Buffer`; dirPath === null || Fail`options.serialized makes :memory: DB`; } + exportCallback === undefined || + typeof exportCallback === 'function' || + Fail`export callback must be a function`; + let crankhasher; function resetCrankhash() { crankhasher = createSHA256(); @@ -181,14 +196,6 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { filePath = ':memory:'; } - const { - traceFile, - keepSnapshots, - keepTranscripts, - archiveSnapshot, - archiveTranscript, - } = options; - let traceOutput = traceFile ? fs.createWriteStream(path.resolve(traceFile), { flags: 'a', @@ -243,7 +250,7 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { } // PRAGMAs have to happen outside a transaction - setUnsafeFastMode(options.unsafeFastMode); + setUnsafeFastMode(unsafeFastMode); // We use IMMEDIATE because the kernel is supposed to be the sole writer of // the DB, and if some other process is holding a write lock, we want to find @@ -286,11 +293,6 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { ) `); - const { exportCallback } = options; - exportCallback === undefined || - typeof exportCallback === 'function' || - Fail`export callback must be a function`; - const sqlAddPendingExport = db.prepare(` INSERT INTO pendingExports (key, value) VALUES (?, ?) From 59f669c045db3e35999451e4f64d324c1e9a903c Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Mon, 16 Dec 2024 13:00:20 -0500 Subject: [PATCH 08/18] feat(swing-store): Add options for opening swing-stores * from file (vs. from swingstore.sqlite in a directory) * read-only (vs. read-write) --- packages/swing-store/src/assertComplete.js | 2 +- packages/swing-store/src/swingStore.js | 112 ++++++++++----------- 2 files changed, 54 insertions(+), 60 deletions(-) diff --git a/packages/swing-store/src/assertComplete.js b/packages/swing-store/src/assertComplete.js index 145a17efc09..d04bf80f0bf 100644 --- a/packages/swing-store/src/assertComplete.js +++ b/packages/swing-store/src/assertComplete.js @@ -1,5 +1,5 @@ /** - * @param {import('./internal.js').SwingStoreInternal} internal + * @param {Pick} internal * @param {Exclude} checkMode * @returns {void} */ diff --git a/packages/swing-store/src/swingStore.js b/packages/swing-store/src/swingStore.js index 1a12bb1fb9c..519462aa33a 100644 --- a/packages/swing-store/src/swingStore.js +++ b/packages/swing-store/src/swingStore.js @@ -1,7 +1,7 @@ // @ts-check /* eslint-env node */ import * as fs from 'fs'; -import * as path from 'path'; +import * as pathlib from 'path'; import sqlite3 from 'better-sqlite3'; @@ -18,6 +18,9 @@ import { createSHA256 } from './hasher.js'; import { makeSnapStoreIO } from './snapStoreIO.js'; import { doRepairMetadata } from './repairMetadata.js'; +// https://github.com/WiseLibs/better-sqlite3/blob/HEAD/docs/api.md#new-databasepath-options +const IN_MEMORY = ':memory:'; + /** * @typedef { import('./kvStore.js').KVStore } KVStore * @@ -123,8 +126,10 @@ import { doRepairMetadata } from './repairMetadata.js'; /** * @typedef {object} SwingStoreOptions + * @property {boolean} [asFile] For testing, interpret path as a file rather than a swingstore.sqlite parent directory * @property {Buffer} [serialized] Binary data to load in memory * @property {boolean} [unsafeFastMode] Disable SQLite safeties for e.g. fast import + * @property {boolean} [readonly] * @property {string} [traceFile] Path at which to record KVStore set/delete activity * @property {boolean} [keepSnapshots] Retain old heap snapshots * @property {boolean} [keepTranscripts] Retain old transcript span items @@ -136,18 +141,21 @@ import { doRepairMetadata } from './repairMetadata.js'; /** * Do the work of `initSwingStore` and `openSwingStore`. * - * @param {string|null} dirPath Path to a directory in which database files may - * be kept. If this is null, the database will be an in-memory ephemeral + * @param {string|null} path Path to a directory in which database files may + * be kept (or when the `asFile` option is true, the path to such a database + * file). If this is null, the database will be an in-memory ephemeral * database that evaporates when the process exits, which is useful for testing. * @param {boolean} forceReset If true, initialize the database to an empty * state if it already exists * @param {SwingStoreOptions} [options] * @returns {SwingStore} */ -export function makeSwingStore(dirPath, forceReset, options = {}) { +export function makeSwingStore(path, forceReset, options = {}) { const { + asFile = false, serialized, unsafeFastMode, + readonly = false, traceFile, keepSnapshots, @@ -159,7 +167,7 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { if (serialized) { Buffer.isBuffer(serialized) || Fail`options.serialized must be Buffer`; - dirPath === null || Fail`options.serialized makes :memory: DB`; + path === null || Fail`options.serialized makes :memory: DB`; } exportCallback === undefined || typeof exportCallback === 'function' || @@ -172,32 +180,22 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { resetCrankhash(); let filePath; - if (dirPath) { + if (path) { if (forceReset) { - try { - // Node.js 16.8.0 warns: - // In future versions of Node.js, fs.rmdir(path, { recursive: true }) will - // be removed. Use fs.rm(path, { recursive: true }) instead - if (fs.rmSync) { - fs.rmSync(dirPath, { recursive: true }); - } else { - fs.rmdirSync(dirPath, { recursive: true }); - } - } catch (e) { - // Attempting to delete a non-existent directory is allowed - if (e.code !== 'ENOENT') { - throw e; - } - } + fs.rmSync(path, { recursive: true, force: true }); + } + if (asFile) { + filePath = path; + } else { + fs.mkdirSync(path, { recursive: true }); + filePath = dbFileInDirectory(path); } - fs.mkdirSync(dirPath, { recursive: true }); - filePath = dbFileInDirectory(dirPath); } else { filePath = ':memory:'; } let traceOutput = traceFile - ? fs.createWriteStream(path.resolve(traceFile), { + ? fs.createWriteStream(pathlib.resolve(traceFile), { flags: 'a', }) : null; @@ -214,10 +212,10 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { } /** @type {*} */ - let db = sqlite3( - serialized || filePath, - // { verbose: console.log }, - ); + let db = sqlite3(/** @type {string} */ (serialized || filePath), { + readonly, + // verbose: console.log, + }); // We use WAL (write-ahead log) mode to allow a background export process to // keep reading from an earlier DB state, while allowing execution to proceed @@ -250,7 +248,7 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { } // PRAGMAs have to happen outside a transaction - setUnsafeFastMode(unsafeFastMode); + if (!readonly) setUnsafeFastMode(unsafeFastMode); // We use IMMEDIATE because the kernel is supposed to be the sole writer of // the DB, and if some other process is holding a write lock, we want to find @@ -340,13 +338,9 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { let inCrank = false; function diskUsage() { - if (dirPath) { - const dataFilePath = dbFileInDirectory(dirPath); - const stat = fs.statSync(dataFilePath); - return stat.size; - } else { - return 0; - } + if (filePath === IN_MEMORY) return 0; + const stat = fs.statSync(filePath); + return stat.size; } const kernelKVStore = { @@ -518,7 +512,8 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { /** @type {import('./internal.js').SwingStoreInternal} */ const internal = harden({ - dirPath, + dirPath: asFile ? null : path, + asFile, snapStore: snapStoreInternal, transcriptStore: transcriptStoreInternal, bundleStore: bundleStoreInternal, @@ -634,41 +629,40 @@ export function makeSwingStore(dirPath, forceReset, options = {}) { } /** - * Create a new swingset store. If given a directory path string, a persistent - * store will be created in that directory; if there is already a store there, - * it will be reinitialized to an empty state. If the path is null or - * undefined, a memory-only ephemeral store will be created that will evaporate - * on program exit. + * Create a new swingset store at the given `path`, overwriting any prior store + * there. * - * @param {string|null} [dirPath] Path to a directory in which database files may - * be kept. This directory need not actually exist yet (if it doesn't it will - * be created) but it is reserved (by the caller) for the exclusive use of - * this swing store instance. If null, an ephemeral (memory only) store will - * be created. + * @param {string|null} [path] Path to a directory in which database files may + * be kept (or when the `asFile` option is true, the path to such a database + * file). This directory or file need not actually exist yet (if it doesn't + * it will be created) but it is reserved (by the caller) for the exclusive + * use of this swing store instance. If null, an ephemeral (memory only) + * store will be created. * @param {SwingStoreOptions} [options] * @returns {SwingStore} */ -export function initSwingStore(dirPath = null, options = {}) { - if (dirPath) { - typeof dirPath === 'string' || Fail`dirPath must be a string`; +export function initSwingStore(path = null, options = {}) { + if (path) { + typeof path === 'string' || Fail`path must be a string`; } - return makeSwingStore(dirPath, true, options); + return makeSwingStore(path, true, options); } /** * Open a persistent swingset store. If there is no existing store at the given - * `dirPath`, a new, empty store will be created. + * `path`, a new, empty store will be created. * - * @param {string} dirPath Path to a directory in which database files may be kept. - * This directory need not actually exist yet (if it doesn't it will be - * created) but it is reserved (by the caller) for the exclusive use of this - * swing store instance. + * @param {string} path Path to a directory in which database files may be kept + * (or when the `asFile` option is true, the path to such a database file). + * This directory or file need not actually exist yet (if it doesn't it will + * be created) but it is reserved (by the caller) for the exclusive use of + * this swing store instance. * @param {SwingStoreOptions} [options] * @returns {SwingStore} */ -export function openSwingStore(dirPath, options = {}) { - typeof dirPath === 'string' || Fail`dirPath must be a string`; - return makeSwingStore(dirPath, false, options); +export function openSwingStore(path, options = {}) { + typeof path === 'string' || Fail`path must be a string`; + return makeSwingStore(path, false, options); } /** From 1048a9273b1bbba319117276bc7c44c6190511d0 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Mon, 6 Jan 2025 21:44:35 -0500 Subject: [PATCH 09/18] feat(cosmic-swingset): Add `getNextKey` support to BufferedStorage --- .../src/helpers/bufferedStorage.js | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/packages/cosmic-swingset/src/helpers/bufferedStorage.js b/packages/cosmic-swingset/src/helpers/bufferedStorage.js index 0233faa492d..a9a32221ad2 100644 --- a/packages/cosmic-swingset/src/helpers/bufferedStorage.js +++ b/packages/cosmic-swingset/src/helpers/bufferedStorage.js @@ -76,10 +76,8 @@ export const makeKVStoreFromMap = map => { let priorKeyIndex; const ensureSorted = () => { - if (!sortedKeys) { - sortedKeys = [...map.keys()]; - sortedKeys.sort(compareByCodePoints); - } + if (sortedKeys) return; + sortedKeys = [...map.keys()].sort(compareByCodePoints); }; const clearGetNextKeyCache = () => { @@ -101,9 +99,16 @@ export const makeKVStoreFromMap = map => { assert.typeof(priorKey, 'string'); ensureSorted(); const start = - compareByCodePoints(priorKeyReturned, priorKey) <= 0 - ? priorKeyIndex + 1 - : 0; + priorKeyReturned === undefined + ? 0 + : // If priorKeyReturned <= priorKey, start just after it. + (compareByCodePoints(priorKeyReturned, priorKey) <= 0 && + priorKeyIndex + 1) || + // Else if priorKeyReturned immediately follows priorKey, start at + // its index (and expect to return it again). + (sortedKeys.at(priorKeyIndex - 1) === priorKey && priorKeyIndex) || + // Otherwise, start at the beginning. + 0; for (let i = start; i < sortedKeys.length; i += 1) { const key = sortedKeys[i]; if (compareByCodePoints(key, priorKey) <= 0) continue; @@ -226,7 +231,8 @@ export function makeBufferedStorage(kvStore, listeners = {}) { // To avoid confusion, additions and deletions are prevented from sharing // the same key at any given time. - const additions = new Map(); + /** @type {Map & KVStore} */ + const additions = provideEnhancedKVStore(makeKVStoreFromMap(new Map())); const deletions = new Set(); /** @type {KVStore} */ @@ -257,13 +263,18 @@ export function makeBufferedStorage(kvStore, listeners = {}) { deletions.add(key); if (onPendingDelete !== undefined) onPendingDelete(key); }, - - /** - * @param {string} previousKey - */ getNextKey(previousKey) { assert.typeof(previousKey, 'string'); - throw Error('not implemented'); + const bufferedNextKey = additions.getNextKey(previousKey); + let nextKey = kvStore.getNextKey(previousKey); + while (nextKey !== undefined) { + if (bufferedNextKey !== undefined) { + if (compareByCodePoints(bufferedNextKey, nextKey) <= 0) break; + } + if (!deletions.has(nextKey)) return nextKey; + nextKey = kvStore.getNextKey(nextKey); + } + return bufferedNextKey; }, }; function commit() { From 250c1dde2252af0cceb97d3219723460a0257844 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Mon, 16 Dec 2024 12:30:28 -0500 Subject: [PATCH 10/18] feat(internal): Add `defineName` utility --- packages/internal/src/js-utils.js | 19 ++++++++++++++++++ .../test/snapshots/exports.test.js.md | 1 + .../test/snapshots/exports.test.js.snap | Bin 687 -> 703 bytes 3 files changed, 20 insertions(+) diff --git a/packages/internal/src/js-utils.js b/packages/internal/src/js-utils.js index 70cd6d19c4b..c46b1f0931c 100644 --- a/packages/internal/src/js-utils.js +++ b/packages/internal/src/js-utils.js @@ -5,6 +5,8 @@ * dependent upon a hardened environment. */ +const { defineProperty } = Object; + /** * Deep-copy a value by round-tripping it through JSON (which drops * function/symbol/undefined values and properties that are non-enumerable @@ -66,6 +68,23 @@ const deepMapObjectInternal = (value, name, container, mapper) => { export const deepMapObject = (obj, mapper) => deepMapObjectInternal(obj, undefined, undefined, mapper); +/** + * Explicitly set a function's name, supporting use of arrow functions for which + * source text doesn't include a name and no initial name is set by + * NamedEvaluation + * https://tc39.es/ecma262/multipage/syntax-directed-operations.html#sec-runtime-semantics-namedevaluation + * + * `name` is the first parameter for readability at call sites (e.g., `return + * defineName('foo', () => { ... })`). + * + * @template {Function} T + * @param {string} name + * @param {T} fn + * @returns {T} + */ +export const defineName = (name, fn) => + defineProperty(fn, 'name', { value: name }); + /** * Returns a function that uses a millisecond-based current-time capability * (such as `performance.now`) to measure execution duration of an async diff --git a/packages/internal/test/snapshots/exports.test.js.md b/packages/internal/test/snapshots/exports.test.js.md index d36f773467d..9f95df26959 100644 --- a/packages/internal/test/snapshots/exports.test.js.md +++ b/packages/internal/test/snapshots/exports.test.js.md @@ -26,6 +26,7 @@ Generated by [AVA](https://avajs.dev). 'deepCopyJsonable', 'deepMapObject', 'deeplyFulfilledObject', + 'defineName', 'forever', 'fromUniqueEntries', 'getMethodNames', diff --git a/packages/internal/test/snapshots/exports.test.js.snap b/packages/internal/test/snapshots/exports.test.js.snap index e3a78b366a094d39c7f13e0a2ce4c66a94fb2b41..6acc3441b55ad825b0fdfab64c9313ad002adefe 100644 GIT binary patch literal 703 zcmV;w0zmyiRzVkZ*bXsICHQpT@#$y_Wr6DVZZ0eCUUFl$1sZG0Xt+6gRgTa|>cwZAgt~FGyoS8pVINh01~2U!rcpIj|ltxCU*hf#y>R#zYI(4D%ZJ)Wmecc*u*9 zaMXE%&W!=GRg?7|IHN^NQS)*z#`{K1WSgyL5YYjXA)f8Ws<)A=w#3`V7qtq_CO)ox zel}Af8*Up5&%uUZnii{#uXRhy-5l1nxAJT*Gj!_&{K<{`M^io<@)^t<_^3|)V_l~G z>52~2K)BpO10ti(<~K(-=sQg-qO+n`=*4w&=WD`8w3_o2nd5=r07G<@CVLuz7w6&y zdSRK2I^o}IR5innYoh;%b!mU$sLu7^lwD6psWn>eL)-+K_zfzB>nl~$<`gYbw3kW+ l=cAjn!+X-FxPD+Q#H$K~&Z+gSAQkpy_7|Sv3a9}E006#!PyPS^ literal 687 zcmV;g0#N-yRzV*N8&y&h%EB)8XsG+Rur6v zr)ssz_%!498J|I;#FluO@v$1Oty&o;uqu;=_U%+;aAt5Gl*v=25r?D{lKPr)L*p7+ z`WH}i%)eqeh|?{jTe(uwr6H??Y~iIfL+M~esV%!m(O8hWHt%3~VD96Tz6-SrvM)5=AE0KBSY>uq%`T-x5Pwdn#m$B1tSH!b8Uvn^L1$ z1Zg@*Q}}jAs2r&M73wCO1M6{v>(G}P=zL~^G1tQp!@LPTH8HI)8}dpd9Cesg6iwp4$x|B>tgYfnyRn4&Dx)^<8 zT{;>ns&gwiRafXJwMNT5h&w>{cAHA!h)UJ5IYoCU+D)Z`i_u-0(_QIP95JvK;xz?A V=hXUMkji^9`vb`Bjm_Z%001K3M6Un< From 4ceea267a3a83e1ee94f77f6fd1d3f5bd250dd60 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Mon, 6 Jan 2025 21:53:34 -0500 Subject: [PATCH 11/18] feat(cosmic-swingset): Introduce inquisitor.mjs --- packages/cosmic-swingset/package.json | 6 +- packages/cosmic-swingset/tools/inquisitor.mjs | 825 ++++++++++++++++++ packages/swing-store/src/bundleStore.js | 5 +- packages/swing-store/src/index.js | 6 +- packages/swing-store/src/internal.js | 2 + packages/swing-store/src/swingStore.js | 42 +- 6 files changed, 863 insertions(+), 23 deletions(-) create mode 100755 packages/cosmic-swingset/tools/inquisitor.mjs diff --git a/packages/cosmic-swingset/package.json b/packages/cosmic-swingset/package.json index f048e15dfd5..ce6b0cfce83 100644 --- a/packages/cosmic-swingset/package.json +++ b/packages/cosmic-swingset/package.json @@ -52,8 +52,12 @@ "tmp": "^0.2.1" }, "devDependencies": { + "@agoric/kmarshal": "^0.1.0", + "@endo/eventual-send": "^1.2.8", "ava": "^5.3.0", - "c8": "^10.1.2" + "better-sqlite3": "^9.1.1", + "c8": "^10.1.2", + "ses": "^1.10.0" }, "publishConfig": { "access": "public" diff --git a/packages/cosmic-swingset/tools/inquisitor.mjs b/packages/cosmic-swingset/tools/inquisitor.mjs new file mode 100755 index 00000000000..eadcbee893b --- /dev/null +++ b/packages/cosmic-swingset/tools/inquisitor.mjs @@ -0,0 +1,825 @@ +#!/usr/bin/env node +/** + * @file Interact with the database and/or vats of a swingstore.sqlite in an + * ephemeral environment. This file functions as both an importable module and + * as a standalone interactive or non-interactive script. See + * "Check for CLI invocation" below for usage detail about the latter. + */ +/* eslint-env node */ +/* global globalThis */ +/* eslint-disable no-empty */ + +// Overwrite the global console for deeper inspection. +// @ts-expect-error TS2307 Cannot find module +import 'data:text/javascript,import { Console } from "node:console"; const { stdout, stderr, env } = process; const inspectOptions = { depth: Number(env.CONSOLE_INSPECT_DEPTH) || 6 }; globalThis.console = new Console({ stdout, stderr, inspectOptions });'; + +import 'ses'; +import '@endo/eventual-send/shim.js'; +import '@endo/init/pre.js'; +// __hardenTaming__: "unsafe" is unfortunate, but without it, automatic +// hardening discovers EventEmitter.prototype and breaks creation of new event +// emitters (e.g., `Readable.from(...)`) because initialization is vulnerable to +// the property assignment override mistake w.r.t. _events/_eventsCount/etc. +// https://github.com/nodejs/node/blob/v22.12.0/lib/events.js#L347 +// @ts-expect-error TS2307 Cannot find module +import 'data:text/javascript,try { lockdown({ domainTaming: "unsafe", errorTaming: "unsafe-debug", __hardenTaming__: "unsafe" }); } catch (_err) {}'; + +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import pathlib from 'node:path'; +import repl from 'node:repl'; +import stream from 'node:stream'; +import { + setImmediate as resolveImmediate, + setTimeout as delay, +} from 'node:timers/promises'; +import { fileURLToPath } from 'node:url'; +import { inspect, parseArgs } from 'node:util'; +import { isMainThread } from 'node:worker_threads'; +// eslint-disable-next-line import/no-extraneous-dependencies +import sqlite3 from 'better-sqlite3'; +import { Fail, b, q } from '@endo/errors'; +import { makePromiseKit } from '@endo/promise-kit'; +import { objectMap, BridgeId } from '@agoric/internal'; +import { QueuedActionType } from '@agoric/internal/src/action-types.js'; +import { defineName } from '@agoric/internal/src/js-utils.js'; +import { makeFakeStorageKit } from '@agoric/internal/src/storage-test-utils.js'; +// eslint-disable-next-line import/no-extraneous-dependencies +import { krefOf, kser, kslot, kunser } from '@agoric/kmarshal'; +import { + openSwingStore, + makeBundleStore, + bundleIDFromName, +} from '@agoric/swing-store'; +import { + makeBufferedStorage, + provideEnhancedKVStore, +} from '../src/helpers/bufferedStorage.js'; +import { + DEFAULT_SIM_SWINGSET_PARAMS, + makeVatCleanupBudgetFromKeywords, +} from '../src/sim-params.js'; +import { makeCosmicSwingsetTestKit } from './test-kit.js'; + +/** @import { ManagerType, SwingSetConfig } from '@agoric/swingset-vat' */ +/** @import { KVStore } from '../src/helpers/bufferedStorage.js' */ + +const useColors = process.stdout?.hasColors?.(); +const inspectDepth = 6; + +const dataProp = { writable: true, enumerable: true, configurable: true }; +const empty = Object.create(null); +const noop = () => {}; +const parseNumber = input => (input.match(/[0-9]/) ? Number(input) : NaN); + +// cf. packages/swing-store/src/exporter.js +const storeExportAPI = ['getExportRecords', 'getArtifactNames']; + +// TODO: getVatAdminNode('v112') # scan the vatAdmin vom v2.vs.vom.* vrefs for value matching /\b${vatID}\b/ +const makeHelpers = ({ db, EV }) => { + const sqlKVGet = db + .prepare('SELECT value FROM kvStore WHERE key = ?') + .pluck(); + const kvGet = key => sqlKVGet.get(key); + const kvGetJSON = key => JSON.parse(kvGet(key)); + + const sqlKVByRange = db.prepare( + `SELECT key, value FROM kvStore WHERE key >= :a AND key < :b AND ${[ + '(:keySuffix IS NULL OR substr(key, -length(:keySuffix)) = :keySuffix)', + '(:keyGlob IS NULL OR key GLOB :keyGlob)', + '(:valueGlob IS NULL OR value GLOB :valueGlob)', + ].join(' AND ')}`, + ); + const sqlKVByHalfRange = db.prepare( + `SELECT key, value FROM kvStore WHERE key >= :a AND ${[ + '(:keySuffix IS NULL OR substr(key, -length(:keySuffix)) = :keySuffix)', + '(:keyGlob IS NULL OR key GLOB :keyGlob)', + '(:valueGlob IS NULL OR value GLOB :valueGlob)', + ].join(' AND ')}`, + ); + const kvGlob = (keyGlob, valueGlob = undefined, lazy = false) => { + const [_keyPattern, keyPrefix, keyTail, keySuffix] = + /** @type {string[]} */ (/^([^*?]*)((?:[*?]([^*?]*))*)$/.exec(keyGlob)); + let sql = sqlKVByHalfRange; + /** @type {Record<'a' | 'b' | 'keySuffix' | 'keyGlob' | 'valueGlob', string | null>} */ + const args = { + a: keyPrefix, + b: null, + keySuffix: keySuffix || null, + keyGlob: keyTail && keyTail !== '*' ? keyGlob : null, + valueGlob: valueGlob ?? null, + }; + const chars = [...keyPrefix]; + const i = chars.findLastIndex(ch => ch < '\u{10FFFF}'); + if (i !== -1) { + sql = sqlKVByRange; + const newLastCP = /** @type {number} */ (chars[i].codePointAt(0)) + 1; + args.b = chars.slice(0, i).join('') + String.fromCodePoint(newLastCP); + } else { + console.warn('Warning: Unprefixed searches can be slow'); + } + return lazy ? sql.iterate(args) : sql.all(args); + }; + + let vatsByID = new Map(); + let vatsByName = new Map(); + try { + // @see {@link ../../SwingSet/src/kernel/state/kernelKeeper.js} + kvGetJSON('vat.names').every( + name => + typeof name === 'string' || + Fail`static vat name ${q(name)} must be a string`, + ); + kvGetJSON('vat.dynamicIDs').every( + vatID => + typeof vatID === 'string' || + Fail`dynamic vatID ${q(vatID)} must be a string`, + ); + const vatQuery = db.prepare(` + WITH vat AS ( + SELECT 1 AS rank, nameJSON.key AS idx, vatNameToID.value AS vatID + FROM kvStore AS nameList + LEFT JOIN json_each(nameList.value) AS nameJSON + LEFT JOIN kvStore AS vatNameToID + ON vatNameToID.key = 'vat.name.' || nameJSON.atom + WHERE nameList.key='vat.names' + UNION SELECT 2 as rank, idJSON.key AS idx, idJSON.value AS vatID + FROM kvStore AS idList, json_each(idList.value) AS idJSON + WHERE idList.key='vat.dynamicIDs' + ) + SELECT vat.vatID, rank, source.value AS sourceText, options.value AS optionsText + FROM vat + LEFT JOIN kvStore AS source ON source.key = vat.vatID || '.source' + LEFT JOIN kvStore AS options ON options.key = vat.vatID || '.options' + ORDER BY vat.rank, vat.idx + `); + for (const dbRecord of vatQuery.iterate()) { + const { vatID, rank, sourceText, optionsText } = dbRecord; + const isStatic = rank === 1; + const source = sourceText ? JSON.parse(sourceText) : undefined; + const options = optionsText ? JSON.parse(optionsText) : undefined; + const name = options?.name; + const vat = harden({ vatID, name, isStatic, source, options }); + vatsByID.set(vatID, vat); + if (name) { + const conflict = vatsByName.get(name); + if (vat.isStatic || !conflict) { + // Static vats trump dynamic vats in vatsByName. + vatsByName.set(name, vat); + } else if (!Array.isArray(conflict)) { + // ...but dynamic vats with duplicate names get collected into arrays. + vatsByName.set(name, [conflict, vat]); + } else { + conflict.push(vat); + } + } + } + } catch (err) { + console.warn('Warning: Could not build vat maps', err); + // @ts-expect-error + vatsByID = undefined; + // @ts-expect-error + vatsByName = undefined; + } + + const vatIDPatt = /^v[1-9][0-9]*$/; + // @see {@link ../../SwingSet/docs/c-lists.md} + // @see {@link ../../swingset-liveslots/src/vatstore-usage.md} + const refPatt = + /(?^k[opd][1-9][0-9]*$)|(?^[opd][+-][1-9][0-9]*$|^(?o[+][vd]?(?[1-9][0-9]*)\/[1-9][0-9]*)(?:(?0|[1-9][0-9]*))?)/; + const krefToVrefValuePatt = /^([R_]) ([^ ]+)$/; + const getKindMeta = (vatID, kindID) => { + const kindMetaJSON = + kvGet(`${vatID}.vs.vom.dkind.${kindID}.descriptor`) || + kvGet(`${vatID}.vs.vom.vkind.${kindID}.descriptor`); + return JSON.parse(kindMetaJSON); + }; + + /** + * @param {string} refID kref or vref + * @param {string} [contextVatID] + * @returns {Array<{vatID: string, kref: string, vref: string, kind?: string, facet?: string}>} + */ + const getRefs = (refID, contextVatID = undefined) => { + const refParts = refID.match(refPatt)?.groups; + if (!refParts) throw Fail`unknown kref or vref format in ${refID}`; + const isKref = !!refParts.kref; + contextVatID === undefined || + contextVatID.match(vatIDPatt) || + Fail`invalid contextVatID ${contextVatID}`; + + // Search for rows like (`v${vatID}.c.${kref}`, `${flag} ${vref}`), where + // kref might be exracted from rows like (`v${vatID}.c.${vref}`, kref). + // @see {@link ../../SwingSet/docs/c-lists.md} + const krefs = []; + let kindMeta; + if (isKref) { + krefs.push(refID); + } else { + const maybeKref = kref => kref && krefs.push(kref); + maybeKref(kvGet(`${contextVatID}.c.${refID}`)); + const { baseref, kindID, facetID } = refParts; + if (kindID && !facetID) { + // Each facet might have its own kref. + kindMeta = getKindMeta(contextVatID, kindID); + const facetNames = kindMeta?.facets; + for (let i = 0; i < (facetNames?.length ?? 0); i += 1) { + maybeKref(kvGet(`${contextVatID}.c.${baseref}:${i}`)); + } + } + } + // Don't scan when we can enumerate keys. + const results = []; + for (const vatID of vatsByID.keys()) { + for (const kref of krefs) { + const value = kvGet(`${vatID}.c.${kref}`); + if (!value) continue; + const [_value, _reachabilityFlag, vref] = + value.match(krefToVrefValuePatt) || + Fail`unexpected c-list value ${value}`; + const result = { vatID, kref, vref }; + const { kindID, facetID } = vref.match(refPatt)?.groups || empty; + if (kindID) { + // kindID appears only in vrefs for the exporting vat, where we either + // get metadata on the first try or not at all. + if (kindMeta !== null) { + kindMeta ||= getKindMeta(vatID, kindID) || null; + } + result.kind = kindMeta?.tag; + if (facetID) result.facet = kindMeta?.facets?.[facetID]; + } + results.push(result); + } + } + return results; + }; + + /** + * Run a core-eval directly through the controller (i.e., without a block). + * + * @param {string} fnText must evaluate to a function that will be invoked in + * a core eval compartment with a "powers" argument as attenuated by + * `permits` (with no attenuation by default). + * @param {import('@agoric/vats/src/core/lib-boot.js').BootstrapManifestPermit} [permits] + */ + const runCoreEval = async (fnText, permits = true) => { + // Fail noisily if fnText does not evaluate to a function. + // This must be refactored if there is ever a need for such input. + const fn = new Compartment().evaluate(fnText); + typeof fn === 'function' || Fail`text must evaluate to a function`; + /** @type {import('@agoric/cosmic-proto/swingset/swingset.js').CoreEvalSDKType} */ + const coreEvalDesc = { + json_permits: JSON.stringify(permits), + js_code: fnText, + }; + const coreEvalAction = { + type: QueuedActionType.CORE_EVAL, + evals: [coreEvalDesc], + }; + // Assume a path to the coreEvalBridgeHandler. + const coreEvalBridgeHandler = await EV.vat('bootstrap').consumeItem( + 'coreEvalBridgeHandler', + ); + return EV(coreEvalBridgeHandler).fromBridge(coreEvalAction); + }; + + return harden({ + runCoreEval, + stable: { db, getRefs, kvGet, kvGetJSON, kvGlob, vatsByID, vatsByName }, + }); +}; + +/** + * Wrap a swing-store sub-store (kvStore/transcriptStore/etc.) with a + * replacement whose functions log and/or track/respond to staleness. + * + * @template {object} Substore + * @param {string} storeName + * @param {Substore} store + * @param {object} [options] + * @param {(...args: unknown[]) => void} [options.log] + * @param {(...args: unknown[]) => void} [options.warn] + * @param {(key?: unknown) => boolean} [options.isClean] + * @param {(key?: unknown) => boolean} [options.isStale] + * @param {(key?: unknown) => void} [options.markStale] + * @param {string[]} [options.allow] functions to allow + * @param {string[]} [options.allowIfClean] functions to allow when `isClean(firstArg)` returns true + * @param {string[]} [options.allowAndMark] functions to augment with `markStale(firstArg)` + * @param {Array} [options.logAndMark] functions to replace with `log(storeName, functionName, firstArg, ...details)` and `markStale(firstArg)` + * @param {string[]} [options.warnIfStale] functions to augment with `warn(storeName, functionName, firstArg, ...details)` when `isStale(firstArg)` returns true + * @param {string[]} [options.disallow] functions to disallow + * @returns {Substore} + */ +export const wrapSubstore = (storeName, store, options = {}) => { + const { + log = console.log, + warn = console.warn, + isClean = () => Fail`[inquisitor] cannot check isClean in ${storeName}`, + isStale = () => Fail`[inquisitor] cannot check isStale in ${storeName}`, + markStale = () => Fail`[inquisitor] cannot markStale in ${storeName}`, + allow = [], + allowIfClean = [], + allowAndMark = [], + logAndMark: rawLogAndMark = [], + warnIfStale = [], + disallow = [], + } = options; + const logAndMarkMap = new Map( + rawLogAndMark.map(x => (Array.isArray(x) ? x : [x, noop])), + ); + const flat = (...arrs) => [].concat(...arrs); + /** @type {Set} */ + const unseen = new Set( + flat(allow, allowIfClean, allowAndMark, warnIfStale, disallow), + ); + for (const name of logAndMarkMap.keys()) unseen.add(name); + const wrapped = objectMap( + /** @type {Record} */ (store), + (fn, name) => { + if (typeof name !== 'string') { + throw Fail`[inquisitor] non-string property ${b(storeName)}[${q(name)}]`; + } + unseen.delete(name); + if (allow.includes(name)) { + return fn; + } else if (allowIfClean.includes(name)) { + return defineName(name, (key, ...rest) => { + isClean(key) || + Fail`[inquisitor] ${b(storeName)}.${b(name)}(${b(key)}) after mutations`; + return fn(key, ...rest); + }); + } else if (allowAndMark.includes(name)) { + return defineName(name, (key, ...rest) => { + markStale(key); + return fn(key, ...rest); + }); + } else if (logAndMarkMap.has(name)) { + const makeResult = /** @type {Function} */ (logAndMarkMap.get(name)); + return defineName(name, (key, ...rest) => { + markStale(key); + log(storeName, name, key, ...rest); + return makeResult(key, ...rest); + }); + } else if (warnIfStale.includes(name)) { + return defineName(name, (key, ...rest) => { + if (isStale(key)) { + warn(storeName, name, key, 'returning stale data'); + } + return fn(key, ...rest); + }); + } else if (disallow.includes(name)) { + return defineName( + name, + () => Fail`[inquisitor] disallowed ${b(storeName)}.${b(name)}`, + ); + } else { + throw Fail`[inquisitor] unknown ${b(storeName)} function ${b(name)}; time to update?`; + } + }, + ); + unseen.size === 0 || + Fail`[inquisitor] ${b(storeName)} lacked ${q([...unseen])}; time to update?`; + // @ts-expect-error cast + return wrapped; +}; +harden(wrapSubstore); + +/** + * Make an overlay-like swing-store that buffers all mutations over a read-only + * database. + * If this ever needs substantial refactoring, consider pushing the + * functionality into swing-store itself. + * + * @param {string} dbPath to a swingstore.sqlite file + * @param {typeof wrapSubstore} wrapStore a function to replace swing-store sub-stores (kvStore/transcriptStore/etc.) + */ +export const makeSwingStoreOverlay = (dbPath, wrapStore = wrapSubstore) => { + /** @type {Array<[storeName: string, operation: string, ...args: unknown[]]>} */ + const mutations = []; + const recordCall = (storeName, operation, ...details) => + mutations.push([storeName, operation, ...details]); + const makeWrapHelpers = () => { + const modifiedVats = new Set(); + return { + log: recordCall, + isClean: () => modifiedVats.size === 0, + isStale: vatID => modifiedVats.has(vatID), + markStale: vatID => modifiedVats.add(vatID), + }; + }; + + const kvListeners = { + onPendingSet: (key, value) => recordCall('kvStore', 'set', key, value), + onPendingDelete: key => recordCall('kvStore', 'delete', key), + }; + const swingStore = openSwingStore(dbPath, { + asFile: true, + readonly: true, + wrapKvStore: base => makeBufferedStorage(base, kvListeners).kvStore, + wrapTranscriptStore: transcriptStore => { + const wrapHelpers = makeWrapHelpers(); + const pendingItemsByVat = new Map(); + /** @type {ReturnType} */ + const transcriptStoreOverride = { + ...transcriptStore, + addItem: (vatID, item) => { + recordCall('transcriptStore', 'addItem', vatID, item); + if (wrapHelpers.isStale(vatID)) return; + const pendingItems = pendingItemsByVat.get(vatID) || []; + const { startPos, endPos, hash, incarnation } = + pendingItems.at(-1) || transcriptStore.getCurrentSpanBounds(vatID); + pendingItems.push({ + item, + startPos, + endPos: endPos + 1, + hash: hash && '', + incarnation, + }); + pendingItemsByVat.set(vatID, pendingItems); + }, + getCurrentSpanBounds: vatID => { + const pendingItems = pendingItemsByVat.get(vatID) || []; + const { startPos, endPos, hash, incarnation } = + pendingItems.at(-1) || transcriptStore.getCurrentSpanBounds(vatID); + return { startPos, endPos, hash, incarnation }; + }, + readSpan: (vatID, startPos) => { + const reader = function* reader() { + try { + // Read from the base store. + yield* transcriptStore.readSpan(vatID, startPos); + } catch (_err) {} + // Read from the overlay, assuming that any transcripts of vatID + // are for the current span. + const pendingItems = pendingItemsByVat.get(vatID) || []; + for (const { item, startPos: itemStartPos } of pendingItems) { + if (startPos !== undefined && itemStartPos !== startPos) break; + yield item; + } + }; + return reader(); + }, + }; + return wrapStore('transcriptStore', transcriptStoreOverride, { + ...wrapHelpers, + logAndMark: [ + 'initTranscript', + 'rolloverSpan', + 'rolloverIncarnation', + 'stopUsingTranscript', + ['deleteVatTranscripts', () => harden({ done: true, cleanups: 0 })], + ], + warnIfStale: ['addItem', 'getCurrentSpanBounds', 'readSpan'], + allowIfClean: [ + ...storeExportAPI, + 'exportSpan', + 'dumpTranscripts', + 'readFullVatTranscript', + ], + disallow: [ + 'importTranscriptSpanRecord', + 'populateTranscriptSpan', + 'assertComplete', + 'repairTranscriptSpanRecord', + ], + }); + }, + wrapSnapStore: snapStore => { + const wrapHelpers = makeWrapHelpers(); + /** @type {ReturnType} */ + const snapStoreOverride = { + ...snapStore, + saveSnapshot: async (vatID, snapPos, dataStream) => { + const entryPrefix = ['snapStore', 'saveSnapshot', vatID, snapPos]; + wrapHelpers.markStale(vatID); + await null; + let size = 0; + try { + for await (const chunk of dataStream) size += chunk.length; + recordCall(...entryPrefix, `<${size} bytes>`); + } catch (err) { + recordCall(...entryPrefix, ``); + throw err; + } + return /** @type {import('@agoric/swing-store').SnapshotResult} */ ( + harden({ uncompressedSize: size }) + ); + }, + }; + return wrapStore('snapStore', snapStoreOverride, { + ...wrapHelpers, + allow: ['saveSnapshot'], + logAndMark: [ + ['deleteVatSnapshots', () => harden({ done: true, cleanups: 0 })], + 'stopUsingLastSnapshot', + ], + warnIfStale: ['loadSnapshot', 'getSnapshotInfo', 'hasHash'], + allowIfClean: [ + ...storeExportAPI, + 'exportSnapshot', + 'listAllSnapshots', + 'dumpSnapshots', + ], + disallow: [ + 'deleteAllUnusedSnapshots', + 'importSnapshotRecord', + 'populateSnapshot', + 'assertComplete', + 'repairSnapshotRecord', + 'deleteSnapshotByHash', + ], + }); + }, + wrapBundleStore: bundleStore => { + const overlayDB = sqlite3(':memory:'); + const overlay = makeBundleStore(overlayDB, noop, noop); + let modified = false; + const onNewBundle = (operation, key, ...details) => { + modified = true; + recordCall('bundleStore', operation, key, ...details); + const bundleID = bundleIDFromName(key); + !bundleStore.hasBundle(bundleID) || + Fail`base bundleStore already has ${bundleID}`; + }; + /** @type {ReturnType} */ + const bundleStoreOverride = { + ...bundleStore, + // writes + importBundleRecord: (key, value) => { + onNewBundle('importBundleRecord', key, value); + return overlay.importBundleRecord(key, value); + }, + importBundle: async (name, dataProvider) => { + const data = await dataProvider(); + onNewBundle('importBundle', name, `<${data.length} bytes>`); + return overlay.importBundle(name, () => Promise.resolve(data)); + }, + addBundle: (bundleID, bundle) => { + onNewBundle('addBundle', `bundle.${bundleID}`, bundle.moduleFormat); + return overlay.addBundle(bundleID, bundle); + }, + deleteBundle: bundleID => { + modified = true; + recordCall('bundleStore', 'deleteBundle', bundleID); + if (overlay.hasBundle(bundleID)) overlay.deleteBundle(bundleID); + }, + // reads + hasBundle: bundleID => + overlay.hasBundle(bundleID) || bundleStore.hasBundle(bundleID), + getBundle: bundleID => { + if (overlay.hasBundle(bundleID)) return overlay.getBundle(bundleID); + return bundleStore.getBundle(bundleID); + }, + }; + return wrapStore('bundleStore', bundleStoreOverride, { + log: recordCall, + isClean: () => !modified, + allow: [ + 'importBundleRecord', + 'importBundle', + 'addBundle', + 'hasBundle', + 'getBundle', + 'deleteBundle', + ], + allowIfClean: [ + ...storeExportAPI, + 'exportBundle', + 'getBundleIDs', + 'dumpBundles', + ], + disallow: ['assertComplete', 'repairBundleRecord'], + }); + }, + }); + + return { swingStore, mutations }; +}; +harden(makeSwingStoreOverlay); + +/** + * Load a swing-store database for either REPL or scripted interactions. + * + * @param {[swingstoreDbPath: string]} argv + * @param {{ interactive?: boolean, historyFile?: string }} [options] + * @param {{ console?: typeof globalThis.console, process?: typeof globalThis.process }} [powers] + */ +const main = async (argv, options = {}, powers = {}) => { + const { interactive, historyFile } = options; + const { console = globalThis.console, process = globalThis.process } = powers; + const { env } = process; + const maxVatsOnline = parseNumber(env.INQUISITOR_MAX_VATS_ONLINE || '3'); + + const { swingStore, mutations } = makeSwingStoreOverlay(argv[0]); + const { db, kvStore } = swingStore.internal; + const fakeStorageKit = makeFakeStorageKit(''); + const { toStorage: handleVstorage } = fakeStorageKit; + const receiveBridgeSend = (destPort, msg) => { + console.log('[bridge] received', msg); + switch (destPort) { + case BridgeId.STORAGE: { + return handleVstorage(msg); + } + default: + Fail`[inquisitor] bridge port ${q(destPort)} not implemented for message ${msg}`; + } + }; + const config = { + swingsetConfig: { maxVatsOnline }, + swingStore, + /** @type {Partial} */ + configOverrides: { + // Default to XS workers with no GC or snapshots. + defaultManagerType: 'xsnap', + defaultReapGCKrefs: 'never', + defaultReapInterval: 'never', + snapshotInterval: Number.MAX_VALUE, + }, + fixupInitMessage: msg => ({ + ...msg, + blockHeight: Number(swingStore.hostStorage.kvStore.get('host.height')), + blockTime: Math.floor(Date.now() / 1000 - 60), + // Default to no cleanup for terminated vats. + params: { + ...DEFAULT_SIM_SWINGSET_PARAMS, + ...msg.params, + vat_cleanup_budget: makeVatCleanupBudgetFromKeywords({ Default: 0 }), + }, + }), + }; + const testKit = await makeCosmicSwingsetTestKit(receiveBridgeSend, config); + + const { + EV, + controller, + shutdown, + getLastBlockInfo, + pushQueueRecord, + pushCoreEval, + runNextBlock, + } = testKit; + const helpers = makeHelpers({ db, EV }); + const endowments = { + // Raw access to overlay data. + ...{ kvStore: provideEnhancedKVStore(kvStore), swingStore }, + // Block interactions + ...{ getLastBlockInfo, pushQueueRecord, pushCoreEval, runNextBlock }, + // Vat interactions. + ...{ EV, controller, krefOf, kser, kslot, kunser }, + // Inquisitor API. + ...{ mutations, ...helpers }, + }; + const contextDescriptors = objectMap( + { console, endowments, ...endowments, shutdown }, + (value, name) => { + // For final cleanup, `shutdown` must be preserved. + if (name === 'shutdown') { + return { ...dataProp, value, writable: false, configurable: false }; + } + return { ...dataProp, value }; + }, + ); + + if (!interactive) { + Object.defineProperties(globalThis, contextDescriptors); + return; + } + + const truthyKeys = obj => + Object.entries(obj).flatMap(([key, value]) => (value ? [key] : [])); + console.warn('endowments:', ...truthyKeys(endowments)); + console.warn('endowments.stable:', ...truthyKeys(endowments.stable)); + const replServer = repl.start({ + useGlobal: true, + // @ts-expect-error TS2322 REPLWriter really is allowed to return an Error + writer: value => { + if (value instanceof Error) { + // Use the SES console. + console.error(value); + return Object.defineProperty(Error(value.message), 'name', { + value: value.name, + }); + } + return inspect(value, { colors: useColors, depth: inspectDepth }); + }, + }); + if (historyFile) replServer.setupHistory(historyFile, _err => {}); + Object.defineProperties(replServer.context, contextDescriptors); + const cleanup = () => shutdown().catch(noop); + replServer.on('exit', cleanup); + process.on('beforeExit', cleanup); +}; + +// Check for CLI invocation. +const isImport = + fs.realpathSync(process.argv[1]) !== fileURLToPath(import.meta.url); +const isCLIEntryPoint = !isImport && !process.send && isMainThread !== false; +const interactive = process.stdin.isTTY && !process.env.INQUISITOR_NO_REPL; +if (isCLIEntryPoint && !interactive) { + // When directly invoked with non-interactive stdin, defer to a child process + // that will read stdin as module statements in the global environment with + // `EV`/`controller`/`kvStore`/etc. + const args = [ + '--input-type=module', + ...['--import', process.argv[1]], + '', + ...process.argv.slice(2), + ]; + const child = spawn(process.argv[0], args, { + env: { ...process.env, INQUISITOR_NO_REPL: '1' }, + stdio: ['pipe', 'inherit', 'inherit', 'ipc'], + }); + const { promise: childDoneP, resolve: finishChild } = makePromiseKit(); + child.on('error', error => setImmediate(finishChild, { error })); + child.on('exit', (code, signal) => finishChild({ code, signal })); + void childDoneP.then(resolveImmediate).then(async result => { + await null; + if (result?.signal) { + process.kill(process.pid, result.signal); + await delay(100); + } + if (typeof result?.code === 'number') process.exit(result.code); + const { error } = result; + console.error(error); + process.exit(error.code || 1); + }); + const childInput = child.stdin; + if (!childInput) throw Fail`[inquisitor] child must have stdin`; + process.stdin.pipe(childInput, { end: false }); + process.stdin.on('end', () => { + const cleanup = `\n; try { await shutdown(); } catch (_err) {}`; + stream.Readable.from([cleanup]).pipe(childInput); + }); +} else if (isCLIEntryPoint || process.env.INQUISITOR_NO_REPL) { + // When directly invoked with interactive stdin OR as a worker above, parse + // CLI arguments and use `main` to setup the environment for either a REPL or + // evaluating stdin as module statements (respectively). + const homedir = os.homedir(); + const defaultHistFile = pathlib.join( + homedir, + '.agoric_inquisitor_repl_history', + ); + /** @typedef {{type: 'string' | 'boolean', short?: string, multiple?: boolean, default?: string | boolean | string[] | boolean[]}} ParseArgsOptionConfig */ + /** @type {Record} */ + const cliOptions = { + help: { type: 'boolean' }, + 'history-file': { + type: 'string', + default: defaultHistFile, + }, + }; + const { values: options, positionals: args } = parseArgs({ + options: cliOptions, + allowPositionals: true, + }); + try { + if (options.help) throw Error(); + args.length >= 1 || Fail`missing swingstore.sqlite`; + args.length === 1 || Fail`extra arguments`; + } catch (err) { + const log = options.help ? console.log : console.error; + if (!options.help) log(`Error: ${err.message}`); + const self = pathlib.relative(process.cwd(), process.argv[1]); + log(`Usage: ${self} swingstore.sqlite \\ + [--history-file PATH (default ${cliOptions['history-file'].default})] + +Loads an ephemeral environment in which one or more vats may be probed +via \`EV\`/\`controller\`/\`kvStore\`/\`mutations\`/etc. without persisting changes. +May be used interactively, or as a recipient of piped commands, or as a module. +Example commands: +* stable.db.prepare("SELECT name FROM sqlite_schema WHERE type='table'").pluck().all(); +* stable.db.pragma("table_info(transcriptSpans)"); +* [vatAdminNodeRow] = stable.db.kvGlob('v2.vs.*', '*v100*'); +* stable.getRefs('o+10', 'v1'); +* board = await EV.vat('bootstrap').consumeItem('board'); +* obj = await EV(board).getValue('board02963'); +* await runCoreEval(\`async powers => { + const ref = await E.get(powers.consume.auctioneerKit).governorAdminFacet; + console.log(ref); + powers.produce.ref.resolve(ref); + }\`); +* (await EV.vat('bootstrap').consumeItem('ref')).getKref() + +ENVIRONMENT VARIABLES + CONSOLE_INSPECT_DEPTH + The number of times to recurse while formatting an object (default 6). + INQUISITOR_MAX_VATS_ONLINE + The maximum number of vats to have in memory at any given time (default 3).`); + process.exit(64); + } + const camelizedOptions = Object.fromEntries( + Object.entries(options).map(([name, value]) => [ + name.replaceAll(/-([a-z])/g, (_, letter) => `${letter.toUpperCase()}`), + value, + ]), + ); + // eslint-disable-next-line @jessie.js/safe-await-separator + await main(/** @type {[string]} */ (args), { + ...camelizedOptions, + interactive, + }).catch(err => { + console.error(err); + process.exit(err.code || 1); + }); +} diff --git a/packages/swing-store/src/bundleStore.js b/packages/swing-store/src/bundleStore.js index ebabd352e59..e140404c36b 100644 --- a/packages/swing-store/src/bundleStore.js +++ b/packages/swing-store/src/bundleStore.js @@ -42,7 +42,7 @@ import { createSHA256 } from './hasher.js'; * */ -function bundleIDFromName(name) { +export const bundleIDFromName = name => { typeof name === 'string' || Fail`artifact name must be a string`; const [tag, ...pieces] = name.split('.'); if (tag !== 'bundle' || pieces.length !== 1) { @@ -52,7 +52,8 @@ function bundleIDFromName(name) { } const bundleID = pieces[0]; return bundleID; -} +}; +harden(bundleIDFromName); /** * @param {*} db diff --git a/packages/swing-store/src/index.js b/packages/swing-store/src/index.js index 1a927443679..c3ae03d4d83 100644 --- a/packages/swing-store/src/index.js +++ b/packages/swing-store/src/index.js @@ -4,10 +4,12 @@ export { importSwingStore } from './importer.js'; export { makeArchiveSnapshot, makeArchiveTranscript } from './archiver.js'; -// temporary, for the benefit of SwingSet/misc-tools/replay-transcript.js +// for the benefit of tools like SwingSet/misc-tools/replay-transcript.js +export { makeKVStore, getKeyType } from './kvStore.js'; +export { makeTranscriptStore } from './transcriptStore.js'; export { makeSnapStore } from './snapStore.js'; -// and less temporary, for SwingSet/test/vat-warehouse/test-reload-snapshot.js export { makeSnapStoreIO } from './snapStoreIO.js'; +export { makeBundleStore, bundleIDFromName } from './bundleStore.js'; // eslint-disable-next-line import/export export * from './types-index.js'; diff --git a/packages/swing-store/src/internal.js b/packages/swing-store/src/internal.js index 6b3ff234ce1..7ba883a4225 100644 --- a/packages/swing-store/src/internal.js +++ b/packages/swing-store/src/internal.js @@ -7,6 +7,8 @@ import { Fail, q } from '@endo/errors'; * * @typedef {{ * dirPath: string | null, + * db: ReturnType, + * kvStore: import('./kvStore.js').KVStore, * transcriptStore: TranscriptStoreInternal, * snapStore: SnapStoreInternal, * bundleStore: BundleStoreInternal, diff --git a/packages/swing-store/src/swingStore.js b/packages/swing-store/src/swingStore.js index 519462aa33a..d579e3ad43c 100644 --- a/packages/swing-store/src/swingStore.js +++ b/packages/swing-store/src/swingStore.js @@ -21,6 +21,11 @@ import { doRepairMetadata } from './repairMetadata.js'; // https://github.com/WiseLibs/better-sqlite3/blob/HEAD/docs/api.md#new-databasepath-options const IN_MEMORY = ':memory:'; +/** + * @template T + * @typedef {(input: T) => T} Replacer + */ + /** * @typedef { import('./kvStore.js').KVStore } KVStore * @@ -136,6 +141,10 @@ const IN_MEMORY = ':memory:'; * @property {import('./snapStore.js').SnapshotCallback} [archiveSnapshot] Called after creation of a new heap snapshot * @property {import('./transcriptStore.js').TranscriptCallback} [archiveTranscript] Called after a formerly-current transcript span is finalized * @property {(pendingExports: Iterable<[key: string, value: string | null]>) => void} [exportCallback] + * @property {Replacer>} [wrapKvStore] + * @property {Replacer>} [wrapTranscriptStore] + * @property {Replacer>} [wrapSnapStore] + * @property {Replacer>} [wrapBundleStore] */ /** @@ -163,6 +172,10 @@ export function makeSwingStore(path, forceReset, options = {}) { archiveSnapshot, archiveTranscript, exportCallback, + wrapKvStore = x => x, + wrapTranscriptStore = x => x, + wrapSnapStore = x => x, + wrapBundleStore = x => x, } = options; if (serialized) { @@ -303,31 +316,22 @@ export function makeSwingStore(path, forceReset, options = {}) { } } - const kvStore = makeKVStore(db, ensureTxn, trace); + const kvStore = wrapKvStore(makeKVStore(db, ensureTxn, trace)); - const { dumpTranscripts, ...transcriptStoreInternal } = makeTranscriptStore( - db, - ensureTxn, - noteExport, - { + const { dumpTranscripts, ...transcriptStoreInternal } = wrapTranscriptStore( + makeTranscriptStore(db, ensureTxn, noteExport, { keepTranscripts, archiveTranscript, - }, + }), ); - const { dumpSnapshots, ...snapStoreInternal } = makeSnapStore( - db, - ensureTxn, - makeSnapStoreIO(), - noteExport, - { + const { dumpSnapshots, ...snapStoreInternal } = wrapSnapStore( + makeSnapStore(db, ensureTxn, makeSnapStoreIO(), noteExport, { keepSnapshots, archiveSnapshot, - }, + }), ); - const { dumpBundles, ...bundleStoreInternal } = makeBundleStore( - db, - ensureTxn, - noteExport, + const { dumpBundles, ...bundleStoreInternal } = wrapBundleStore( + makeBundleStore(db, ensureTxn, noteExport), ); const sqlCommit = db.prepare('COMMIT'); @@ -514,6 +518,8 @@ export function makeSwingStore(path, forceReset, options = {}) { const internal = harden({ dirPath: asFile ? null : path, asFile, + db, + kvStore, snapStore: snapStoreInternal, transcriptStore: transcriptStoreInternal, bundleStore: bundleStoreInternal, From 009b11323b05cb3f1858db57a7fedd3909837770 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Mon, 6 Jan 2025 21:55:27 -0500 Subject: [PATCH 12/18] chore: Add lint coverage for tools/ --- eslint.config.mjs | 3 ++- packages/cosmic-swingset/tsconfig.json | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/eslint.config.mjs b/eslint.config.mjs index ddcd0366086..cb5b31731ac 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -117,7 +117,7 @@ export default [ }, tsconfigRootDir: __dirname, - extraFileExtensions: ['.cjs'], + extraFileExtensions: ['.cjs', '.mjs'], }, }, @@ -182,6 +182,7 @@ export default [ files: [ 'packages/*/src/**/*.js', 'packages/*/tools/**/*.js', + 'packages/*/tools/**/*.mjs', 'packages/*/*.js', 'packages/wallet/api/src/**/*.js', ], diff --git a/packages/cosmic-swingset/tsconfig.json b/packages/cosmic-swingset/tsconfig.json index 1925e9caba4..dcc4b262ea4 100644 --- a/packages/cosmic-swingset/tsconfig.json +++ b/packages/cosmic-swingset/tsconfig.json @@ -9,5 +9,7 @@ "src/**/*.js", "test/**/*.js", "*.cjs", + "tools/**/*.js", + "tools/**/*.mjs", ], } From e39c8022c2a3b3edb0b4912bda733ad9be8d4903 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Mon, 10 Feb 2025 21:18:48 -0500 Subject: [PATCH 13/18] feat(internal): Generalize single-level `pick` utility to recursive `attenuate` --- packages/cosmic-swingset/src/launch-chain.js | 14 ++-- packages/internal/src/ses-utils.js | 75 +++++++++++++----- packages/internal/src/types.ts | 21 +++++ .../test/snapshots/exports.test.js.md | 2 +- .../test/snapshots/exports.test.js.snap | Bin 703 -> 707 bytes packages/internal/test/types.test-d.ts | 28 ++++++- packages/swing-store/src/swingStore.js | 59 ++++++-------- 7 files changed, 136 insertions(+), 63 deletions(-) diff --git a/packages/cosmic-swingset/src/launch-chain.js b/packages/cosmic-swingset/src/launch-chain.js index 4d182d29490..c8a5c4fe33c 100644 --- a/packages/cosmic-swingset/src/launch-chain.js +++ b/packages/cosmic-swingset/src/launch-chain.js @@ -26,7 +26,7 @@ import { } from '@agoric/swingset-vat'; import { waitUntilQuiescent } from '@agoric/internal/src/lib-nodejs/waitUntilQuiescent.js'; import { openSwingStore } from '@agoric/swing-store'; -import { pick, BridgeId as BRIDGE_ID } from '@agoric/internal'; +import { attenuate, BridgeId as BRIDGE_ID } from '@agoric/internal'; import { makeWithQueue } from '@agoric/internal/src/queue.js'; import * as ActionType from '@agoric/internal/src/action-types.js'; @@ -1278,11 +1278,11 @@ export async function launchAndShareInternals({ */ export async function launch(options) { const launchResult = await launchAndShareInternals(options); - return pick(launchResult, { - blockingSend: true, - shutdown: true, - writeSlogObject: true, - savedHeight: true, - savedChainSends: true, + return attenuate(launchResult, { + blockingSend: 'pick', + shutdown: 'pick', + writeSlogObject: 'pick', + savedHeight: 'pick', + savedChainSends: 'pick', }); } diff --git a/packages/internal/src/ses-utils.js b/packages/internal/src/ses-utils.js index b6f27e6cdc3..26308c37218 100644 --- a/packages/internal/src/ses-utils.js +++ b/packages/internal/src/ses-utils.js @@ -16,12 +16,13 @@ import { makeQueue } from '@endo/stream'; // @ts-ignore TS7016 The 'jessie.js' library may need to update its package.json or typings import { asyncGenerate } from 'jessie.js'; +/** @import {ERef} from '@endo/far'; */ +/** @import {Permit, Attenuated} from './types.js'; */ + export { objectMap, objectMetaMap, fromUniqueEntries }; const { fromEntries, keys, values } = Object; -/** @import {ERef} from '@endo/far' */ - /** * @template T * @typedef {{ [KeyType in keyof T]: T[KeyType] } & {}} Simplify flatten the @@ -171,25 +172,59 @@ export const assertAllDefined = obj => { }; /** - * @template {Record} T - * @template {Partial<{ [K in keyof T]: true }>} U - * @param {T} target - * @param {U} [permits] - * @returns {keyof U extends keyof T ? Pick : never} + * Attenuate `specimen` to only properties allowed by `permit`. + * + * @template T + * @template {Permit} P + * @param {T} specimen + * @param {P} permit + * @param {>(attenuation: U, permit: SubP) => U} [transform] + * @returns {Attenuated} */ -export const pick = ( - target, - permits = /** @type {U} */ (objectMap(target, () => true)), -) => { - const attenuation = objectMap(permits, (permit, key) => { - permit === true || Fail`internal: ${q(key)} permit must be true`; - // eslint-disable-next-line no-restricted-syntax - key in target || Fail`internal: target is missing ${q(key)}`; - // eslint-disable-next-line no-restricted-syntax - return target[key]; - }); - // @ts-expect-error cast - return attenuation; +export const attenuate = (specimen, permit, transform = x => x) => { + // Entry-point arguments get special checks and error messages. + if (permit === true || typeof permit === 'string') { + return /** @type {Attenuated} */ (specimen); + } else if (permit === null || typeof permit !== 'object') { + throw Fail`invalid permit: ${q(permit)}`; + } else if (specimen === null || typeof specimen !== 'object') { + throw Fail`specimen must be an object for permit ${q(permit)}`; + } + + /** @type {string[]} */ + const path = []; + /** + * @template SubT + * @template {Permit} SubP + * @type {(specimen: SubT, permit: SubP) => Attenuated} + */ + const extract = (subSpecimen, subPermit) => { + if (subPermit === true || typeof subPermit === 'string') { + return /** @type {Attenuated} */ (subSpecimen); + } else if (subPermit === null || typeof subPermit !== 'object') { + throw Fail`invalid permit at path ${q(path)}: ${q(subPermit)}`; + } else if (subSpecimen === null || typeof subSpecimen !== 'object') { + throw Fail`specimen at path ${q(path)} must be an object for permit ${q(subPermit)}`; + } + const attenuated = Object.fromEntries( + Object.entries(subPermit).map(([subKey, deepPermit]) => { + path.push(subKey); + // eslint-disable-next-line no-restricted-syntax + subKey in subSpecimen || Fail`specimen is missing path ${q(path)}`; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore TS7053 does not manifest in all import sites + // eslint-disable-next-line no-restricted-syntax + const deepSpecimen = subSpecimen[subKey]; + const entry = [subKey, extract(deepSpecimen, deepPermit)]; + path.pop(); + return entry; + }), + ); + return transform(attenuated, subPermit); + }; + + // @ts-expect-error + return extract(specimen, permit); }; /** @type {IteratorResult} */ diff --git a/packages/internal/src/types.ts b/packages/internal/src/types.ts index 9d5420ff3df..ce036dd5992 100644 --- a/packages/internal/src/types.ts +++ b/packages/internal/src/types.ts @@ -15,6 +15,27 @@ export type TotalMap = Omit, 'get'> & { export type TotalMapFrom> = M extends Map ? TotalMap : never; +/** + * A permit is either `true` or a string (both meaning no attenuation, with a + * string serving as a grouping label for convenience and/or diagram + * generation), or an object whose keys identify child properties and whose + * corresponding values are theirselves (recursive) Permits. + */ +export type Permit = + | true + | string + | Partial<{ [K in keyof T]: K extends string ? Permit : never }>; + +export type Attenuated> = P extends object + ? { + [K in keyof P]: K extends keyof T + ? P[K] extends Permit + ? Attenuated + : never + : never; + } + : T; + export declare class Callback any> { private iface: I; diff --git a/packages/internal/test/snapshots/exports.test.js.md b/packages/internal/test/snapshots/exports.test.js.md index 9f95df26959..6c74f524c5f 100644 --- a/packages/internal/test/snapshots/exports.test.js.md +++ b/packages/internal/test/snapshots/exports.test.js.md @@ -21,6 +21,7 @@ Generated by [AVA](https://avajs.dev). 'aggregateTryFinally', 'allValues', 'assertAllDefined', + 'attenuate', 'bindAllMethods', 'cast', 'deepCopyJsonable', @@ -36,7 +37,6 @@ Generated by [AVA](https://avajs.dev). 'mustMatch', 'objectMap', 'objectMetaMap', - 'pick', 'pureDataMarshaller', 'synchronizedTee', 'untilTrue', diff --git a/packages/internal/test/snapshots/exports.test.js.snap b/packages/internal/test/snapshots/exports.test.js.snap index 6acc3441b55ad825b0fdfab64c9313ad002adefe..8efed672d75efe2b9ad623a54d616c5378bd0484 100644 GIT binary patch literal 707 zcmV;!0zCaeRzV00000000ARlFd$2K@`VlXn}%2DIX%D;>W`H1TI|h87sD!v|x;@>Ak1- zj!b8+GjqX`xM1bRl`ml60gNklu3fotrHQZL#<(_~0&@z+Ccm`l?|)~`Irkj4TgpT` z?%)W#^a?=g7+y)IMdniD-N9iT3oo7(e-zj->wS2M-}#8&$H+J27qY+^yN^6aUL)_3 zPsn%V4>CW%*aKu6*+t$WACND|Z)CE-*gfPiqL6pUG4d7piIj?r-90%87HtTldATORAg|L;5ztpxZS61`8oLS~t3$DI99 zl2}NDho!b*R1Q@AC8|w02i9YQtI(DjXg*{8hrI`#;|)QL2Q|^c z4a7%l;8PRR;S<3ZBjKo>GF=%1WGg1?K5|BjmZEbfLt?y_sEKT|^#UR~pc3?KPpj^F zuG$g@6JLx?(b(dXj?Ry5I=JDsv9Jp^h#A_sHooyKEnnuaroEN@k@SzM^)dzp007}2Q=9+* literal 703 zcmV;w0zmyiRzVkZ*bXsICHQpT@#$y_Wr6DVZZ0eCUUFl$1sZG0Xt+6gRgTa|>cwZAgt~FGyoS8pVINh01~2U!rcpIj|ltxCU*hf#y>R#zYI(4D%ZJ)Wmecc*u*9 zaMXE%&W!=GRg?7|IHN^NQS)*z#`{K1WSgyL5YYjXA)f8Ws<)A=w#3`V7qtq_CO)ox zel}Af8*Up5&%uUZnii{#uXRhy-5l1nxAJT*Gj!_&{K<{`M^io<@)^t<_^3|)V_l~G z>52~2K)BpO10ti(<~K(-=sQg-qO+n`=*4w&=WD`8w3_o2nd5=r07G<@CVLuz7w6&y zdSRK2I^o}IR5innYoh;%b!mU$sLu7^lwD6psWn>eL)-+K_zfzB>nl~$<`gYbw3kW+ l=cAjn!+X-FxPD+Q#H$K~&Z+gSAQkpy_7|Sv3a9}E006#!PyPS^ diff --git a/packages/internal/test/types.test-d.ts b/packages/internal/test/types.test-d.ts index e9ae4ef4d2e..f31a4472966 100644 --- a/packages/internal/test/types.test-d.ts +++ b/packages/internal/test/types.test-d.ts @@ -1,8 +1,34 @@ import { expectNotType, expectType } from 'tsd'; import { E, type ERef } from '@endo/far'; -import type { Remote } from '../src/types.js'; +import { attenuate } from '../src/ses-utils.js'; +import type { Permit, Remote } from '../src/types.js'; import type { StorageNode } from '../src/lib-chainStorage.js'; +{ + const obj = { + m1: () => {}, + m2: () => {}, + data: { + log: ['string'], + counter: 0, + }, + internalData: { + realCount: 0, + }, + }; + expectType<{ m1: () => void }>(attenuate(obj, { m1: true })); + expectType<{ m2: () => void }>(attenuate(obj, { m2: 'pick' })); + expectType<{ data: { log: string[]; counter: number } }>( + attenuate(obj, { data: 'pick' }), + ); + expectType<{ m1: () => void; data: { log: string[] } }>( + attenuate(obj, { m1: 'pick', data: { log: true } }), + ); + expectNotType<{ m1: () => void; m2: () => void; data: { log: string[] } }>( + attenuate(obj, { m1: 'pick', data: { log: true } }), + ); +} + const eventualStorageNode: ERef = null as any; const remoteStorageNode: Remote = null as any; diff --git a/packages/swing-store/src/swingStore.js b/packages/swing-store/src/swingStore.js index d579e3ad43c..f321b53802a 100644 --- a/packages/swing-store/src/swingStore.js +++ b/packages/swing-store/src/swingStore.js @@ -7,7 +7,7 @@ import sqlite3 from 'better-sqlite3'; import { Fail, q } from '@endo/errors'; -import { pick } from '@agoric/internal'; +import { attenuate } from '@agoric/internal'; import { dbFileInDirectory } from './util.js'; import { makeKVStore, getKeyType } from './kvStore.js'; @@ -565,41 +565,32 @@ export function makeSwingStore(path, forceReset, options = {}) { return db; } - const transcriptStore = pick( - transcriptStoreInternal, - /** @type {const} */ ({ - initTranscript: true, - rolloverSpan: true, - rolloverIncarnation: true, - getCurrentSpanBounds: true, - addItem: true, - readSpan: true, - stopUsingTranscript: true, - deleteVatTranscripts: true, - }), - ); + const transcriptStore = attenuate(transcriptStoreInternal, { + initTranscript: 'pick', + rolloverSpan: 'pick', + rolloverIncarnation: 'pick', + getCurrentSpanBounds: 'pick', + addItem: 'pick', + readSpan: 'pick', + stopUsingTranscript: 'pick', + deleteVatTranscripts: 'pick', + }); - const snapStore = pick( - snapStoreInternal, - /** @type {const} */ ({ - loadSnapshot: true, - saveSnapshot: true, - deleteAllUnusedSnapshots: true, - deleteVatSnapshots: true, - stopUsingLastSnapshot: true, - getSnapshotInfo: true, - }), - ); + const snapStore = attenuate(snapStoreInternal, { + loadSnapshot: 'pick', + saveSnapshot: 'pick', + deleteAllUnusedSnapshots: 'pick', + deleteVatSnapshots: 'pick', + stopUsingLastSnapshot: 'pick', + getSnapshotInfo: 'pick', + }); - const bundleStore = pick( - bundleStoreInternal, - /** @type {const} */ ({ - addBundle: true, - hasBundle: true, - getBundle: true, - deleteBundle: true, - }), - ); + const bundleStore = attenuate(bundleStoreInternal, { + addBundle: 'pick', + hasBundle: 'pick', + getBundle: 'pick', + deleteBundle: 'pick', + }); const kernelStorage = { kvStore: kernelKVStore, From d255a9860cb7de5f6a884716de101035ae4ce386 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Tue, 11 Feb 2025 12:16:55 -0500 Subject: [PATCH 14/18] fixup! feat(internal): Generalize single-level `pick` utility to recursive `attenuate` --- packages/internal/src/ses-utils.js | 54 +++++++++++++++--------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/packages/internal/src/ses-utils.js b/packages/internal/src/ses-utils.js index 26308c37218..e2b779e3783 100644 --- a/packages/internal/src/ses-utils.js +++ b/packages/internal/src/ses-utils.js @@ -17,6 +17,7 @@ import { makeQueue } from '@endo/stream'; import { asyncGenerate } from 'jessie.js'; /** @import {ERef} from '@endo/far'; */ +/** @import {Primitive} from '@endo/pass-style'; */ /** @import {Permit, Attenuated} from './types.js'; */ export { objectMap, objectMetaMap, fromUniqueEntries }; @@ -179,51 +180,50 @@ export const assertAllDefined = obj => { * @param {T} specimen * @param {P} permit * @param {>(attenuation: U, permit: SubP) => U} [transform] + * used to replace the results of recursive picks (but not blanket permits) * @returns {Attenuated} */ export const attenuate = (specimen, permit, transform = x => x) => { - // Entry-point arguments get special checks and error messages. + // Fast-path for no attenuation. if (permit === true || typeof permit === 'string') { return /** @type {Attenuated} */ (specimen); - } else if (permit === null || typeof permit !== 'object') { - throw Fail`invalid permit: ${q(permit)}`; - } else if (specimen === null || typeof specimen !== 'object') { - throw Fail`specimen must be an object for permit ${q(permit)}`; } /** @type {string[]} */ const path = []; /** * @template SubT - * @template {Permit} SubP + * @template {Exclude, Primitive>} SubP * @type {(specimen: SubT, permit: SubP) => Attenuated} */ const extract = (subSpecimen, subPermit) => { - if (subPermit === true || typeof subPermit === 'string') { - return /** @type {Attenuated} */ (subSpecimen); - } else if (subPermit === null || typeof subPermit !== 'object') { - throw Fail`invalid permit at path ${q(path)}: ${q(subPermit)}`; + if (subPermit === null || typeof subPermit !== 'object') { + throw path.length === 0 + ? Fail`invalid permit: ${q(permit)}` + : Fail`invalid permit at path ${q(path)}: ${q(subPermit)}`; } else if (subSpecimen === null || typeof subSpecimen !== 'object') { - throw Fail`specimen at path ${q(path)} must be an object for permit ${q(subPermit)}`; + throw path.length === 0 + ? Fail`specimen must be an object for permit ${q(permit)}` + : Fail`specimen at path ${q(path)} must be an object for permit ${q(subPermit)}`; } - const attenuated = Object.fromEntries( - Object.entries(subPermit).map(([subKey, deepPermit]) => { - path.push(subKey); - // eslint-disable-next-line no-restricted-syntax - subKey in subSpecimen || Fail`specimen is missing path ${q(path)}`; - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore TS7053 does not manifest in all import sites - // eslint-disable-next-line no-restricted-syntax - const deepSpecimen = subSpecimen[subKey]; - const entry = [subKey, extract(deepSpecimen, deepPermit)]; - path.pop(); - return entry; - }), - ); - return transform(attenuated, subPermit); + const picks = Object.entries(subPermit).map(([subKey, deepPermit]) => { + if (!Object.hasOwn(subSpecimen, subKey)) { + throw Fail`specimen is missing path ${q(path.concat(subKey))}`; + } + const deepSpecimen = Reflect.get(subSpecimen, subKey); + if (deepPermit === true || typeof deepPermit === 'string') { + return [subKey, deepSpecimen]; + } + path.push(subKey); + const extracted = extract(/** @type {any} */ (deepSpecimen), deepPermit); + const entry = [subKey, extracted]; + path.pop(); + return entry; + }); + return transform(Object.fromEntries(picks), subPermit); }; - // @ts-expect-error + // @ts-expect-error cast return extract(specimen, permit); }; From 090a6c7023af84f15fcbc7a39a4847b61c26c934 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Fri, 14 Feb 2025 15:24:46 -0500 Subject: [PATCH 15/18] fixup! feat(internal): Generalize single-level `pick` utility to recursive `attenuate` --- packages/cosmic-swingset/src/launch-chain.js | 17 +++++---- packages/swing-store/src/swingStore.js | 36 ++++++++++---------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/packages/cosmic-swingset/src/launch-chain.js b/packages/cosmic-swingset/src/launch-chain.js index c8a5c4fe33c..579a4e855ad 100644 --- a/packages/cosmic-swingset/src/launch-chain.js +++ b/packages/cosmic-swingset/src/launch-chain.js @@ -1278,11 +1278,14 @@ export async function launchAndShareInternals({ */ export async function launch(options) { const launchResult = await launchAndShareInternals(options); - return attenuate(launchResult, { - blockingSend: 'pick', - shutdown: 'pick', - writeSlogObject: 'pick', - savedHeight: 'pick', - savedChainSends: 'pick', - }); + return attenuate( + launchResult, + /** @type {const} */ ({ + blockingSend: true, + shutdown: true, + writeSlogObject: true, + savedHeight: true, + savedChainSends: true, + }), + ); } diff --git a/packages/swing-store/src/swingStore.js b/packages/swing-store/src/swingStore.js index f321b53802a..a613522ad1a 100644 --- a/packages/swing-store/src/swingStore.js +++ b/packages/swing-store/src/swingStore.js @@ -566,30 +566,30 @@ export function makeSwingStore(path, forceReset, options = {}) { } const transcriptStore = attenuate(transcriptStoreInternal, { - initTranscript: 'pick', - rolloverSpan: 'pick', - rolloverIncarnation: 'pick', - getCurrentSpanBounds: 'pick', - addItem: 'pick', - readSpan: 'pick', - stopUsingTranscript: 'pick', - deleteVatTranscripts: 'pick', + initTranscript: true, + rolloverSpan: true, + rolloverIncarnation: true, + getCurrentSpanBounds: true, + addItem: true, + readSpan: true, + stopUsingTranscript: true, + deleteVatTranscripts: true, }); const snapStore = attenuate(snapStoreInternal, { - loadSnapshot: 'pick', - saveSnapshot: 'pick', - deleteAllUnusedSnapshots: 'pick', - deleteVatSnapshots: 'pick', - stopUsingLastSnapshot: 'pick', - getSnapshotInfo: 'pick', + loadSnapshot: true, + saveSnapshot: true, + deleteAllUnusedSnapshots: true, + deleteVatSnapshots: true, + stopUsingLastSnapshot: true, + getSnapshotInfo: true, }); const bundleStore = attenuate(bundleStoreInternal, { - addBundle: 'pick', - hasBundle: 'pick', - getBundle: 'pick', - deleteBundle: 'pick', + addBundle: true, + hasBundle: true, + getBundle: true, + deleteBundle: true, }); const kernelStorage = { From c800d1cd15a14bc3887c97088dfacee880273db7 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Fri, 14 Feb 2025 15:33:13 -0500 Subject: [PATCH 16/18] fixup! feat(cosmic-swingset): Expose a controller and associated helpers from test-kit.js --- packages/cosmic-swingset/src/chain-main.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/cosmic-swingset/src/chain-main.js b/packages/cosmic-swingset/src/chain-main.js index 44684b56587..e22db6b6cff 100644 --- a/packages/cosmic-swingset/src/chain-main.js +++ b/packages/cosmic-swingset/src/chain-main.js @@ -229,7 +229,10 @@ export const makeQueueStorage = (call, queuePath) => { * swingStore?: import('@agoric/swing-store').SwingStore, * vatconfig?: Parameters[0]['vatconfig'], * withInternals?: boolean, - * }} [options.testingOverrides] + * }} [options.testingOverrides] Exposed only for testing purposes. + * `debugName`/`slogSender`/`swingStore`/`vatConfig` are pure overrides, while + * `withInternals` expands the return value to expose internal objects + * `controller`/`bridgeInbound`/`timer`. */ export const makeLaunchChain = ( agcc, From 2f4ef9bb6e940b773e0b4916229797962222e395 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Fri, 14 Feb 2025 15:36:25 -0500 Subject: [PATCH 17/18] fixup! chore: Add lint coverage for tools/ --- packages/cosmic-swingset/tsconfig.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/cosmic-swingset/tsconfig.json b/packages/cosmic-swingset/tsconfig.json index dcc4b262ea4..8069341b857 100644 --- a/packages/cosmic-swingset/tsconfig.json +++ b/packages/cosmic-swingset/tsconfig.json @@ -9,7 +9,6 @@ "src/**/*.js", "test/**/*.js", "*.cjs", - "tools/**/*.js", - "tools/**/*.mjs", + "tools", ], } From 12e96c8de3052cadd10d58935bdda6344c1711b5 Mon Sep 17 00:00:00 2001 From: Richard Gibson Date: Fri, 14 Feb 2025 18:56:14 -0500 Subject: [PATCH 18/18] fixup! feat(cosmic-swingset): Expose a controller and associated helpers from test-kit.js --- packages/boot/tools/supports.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/boot/tools/supports.ts b/packages/boot/tools/supports.ts index 7df7baaa72e..d143a29f5c8 100644 --- a/packages/boot/tools/supports.ts +++ b/packages/boot/tools/supports.ts @@ -587,6 +587,9 @@ export const makeSwingsetTestKit = async ( console.timeLog('makeBaseSwingsetTestKit', 'buildSwingset'); + // XXX This initial run() might not be necessary. Tests pass without it as of + // 2025-02, but we suspect that `makeSwingsetTestKit` just isn't being + // exercised in the right way. await controller.run(); const runUtils = makeBootstrapRunUtils(controller, harness);