From d6d5145e8a07ae3b3dd7717a8a4e9b17c246d451 Mon Sep 17 00:00:00 2001 From: Kevin Barabash Date: Sun, 24 Mar 2024 12:33:23 -0400 Subject: [PATCH 1/2] Add 'fileDescriptors' option to allow file I/O to be used in the browser --- package-lock.json | 8 +++---- package.json | 2 +- src/foreground-plugin.ts | 2 +- src/interfaces.ts | 15 ++++++++++++++ src/mod.ts | 1 + src/polyfills/browser-wasi.ts | 39 +++++++++-------------------------- src/polyfills/deno-wasi.ts | 8 ++++--- src/polyfills/node-wasi.ts | 3 +++ tsconfig.json | 2 ++ 9 files changed, 42 insertions(+), 38 deletions(-) diff --git a/package-lock.json b/package-lock.json index e7e7b1e..7354842 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.0.0-replaced-by-ci", "license": "BSD-3-Clause", "devDependencies": { - "@bjorn3/browser_wasi_shim": "^0.2.17", + "@bjorn3/browser_wasi_shim": "^0.2.20", "@playwright/test": "^1.39.0", "@types/node": "^20.8.7", "@typescript-eslint/eslint-plugin": "^6.8.0", @@ -37,9 +37,9 @@ } }, "node_modules/@bjorn3/browser_wasi_shim": { - "version": "0.2.17", - "resolved": "https://registry.npmjs.org/@bjorn3/browser_wasi_shim/-/browser_wasi_shim-0.2.17.tgz", - "integrity": "sha512-B2qcaGROo4e2s4nXb3VPATrczVrntM4BUXtAU1gEzUOfqKTcVuePq4NfhH5hmLBSvZ45YcT4gflDRUFYqLhkxA==", + "version": "0.2.20", + "resolved": "https://registry.npmjs.org/@bjorn3/browser_wasi_shim/-/browser_wasi_shim-0.2.20.tgz", + "integrity": "sha512-URvkOAWWRQ8gazkBRgQ/e0H6R0VlOALJ9/vI2VBttG23MHzUZzy5KFyKAFVI1qL/hbqLwnZw/sIXFkeS6OH5EQ==", "dev": true }, "node_modules/@esbuild/android-arm": { diff --git a/package.json b/package.json index fca8f88..e852a13 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "author": "The Extism Authors ", "license": "BSD-3-Clause", "devDependencies": { - "@bjorn3/browser_wasi_shim": "^0.2.17", + "@bjorn3/browser_wasi_shim": "^0.2.20", "@playwright/test": "^1.39.0", "@types/node": "^20.8.7", "@typescript-eslint/eslint-plugin": "^6.8.0", diff --git a/src/foreground-plugin.ts b/src/foreground-plugin.ts index b85229c..bb63a3b 100644 --- a/src/foreground-plugin.ts +++ b/src/foreground-plugin.ts @@ -156,7 +156,7 @@ async function instantiateModule( } if (wasi === null) { - wasi = await loadWasi(opts.allowedPaths, opts.enableWasiOutput); + wasi = await loadWasi(opts.allowedPaths, opts.enableWasiOutput, opts.fileDescriptors); wasiList.push(wasi); imports.wasi_snapshot_preview1 = await wasi.importObject(); } diff --git a/src/interfaces.ts b/src/interfaces.ts index 7bcab64..2141314 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -123,6 +123,19 @@ export interface Plugin { reset(): Promise; } +/** + * Options for initializing WASI. + */ +export interface WASIOptions { + /** + * A list of file descriptors; only available in browser environments. + * + * See [`@bjorn3/browser_wasi_shim`](https://github.com/bjorn3/browser_wasi_shim) for more + * details on use. + */ + fileDescriptors: (import('@bjorn3/browser_wasi_shim').Fd)[]; +} + /** * Options for initializing an Extism plugin. */ @@ -167,6 +180,7 @@ export interface ExtismPluginOptions { functions?: { [key: string]: { [key: string]: (callContext: CallContext, ...args: any[]) => any } } | undefined; allowedPaths?: { [key: string]: string } | undefined; allowedHosts?: string[] | undefined; + wasiOptions?: WasiOptions; /** * Whether WASI stdout should be forwarded to the host. @@ -189,6 +203,7 @@ export interface InternalConfig { wasiEnabled: boolean; config: PluginConfig; sharedArrayBufferSize: number; + wasiOptions: WasiOptions | null; } export interface InternalWasi { diff --git a/src/mod.ts b/src/mod.ts index 075cb38..edc108d 100644 --- a/src/mod.ts +++ b/src/mod.ts @@ -99,6 +99,7 @@ export async function createPlugin( config: opts.config, enableWasiOutput: opts.enableWasiOutput, sharedArrayBufferSize: Number(opts.sharedArrayBufferSize) || 1 << 16, + fileDescriptors: opts.fileDescriptors ?? [], }; return (opts.runInWorker ? _createBackgroundPlugin : _createForegroundPlugin)(ic, names, moduleData); diff --git a/src/polyfills/browser-wasi.ts b/src/polyfills/browser-wasi.ts index e2d4db3..bc8c1a5 100644 --- a/src/polyfills/browser-wasi.ts +++ b/src/polyfills/browser-wasi.ts @@ -1,48 +1,29 @@ -import { WASI, Fd, File, OpenFile, wasi } from '@bjorn3/browser_wasi_shim'; -import { type InternalWasi } from '../mod.ts'; - -class Output extends Fd { - #mode: string; - - constructor(mode: string) { - super(); - this.#mode = mode; - } - - fd_write(view8: Uint8Array, iovs: [wasi.Iovec]): { ret: number; nwritten: number } { - let nwritten = 0; - const decoder = new TextDecoder(); - const str = iovs.reduce((acc, iovec, idx, all) => { - nwritten += iovec.buf_len; - const buffer = view8.slice(iovec.buf, iovec.buf + iovec.buf_len); - return acc + decoder.decode(buffer, { stream: idx !== all.length - 1 }); - }, ''); - - (console[this.#mode] as any)(str); - - return { ret: 0, nwritten }; - } -} +import { WASI, Fd, File, OpenFile, ConsoleStdout } from '@bjorn3/browser_wasi_shim'; +import { type InternalWasi } from '../interfaces.ts'; export async function loadWasi( _allowedPaths: { [from: string]: string }, enableWasiOutput: boolean, + fileDescriptors: Fd[], ): Promise { + console.log('fileDescriptors = ', fileDescriptors); const args: Array = []; const envVars: Array = []; const fds: Fd[] = enableWasiOutput ? [ - new Output('log'), // fd 0 is dup'd to stdout - new Output('log'), - new Output('error'), + ConsoleStdout.lineBuffered((msg) => console.log(msg)), // fd 0 is dup'd to stdout + ConsoleStdout.lineBuffered((msg) => console.log(msg)), + ConsoleStdout.lineBuffered((msg) => console.warn(msg)), + ...fileDescriptors, ] : [ new OpenFile(new File([])), // stdin new OpenFile(new File([])), // stdout new OpenFile(new File([])), // stderr + ...fileDescriptors, ]; - const context = new WASI(args, envVars, fds); + const context = new WASI(args, envVars, fds, { debug: false }); return { async importObject() { diff --git a/src/polyfills/deno-wasi.ts b/src/polyfills/deno-wasi.ts index 3aa3067..6e49256 100644 --- a/src/polyfills/deno-wasi.ts +++ b/src/polyfills/deno-wasi.ts @@ -20,7 +20,7 @@ async function createDevNullFDs() { return { async close() { needsClose = false; - await Promise.all([stdin.close(), stdout.close()]).catch(() => {}); + await Promise.all([stdin.close(), stdout.close()]).catch(() => { }); }, fds: [stdin.fd, stdout.fd, stdout.fd], }; @@ -29,11 +29,13 @@ async function createDevNullFDs() { export async function loadWasi( allowedPaths: { [from: string]: string }, enableWasiOutput: boolean, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + fileDescriptors: Fd[], ): Promise { const { close, fds: [stdin, stdout, stderr], - } = enableWasiOutput ? { async close() {}, fds: [0, 1, 2] } : await createDevNullFDs(); + } = enableWasiOutput ? { async close() { }, fds: [0, 1, 2] } : await createDevNullFDs(); const context = new Context({ preopens: allowedPaths, exitOnReturn: false, @@ -76,7 +78,7 @@ export async function loadWasi( context.start({ exports: { memory, - _start: () => {}, + _start: () => { }, }, }); } diff --git a/src/polyfills/node-wasi.ts b/src/polyfills/node-wasi.ts index 512515a..ac54d3c 100644 --- a/src/polyfills/node-wasi.ts +++ b/src/polyfills/node-wasi.ts @@ -1,3 +1,4 @@ +import { Fd } from '@bjorn3/browser_wasi_shim'; import { WASI } from 'wasi'; import { type InternalWasi } from '../interfaces.ts'; import { devNull } from 'node:os'; @@ -39,6 +40,8 @@ async function createDevNullFDs() { export async function loadWasi( allowedPaths: { [from: string]: string }, enableWasiOutput: boolean, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + fileDescriptors: Fd[], ): Promise { const { close, diff --git a/tsconfig.json b/tsconfig.json index 0ac1e5f..746de13 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,9 +2,11 @@ "compilerOptions": { "target": "es2022", "module": "esnext", + "moduleResolution": "Node", "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "allowImportingTsExtensions": true, + "emitDeclarationOnly": true, "declaration": true, "strict": true, "skipLibCheck": true, From 31777df732f3311bd90f9ae7073841b67f5edb53 Mon Sep 17 00:00:00 2001 From: Chris Dickinson Date: Mon, 6 May 2024 14:51:34 -0700 Subject: [PATCH 2/2] feat: expose wasiOptions as part of pluginconfig Expose a new interface, `wasiOptions`, to accommodate platform-specific configuration for WASI. For now, this is limited to supporting fd preopens in browser environments (since none of Deno, Node, nor Bun support pre-allocating fds in this fashion.) --- src/foreground-plugin.ts | 16 ++++++++-------- src/interfaces.ts | 6 +++--- src/mod.ts | 2 +- src/polyfills/browser-wasi.ts | 28 ++++++++++++++-------------- src/polyfills/deno-wasi.ts | 5 ++--- src/polyfills/node-wasi.ts | 12 +++++------- tsconfig.json | 2 -- 7 files changed, 33 insertions(+), 38 deletions(-) diff --git a/src/foreground-plugin.ts b/src/foreground-plugin.ts index bb63a3b..bbb9c96 100644 --- a/src/foreground-plugin.ts +++ b/src/foreground-plugin.ts @@ -156,7 +156,7 @@ async function instantiateModule( } if (wasi === null) { - wasi = await loadWasi(opts.allowedPaths, opts.enableWasiOutput, opts.fileDescriptors); + wasi = await loadWasi(opts.allowedPaths, opts.enableWasiOutput, opts.wasiOptions); wasiList.push(wasi); imports.wasi_snapshot_preview1 = await wasi.importObject(); } @@ -211,9 +211,9 @@ async function instantiateModule( const instance = providerExports.find((xs) => xs.name === '_start') ? await instantiateModule([...current, module], provider, imports, opts, wasiList, names, modules, new Map()) : !linked.has(provider) - ? (await instantiateModule([...current, module], provider, imports, opts, wasiList, names, modules, linked), - linked.get(provider)) - : linked.get(provider); + ? (await instantiateModule([...current, module], provider, imports, opts, wasiList, names, modules, linked), + linked.get(provider)) + : linked.get(provider); if (!instance) { // circular import, either make a trampoline or bail @@ -254,10 +254,10 @@ async function instantiateModule( const guestType = instance.exports.hs_init ? 'haskell' : instance.exports._initialize - ? 'reactor' - : instance.exports._start - ? 'command' - : 'none'; + ? 'reactor' + : instance.exports._start + ? 'command' + : 'none'; if (wasi) { await wasi?.initialize(instance); diff --git a/src/interfaces.ts b/src/interfaces.ts index 2141314..3942937 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -133,7 +133,7 @@ export interface WASIOptions { * See [`@bjorn3/browser_wasi_shim`](https://github.com/bjorn3/browser_wasi_shim) for more * details on use. */ - fileDescriptors: (import('@bjorn3/browser_wasi_shim').Fd)[]; + fileDescriptors: any[]; } /** @@ -180,7 +180,7 @@ export interface ExtismPluginOptions { functions?: { [key: string]: { [key: string]: (callContext: CallContext, ...args: any[]) => any } } | undefined; allowedPaths?: { [key: string]: string } | undefined; allowedHosts?: string[] | undefined; - wasiOptions?: WasiOptions; + wasiOptions?: WASIOptions; /** * Whether WASI stdout should be forwarded to the host. @@ -203,7 +203,7 @@ export interface InternalConfig { wasiEnabled: boolean; config: PluginConfig; sharedArrayBufferSize: number; - wasiOptions: WasiOptions | null; + wasiOptions: WASIOptions | null; } export interface InternalWasi { diff --git a/src/mod.ts b/src/mod.ts index edc108d..089cb85 100644 --- a/src/mod.ts +++ b/src/mod.ts @@ -99,7 +99,7 @@ export async function createPlugin( config: opts.config, enableWasiOutput: opts.enableWasiOutput, sharedArrayBufferSize: Number(opts.sharedArrayBufferSize) || 1 << 16, - fileDescriptors: opts.fileDescriptors ?? [], + wasiOptions: opts.wasiOptions ?? null, }; return (opts.runInWorker ? _createBackgroundPlugin : _createForegroundPlugin)(ic, names, moduleData); diff --git a/src/polyfills/browser-wasi.ts b/src/polyfills/browser-wasi.ts index bc8c1a5..1aacfdd 100644 --- a/src/polyfills/browser-wasi.ts +++ b/src/polyfills/browser-wasi.ts @@ -1,27 +1,27 @@ import { WASI, Fd, File, OpenFile, ConsoleStdout } from '@bjorn3/browser_wasi_shim'; -import { type InternalWasi } from '../interfaces.ts'; +import { type WASIOptions, type InternalWasi } from '../interfaces.ts'; export async function loadWasi( _allowedPaths: { [from: string]: string }, enableWasiOutput: boolean, - fileDescriptors: Fd[], + wasiOptions: WASIOptions | null ): Promise { - console.log('fileDescriptors = ', fileDescriptors); const args: Array = []; const envVars: Array = []; + const fileDescriptors = (wasiOptions?.fileDescriptors || []) as Fd[] const fds: Fd[] = enableWasiOutput ? [ - ConsoleStdout.lineBuffered((msg) => console.log(msg)), // fd 0 is dup'd to stdout - ConsoleStdout.lineBuffered((msg) => console.log(msg)), - ConsoleStdout.lineBuffered((msg) => console.warn(msg)), - ...fileDescriptors, - ] + ConsoleStdout.lineBuffered((msg) => console.log(msg)), // fd 0 is dup'd to stdout + ConsoleStdout.lineBuffered((msg) => console.log(msg)), + ConsoleStdout.lineBuffered((msg) => console.warn(msg)), + ...fileDescriptors, + ] : [ - new OpenFile(new File([])), // stdin - new OpenFile(new File([])), // stdout - new OpenFile(new File([])), // stderr - ...fileDescriptors, - ]; + new OpenFile(new File([])), // stdin + new OpenFile(new File([])), // stdout + new OpenFile(new File([])), // stderr + ...fileDescriptors, + ]; const context = new WASI(args, envVars, fds, { debug: false }); @@ -59,7 +59,7 @@ export async function loadWasi( context.start({ exports: { memory, - _start: () => {}, + _start: () => { }, }, }); } diff --git a/src/polyfills/deno-wasi.ts b/src/polyfills/deno-wasi.ts index 6e49256..a1e4e95 100644 --- a/src/polyfills/deno-wasi.ts +++ b/src/polyfills/deno-wasi.ts @@ -1,5 +1,5 @@ import Context from './deno-snapshot_preview1.ts'; -import { type InternalWasi } from '../interfaces.ts'; +import { type WASIOptions, type InternalWasi } from '../interfaces.ts'; import { devNull } from 'node:os'; import { open } from 'node:fs/promises'; import { closeSync } from 'node:fs'; @@ -29,8 +29,7 @@ async function createDevNullFDs() { export async function loadWasi( allowedPaths: { [from: string]: string }, enableWasiOutput: boolean, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - fileDescriptors: Fd[], + _wasiOptions: WASIOptions | null, ): Promise { const { close, diff --git a/src/polyfills/node-wasi.ts b/src/polyfills/node-wasi.ts index ac54d3c..2c39ff1 100644 --- a/src/polyfills/node-wasi.ts +++ b/src/polyfills/node-wasi.ts @@ -1,6 +1,5 @@ -import { Fd } from '@bjorn3/browser_wasi_shim'; import { WASI } from 'wasi'; -import { type InternalWasi } from '../interfaces.ts'; +import { type WASIOptions, type InternalWasi } from '../interfaces.ts'; import { devNull } from 'node:os'; import { open } from 'node:fs/promises'; import { closeSync } from 'node:fs'; @@ -27,7 +26,7 @@ async function createDevNullFDs() { fr.register(stdout, stdout.fd); close = async () => { needsClose = false; - await Promise.all([stdin.close(), stdout.close()]).catch(() => {}); + await Promise.all([stdin.close(), stdout.close()]).catch(() => { }); }; } @@ -40,13 +39,12 @@ async function createDevNullFDs() { export async function loadWasi( allowedPaths: { [from: string]: string }, enableWasiOutput: boolean, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - fileDescriptors: Fd[], + _wasiOptions: WASIOptions | null, ): Promise { const { close, fds: [stdin, stdout, stderr], - } = enableWasiOutput ? { async close() {}, fds: [0, 1, 2] } : await createDevNullFDs(); + } = enableWasiOutput ? { async close() { }, fds: [0, 1, 2] } : await createDevNullFDs(); const context = new WASI({ version: 'preview1', @@ -90,7 +88,7 @@ export async function loadWasi( context.start({ exports: { memory, - _start: () => {}, + _start: () => { }, }, }); } diff --git a/tsconfig.json b/tsconfig.json index 746de13..0ac1e5f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,11 +2,9 @@ "compilerOptions": { "target": "es2022", "module": "esnext", - "moduleResolution": "Node", "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "allowImportingTsExtensions": true, - "emitDeclarationOnly": true, "declaration": true, "strict": true, "skipLibCheck": true,