Support native system access from wasm

.github/workflows/ci.yml

@@ -2,12 +2,12 @@ name: CI
 
 on:
   push:
-    branches: ['*']
+    branches: ["*"]
   pull_request:
-    branches: ['*']
+    branches: ["*"]
 
 permissions:
-  contents: read  #  to fetch code (actions/checkout)
+  contents: read #  to fetch code (actions/checkout)
 
 jobs:
   esbuild-slow:
@@ -58,12 +58,10 @@ jobs:
         with:
           node-version: 16
 
-      # The version of Deno is pinned because version 1.25.1 was causing test
-      # flakes due to random segfaults.
-      - name: Setup Deno 1.24.0
+      - name: Setup Deno 1.31.3
         uses: denoland/setup-deno@main
         with:
-          deno-version: v1.24.0
+          deno-version: v1.31.3
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3

lib/deno/wasm.ts

@@ -60,24 +60,28 @@ let initializePromise: Promise<Service> | undefined
 let stopService: (() => void) | undefined
 
 let ensureServiceIsRunning = (): Promise<Service> => {
-  return initializePromise || startRunningService('esbuild.wasm', undefined, true)
+  return initializePromise || startRunningService('esbuild.wasm', undefined, true, undefined)
 }
 
 export const initialize: typeof types.initialize = async (options) => {
   options = common.validateInitializeOptions(options || {})
   let wasmURL = options.wasmURL
   let wasmModule = options.wasmModule
   let useWorker = options.worker !== false
+  let wasmSystemAccess = options.wasmSystemAccess
   if (initializePromise) throw new Error('Cannot call "initialize" more than once')
-  initializePromise = startRunningService(wasmURL || 'esbuild.wasm', wasmModule, useWorker)
+  initializePromise = startRunningService(wasmURL || 'esbuild.wasm', wasmModule, useWorker, wasmSystemAccess)
   initializePromise.catch(() => {
     // Let the caller try again if this fails
     initializePromise = void 0
   })
   await initializePromise
 }
 
-const startRunningService = async (wasmURL: string | URL, wasmModule: WebAssembly.Module | undefined, useWorker: boolean): Promise<Service> => {
+const startRunningService = async (
+  wasmURL: string | URL, wasmModule: WebAssembly.Module | undefined,
+  useWorker: boolean, wasmSystemAccess: types.WasmSystemAccess | undefined,
+): Promise<Service> => {
   let worker: {
     onmessage: ((event: any) => void) | null
     postMessage: (data: Uint8Array | ArrayBuffer | WebAssembly.Module) => void
@@ -86,10 +90,27 @@ const startRunningService = async (wasmURL: string | URL, wasmModule: WebAssembl
 
   if (useWorker) {
     // Run esbuild off the main thread
-    let blob = new Blob([`onmessage=${WEB_WORKER_SOURCE_CODE}(postMessage)`], { type: 'text/javascript' })
+    let script = `onmessage=${WEB_WORKER_SOURCE_CODE}(postMessage)`;
+    if (wasmSystemAccess?.fsSpecifier) {
+      script = `import fs from "${wasmSystemAccess.fsSpecifier}";globalThis.fs=fs;${script}`
+    }
+    if (wasmSystemAccess?.processSpecifier) {
+      script = `import process from "${wasmSystemAccess.processSpecifier}";globalThis.process=process;${script}`
+    }
+    let blob = new Blob([script], { type: 'text/javascript' })
     worker = new Worker(URL.createObjectURL(blob), { type: 'module' })
   } else {
     // Run esbuild on the main thread
+    if (wasmSystemAccess?.fsSpecifier) {
+      (globalThis as any).fs = await import(wasmSystemAccess.fsSpecifier)
+    } else if (wasmSystemAccess?.fsNamespace) {
+      (globalThis as any).fs = wasmSystemAccess.fsNamespace
+    }
+    if (wasmSystemAccess?.processSpecifier) {
+      (globalThis as any).process = await import(wasmSystemAccess.processSpecifier)
+    } else if (wasmSystemAccess?.processNamespace) {
+      (globalThis as any).process = wasmSystemAccess?.processNamespace
+    }
     let onmessage = WEB_WORKER_FUNCTION((data: Uint8Array) => worker.onmessage!({ data }))
     worker = {
       onmessage: null,
@@ -99,18 +120,18 @@ const startRunningService = async (wasmURL: string | URL, wasmModule: WebAssembl
     }
   }
 
-  let firstMessageResolve: (value: void) => void
+  let firstMessageResolve: (value: string) => void
   let firstMessageReject: (error: any) => void
 
-  const firstMessagePromise = new Promise((resolve, reject) => {
+  const firstMessagePromise = new Promise<string>((resolve, reject) => {
     firstMessageResolve = resolve
     firstMessageReject = reject
   })
 
-  worker.onmessage = ({ data: error }) => {
+  worker.onmessage = ({ data }) => {
     worker.onmessage = ({ data }) => readFromStdout(data)
-    if (error) firstMessageReject(error)
-    else firstMessageResolve()
+    if (data.error) firstMessageReject(data.error)
+    else firstMessageResolve(data.ok)
   }
 
   worker.postMessage(wasmModule || new URL(wasmURL, import.meta.url).toString())
@@ -120,12 +141,12 @@ const startRunningService = async (wasmURL: string | URL, wasmModule: WebAssembl
       worker.postMessage(bytes)
     },
     isSync: false,
-    hasFS: false,
+    hasFS: wasmSystemAccess !== undefined,
     esbuild: ourselves,
   })
 
   // This will throw if WebAssembly module instantiation fails
-  await firstMessagePromise
+  const defaultWD = await firstMessagePromise
 
   stopService = () => {
     worker.terminate()
@@ -141,7 +162,7 @@ const startRunningService = async (wasmURL: string | URL, wasmModule: WebAssembl
           refs: null,
           options,
           isTTY: false,
-          defaultWD: '/',
+          defaultWD,
           callback: (err, res) => err ? reject(err) : resolve(res as types.BuildResult),
         })),
 
@@ -152,7 +173,7 @@ const startRunningService = async (wasmURL: string | URL, wasmModule: WebAssembl
           refs: null,
           options,
           isTTY: false,
-          defaultWD: '/',
+          defaultWD,
           callback: (err, res) => err ? reject(err) : resolve(res as types.BuildContext),
         })),
 

lib/npm/browser.ts

@@ -61,17 +61,21 @@ export const initialize: typeof types.initialize = options => {
   let wasmURL = options.wasmURL
   let wasmModule = options.wasmModule
   let useWorker = options.worker !== false
+  let wasmSystemAccess = options.wasmSystemAccess
   if (!wasmURL && !wasmModule) throw new Error('Must provide either the "wasmURL" option or the "wasmModule" option')
   if (initializePromise) throw new Error('Cannot call "initialize" more than once')
-  initializePromise = startRunningService(wasmURL || '', wasmModule, useWorker)
+  initializePromise = startRunningService(wasmURL || '', wasmModule, useWorker, wasmSystemAccess)
   initializePromise.catch(() => {
     // Let the caller try again if this fails
     initializePromise = void 0
   })
   return initializePromise
 }
 
-const startRunningService = async (wasmURL: string | URL, wasmModule: WebAssembly.Module | undefined, useWorker: boolean): Promise<void> => {
+const startRunningService = async (
+  wasmURL: string | URL, wasmModule: WebAssembly.Module | undefined,
+  useWorker: boolean, wasmSystemAccess: types.WasmSystemAccess | undefined,
+): Promise<void> => {
   let worker: {
     onmessage: ((event: any) => void) | null
     postMessage: (data: Uint8Array | ArrayBuffer | WebAssembly.Module) => void
@@ -80,10 +84,27 @@ const startRunningService = async (wasmURL: string | URL, wasmModule: WebAssembl
 
   if (useWorker) {
     // Run esbuild off the main thread
-    let blob = new Blob([`onmessage=${WEB_WORKER_SOURCE_CODE}(postMessage)`], { type: 'text/javascript' })
+    let script = `onmessage=${WEB_WORKER_SOURCE_CODE}(postMessage)`;
+    if (wasmSystemAccess?.fsSpecifier) {
+      script = `import fs from "${wasmSystemAccess.fsSpecifier}";globalThis.fs=fs;${script}`
+    }
+    if (wasmSystemAccess?.processSpecifier) {
+      script = `import process from "${wasmSystemAccess.processSpecifier}";globalThis.process=process;${script}`
+    }
+    let blob = new Blob([script], { type: 'text/javascript' })
     worker = new Worker(URL.createObjectURL(blob))
   } else {
     // Run esbuild on the main thread
+    if (wasmSystemAccess?.fsSpecifier) {
+      (globalThis as any).fs = await import(wasmSystemAccess.fsSpecifier)
+    } else if (wasmSystemAccess?.fsNamespace) {
+      (globalThis as any).fs = wasmSystemAccess.fsNamespace
+    }
+    if (wasmSystemAccess?.processSpecifier) {
+      (globalThis as any).process = await import(wasmSystemAccess.processSpecifier)
+    } else if (wasmSystemAccess?.processNamespace) {
+      (globalThis as any).process = wasmSystemAccess?.processNamespace
+    }
     let onmessage = WEB_WORKER_FUNCTION((data: Uint8Array) => worker.onmessage!({ data }))
     worker = {
       onmessage: null,
@@ -93,18 +114,18 @@ const startRunningService = async (wasmURL: string | URL, wasmModule: WebAssembl
     }
   }
 
-  let firstMessageResolve: (value: void) => void
+  let firstMessageResolve: (value: string) => void
   let firstMessageReject: (error: any) => void
 
-  const firstMessagePromise = new Promise((resolve, reject) => {
+  const firstMessagePromise = new Promise<string>((resolve, reject) => {
     firstMessageResolve = resolve
     firstMessageReject = reject
   })
 
-  worker.onmessage = ({ data: error }) => {
+  worker.onmessage = ({ data }) => {
     worker.onmessage = ({ data }) => readFromStdout(data)
-    if (error) firstMessageReject(error)
-    else firstMessageResolve()
+    if (data.error) firstMessageReject(data.error)
+    else firstMessageResolve(data.ok)
   }
 
   worker.postMessage(wasmModule || new URL(wasmURL, location.href).toString())
@@ -114,12 +135,12 @@ const startRunningService = async (wasmURL: string | URL, wasmModule: WebAssembl
       worker.postMessage(bytes)
     },
     isSync: false,
-    hasFS: false,
+    hasFS: wasmSystemAccess !== undefined,
     esbuild: ourselves,
   })
 
   // This will throw if WebAssembly module instantiation fails
-  await firstMessagePromise
+  const defaultWD: string = await firstMessagePromise
 
   longLivedService = {
     build: (options: types.BuildOptions) =>
@@ -129,7 +150,7 @@ const startRunningService = async (wasmURL: string | URL, wasmModule: WebAssembl
           refs: null,
           options,
           isTTY: false,
-          defaultWD: '/',
+          defaultWD,
           callback: (err, res) => err ? reject(err) : resolve(res as types.BuildResult),
         })),
 
@@ -140,7 +161,7 @@ const startRunningService = async (wasmURL: string | URL, wasmModule: WebAssembl
           refs: null,
           options,
           isTTY: false,
-          defaultWD: '/',
+          defaultWD,
           callback: (err, res) => err ? reject(err) : resolve(res as types.BuildContext),
         })),
 

lib/shared/common.ts

@@ -61,6 +61,9 @@ let mustBeStringOrUint8Array = (value: string | Uint8Array | undefined): string
 let mustBeStringOrURL = (value: string | URL | undefined): string | null =>
   typeof value === 'string' || value instanceof URL ? null : 'a string or a URL'
 
+let mustBeObjectOrBoolean = (value: Object | boolean | undefined): string | null =>
+  typeof value === 'boolean' || (typeof value === 'object' && value !== null && !Array.isArray(value)) ? null : 'a boolean or an object'
+
 type OptionKeys = { [key: string]: boolean }
 
 function getFlag<T, K extends (keyof T & string)>(object: T, keys: OptionKeys, key: K, mustBeFn: (value: T[K]) => string | null): T[K] | undefined {
@@ -85,11 +88,43 @@ export function validateInitializeOptions(options: types.InitializeOptions): typ
   let wasmURL = getFlag(options, keys, 'wasmURL', mustBeStringOrURL)
   let wasmModule = getFlag(options, keys, 'wasmModule', mustBeWebAssemblyModule)
   let worker = getFlag(options, keys, 'worker', mustBeBoolean)
+  let wasmSystemAccess = getFlag(options, keys, 'wasmSystemAccess', mustBeObjectOrBoolean)
+  if (typeof wasmSystemAccess === "object") {
+    let fsSpecifier = getFlag(wasmSystemAccess, keys, 'fsSpecifier', mustBeString)
+    let fsNamespace = getFlag(wasmSystemAccess, keys, 'fsNamespace', mustBeObject)
+    if (fsSpecifier && fsNamespace) {
+      throw new Error(`The "wasmSystemAccess.fsSpecifier option is mutually exclusive with the "wasmSystemAccess.fsNamespace option.`)
+    }
+    if (!fsNamespace && !fsSpecifier) {
+      throw new Error(`Must provide either of "fsNamespace" or "fsSpecifier" when the "wasmSystemAccess" option is specified.`)
+    }
+    let processSpecifier = getFlag(wasmSystemAccess, keys, 'processSpecifier', mustBeString)
+    let processNamespace = getFlag(wasmSystemAccess, keys, 'processNamespace', mustBeObject)
+    if (processSpecifier && processNamespace) {
+      throw new Error(`The "wasmSystemAccess.processSpecifier" option is mutually exclusive with the "wasmSystemAccess.processNamespace" option.`)
+    }
+    if (!processNamespace && !processSpecifier) {
+      throw new Error(`Must provide either of "processNamespace" or "processSpecifier" when the "wasmSystemAccess" option is specified.`)
+    }
+    if (fsNamespace && worker === true) {
+      throw new Error(`The "wasmSystemAccess.fsNamespace" option is not compatible with the "worker" option.`)
+    }
+    if (processNamespace && worker === true) {
+      throw new Error(`The "wasmSystemAccess.processNamespace" option is not compatible with the "worker" option.`)
+    }
+    wasmSystemAccess = {
+      fsSpecifier,
+      fsNamespace,
+      processSpecifier,
+      processNamespace
+    }
+  }
   checkForInvalidFlags(options, keys, 'in initialize() call')
   return {
     wasmURL,
     wasmModule,
     worker,
+    wasmSystemAccess,
   }
 }
 

lib/shared/types.ts

@@ -631,6 +631,22 @@ export interface InitializeOptions {
    */
   wasmModule?: WebAssembly.Module
 
+  /**
+   * Enable system access (fs / process) when using WebAssembly. This requires
+   * that the host environment provide a namespace providing all APIs in
+   * Node.js' "fs" and "process" built-in modules.
+   * 
+   * By default this feature is disabled. To enable it, implementations for the
+   * "fs" and "process" namespaces must be provided by specifying either a
+   * namespace containing for "fs" and "process", or specifiers to an ES module that exports these bindings
+   * as a default export. The former is unsupported when using "worker: true".
+   * 
+   * The passed "fs" namespace must operate on Unix-style paths. One can thus
+   * not use this feature to access the local file system on Windows when using
+   * Node.js' built-in "fs" module.
+   */
+  wasmSystemAccess?: WasmSystemAccess
+
   /**
    * By default esbuild runs the WebAssembly-based browser API in a web worker
    * to avoid blocking the UI thread. This can be disabled by setting "worker"
@@ -639,4 +655,33 @@ export interface InitializeOptions {
   worker?: boolean
 }
 
+export interface WasmSystemAccess {
+  /**
+   * A module specifier to an ES module exporting an object that has a "node:fs"
+   * compatible signature.
+   *
+   * Mutually exclusive with the "fsNamespace" option. 
+   */
+  fsSpecifier?: string,
+  /**
+   * An object with a "node:fs" compatible signature.
+   * 
+   * Mutually exclusive with the "fsSpecifier" option.
+   */
+  fsNamespace?: any
+  /**
+   * A module specifier to an ES module exporting an object that has a
+   * "node:process" compatible signature.
+   * 
+   * Mutually exclusive with the "processNamespace" option.
+   */
+  processSpecifier?: string
+  /**
+   * An object with a "node:process" compatible signature.
+   * 
+   * Mutually exclusive with the "processSpecifier" option.
+   */
+  processNamespace?: any
+}
+
 export let version: string