From ee8ce0fdd268042c28074c79f8a8a64cdc678776 Mon Sep 17 00:00:00 2001 From: Sunil Pai Date: Fri, 1 Apr 2022 11:10:01 +0530 Subject: [PATCH] implement `[data_blobs]` This implements support for `[data_blobs]` for service workers, which lets you bind some data from a file as an ArrayBuffer in service-worker format workers. It also adds a `--data-blob` cli argument for the same. --- packages/core/src/plugins/bindings.ts | 32 ++++++++++-- packages/core/test/plugins/bindings.spec.ts | 56 ++++++++++++++++++--- packages/shared/src/wrangler.ts | 5 +- 3 files changed, 79 insertions(+), 14 deletions(-) diff --git a/packages/core/src/plugins/bindings.ts b/packages/core/src/plugins/bindings.ts index 9b135088c..20d8de6e0 100644 --- a/packages/core/src/plugins/bindings.ts +++ b/packages/core/src/plugins/bindings.ts @@ -12,6 +12,7 @@ import { RequestContext, SetupResult, getRequestContext, + viewToBuffer, } from "@miniflare/shared"; import dotenv from "dotenv"; import { MiniflareCoreError } from "../error"; @@ -46,6 +47,7 @@ export interface BindingsOptions { globals?: Record; wasmBindings?: Record; textBlobBindings?: Record; + dataBlobBindings?: Record; serviceBindings?: ServiceBindingsOptions; } @@ -175,6 +177,16 @@ export class BindingsPlugin }) textBlobBindings?: Record; + @Option({ + type: OptionType.OBJECT, + typeFormat: "NAME=PATH", + name: "data-blob", + description: "Data blob to bind", + logName: "Data Blob Bindings", + fromWrangler: ({ data_blobs }) => data_blobs, + }) + dataBlobBindings?: Record; + @Option({ type: OptionType.OBJECT, typeFormat: "NAME=MOUNT[@ENV]", @@ -258,8 +270,9 @@ export class BindingsPlugin // 2) .env Variables // 3) WASM Module Bindings // 4) Text blob Bindings - // 5) Service Bindings - // 6) Custom Bindings + // 5) Data blob Bindings + // 6) Service Bindings + // 7) Custom Bindings const bindings: Context = {}; const watch: string[] = []; @@ -303,12 +316,23 @@ export class BindingsPlugin } } - // 5) Load service bindings + // 5) Load data blobs from files + if (this.dataBlobBindings) { + // eslint-disable-next-line prefer-const + for (let [name, dataPath] of Object.entries(this.dataBlobBindings)) { + dataPath = path.resolve(this.ctx.rootPath, dataPath); + const fileContent = await fs.readFile(dataPath); + bindings[name] = viewToBuffer(fileContent); + watch.push(dataPath); + } + } + + // 6) Load service bindings for (const { name, service } of this.#processedServiceBindings) { bindings[name] = new Fetcher(service, this.#getServiceFetch); } - // 6) Copy user's arbitrary bindings + // 7) Copy user's arbitrary bindings Object.assign(bindings, this.bindings); return { globals: this.globals, bindings, watch }; diff --git a/packages/core/test/plugins/bindings.spec.ts b/packages/core/test/plugins/bindings.spec.ts index 3b8ba0ee6..74c8796ad 100644 --- a/packages/core/test/plugins/bindings.spec.ts +++ b/packages/core/test/plugins/bindings.spec.ts @@ -16,6 +16,7 @@ import { NoOpLog, PluginContext, getRequestContext, + viewToBuffer, } from "@miniflare/shared"; import { getObjectProperties, @@ -41,6 +42,8 @@ const addModulePath = path.join(fixturesPath, "add.wasm"); // lorem-ipsum.txt is five paragraphs of lorem ipsum nonsense text const loremIpsumPath = path.join(fixturesPath, "lorem-ipsum.txt"); const loremIpsum = readFileSync(loremIpsumPath, "utf-8"); +// we also make a data version of it to verify aganst data blobs +const loremIpsumData = viewToBuffer(readFileSync(loremIpsumPath)); test("BindingsPlugin: parses options from argv", (t) => { let options = parsePluginArgv(BindingsPlugin, [ @@ -62,6 +65,10 @@ test("BindingsPlugin: parses options from argv", (t) => { "TEXT1=text-blob-1.txt", "--text-blob", "TEXT2=text-blob-2.txt", + "--data-blob", + "DATA1=data-blob-1.bin", + "--data-blob", + "DATA2=data-blob-2.bin", "--service", "SERVICE1=service1", "--service", @@ -73,6 +80,7 @@ test("BindingsPlugin: parses options from argv", (t) => { globals: { KEY3: "value3", KEY4: "value4" }, wasmBindings: { MODULE1: "module1.wasm", MODULE2: "module2.wasm" }, textBlobBindings: { TEXT1: "text-blob-1.txt", TEXT2: "text-blob-2.txt" }, + dataBlobBindings: { DATA1: "data-blob-1.bin", DATA2: "data-blob-2.bin" }, serviceBindings: { SERVICE1: "service1", SERVICE2: { service: "service2", environment: "development" }, @@ -109,6 +117,10 @@ test("BindingsPlugin: parses options from wrangler config", async (t) => { TEXT1: "text-blob-1.txt", TEXT2: "text-blob-2.txt", }, + data_blobs: { + DATA1: "data-blob-1.bin", + DATA2: "data-blob-2.bin", + }, experimental_services: [ { name: "SERVICE1", service: "service1", environment: "development" }, { name: "SERVICE2", service: "service2", environment: "production" }, @@ -123,6 +135,7 @@ test("BindingsPlugin: parses options from wrangler config", async (t) => { globals: { KEY5: "value5", KEY6: false, KEY7: 10 }, wasmBindings: { MODULE1: "module1.wasm", MODULE2: "module2.wasm" }, textBlobBindings: { TEXT1: "text-blob-1.txt", TEXT2: "text-blob-2.txt" }, + dataBlobBindings: { DATA1: "data-blob-1.bin", DATA2: "data-blob-2.bin" }, serviceBindings: { SERVICE1: { service: "service1", environment: "development" }, SERVICE2: { service: "service2", environment: "production" }, @@ -156,6 +169,7 @@ test("BindingsPlugin: logs options", (t) => { globals: { KEY5: "value5", KEY6: "value6" }, wasmBindings: { MODULE1: "module1.wasm", MODULE2: "module2.wasm" }, textBlobBindings: { TEXT1: "text-blob-1.txt", TEXT2: "text-blob-2.txt" }, + dataBlobBindings: { DATA1: "data-blob-1.bin", DATA2: "data-blob-2.bin" }, serviceBindings: { SERVICE1: "service1", SERVICE2: { service: "service2", environment: "development" }, @@ -168,6 +182,7 @@ test("BindingsPlugin: logs options", (t) => { "Custom Globals: KEY5, KEY6", "WASM Bindings: MODULE1, MODULE2", "Text Blob Bindings: TEXT1, TEXT2", + "Data Blob Bindings: DATA1, DATA2", "Service Bindings: SERVICE1, SERVICE2", ]); logs = logPluginOptions(BindingsPlugin, { envPath: true }); @@ -292,23 +307,40 @@ test("BindingsPlugin: setup: loads text blob bindings", async (t) => { t.is(result.bindings?.LOREM_IPSUM, loremIpsum); }); +test("BindingsPlugin: setup: loads data blob bindings", async (t) => { + let plugin = new BindingsPlugin(ctx, { + dataBlobBindings: { BINARY_DATA: loremIpsumPath }, + }); + let result = await plugin.setup(); + t.deepEqual(result.bindings?.BINARY_DATA, loremIpsumData); + + // Check resolves data blob bindings path relative to rootPath + plugin = new BindingsPlugin( + { log, compat, rootPath: path.dirname(loremIpsumPath) }, + { dataBlobBindings: { BINARY_DATA: "lorem-ipsum.txt" } } + ); + result = await plugin.setup(); + t.deepEqual(result.bindings?.BINARY_DATA, loremIpsumData); +}); + test("BindingsPlugin: setup: loads bindings from all sources", async (t) => { // Bindings should be loaded in this order, from lowest to highest priority: // 1) Wrangler [vars] // 2) .env Variables // 3) WASM Module Bindings // 4) Text Blob Bindings - // 5) Service Bindings - // 6) Custom Bindings + // 5) Data Blob Bindings + // 6) Service Bindings + // 7) Custom Bindings // wranglerOptions should contain [kWranglerBindings] const wranglerOptions = parsePluginWranglerConfig(BindingsPlugin, { - vars: { A: "w", B: "w", C: "w", D: "w", E: "w", F: "w" }, + vars: { A: "w", B: "w", C: "w", D: "w", E: "w", F: "w", G: "w" }, }); const tmp = await useTmp(t); const envPath = path.join(tmp, ".env"); - await fs.writeFile(envPath, "A=env\nB=env\nC=env\nD=env\nE=env"); + await fs.writeFile(envPath, "A=env\nB=env\nC=env\nD=env\nE=env\nF=env"); const obj = { ping: "pong" }; const throws = () => { @@ -321,11 +353,18 @@ test("BindingsPlugin: setup: loads bindings from all sources", async (t) => { B: addModulePath, C: addModulePath, D: addModulePath, + E: addModulePath, }, textBlobBindings: { A: loremIpsumPath, B: loremIpsumPath, C: loremIpsumPath, + D: loremIpsumPath, + }, + dataBlobBindings: { + A: loremIpsumPath, + B: loremIpsumPath, + C: loremIpsumPath, }, serviceBindings: { A: throws, B: throws }, bindings: { A: obj }, @@ -334,10 +373,11 @@ test("BindingsPlugin: setup: loads bindings from all sources", async (t) => { const result = await plugin.setup(); assert(result.bindings); - t.is(result.bindings.F, "w"); - t.is(result.bindings.E, "env"); - t.true(result.bindings.D instanceof WebAssembly.Module); - t.is(result.bindings.C, loremIpsum); + t.is(result.bindings.G, "w"); + t.is(result.bindings.F, "env"); + t.true(result.bindings.E instanceof WebAssembly.Module); + t.is(result.bindings.D, loremIpsum); + t.deepEqual(result.bindings.C, loremIpsumData); t.true(result.bindings.B instanceof Fetcher); t.is(result.bindings.A, obj); }); diff --git a/packages/shared/src/wrangler.ts b/packages/shared/src/wrangler.ts index 8f2a97800..2f59a6b03 100644 --- a/packages/shared/src/wrangler.ts +++ b/packages/shared/src/wrangler.ts @@ -34,8 +34,9 @@ export interface WranglerEnvironmentConfig { crons?: string[]; }; // inherited usage_model?: "bundled" | "unbound"; // inherited - wasm_modules?: Record; // (probably) inherited - text_blobs?: Record; + wasm_modules?: Record; // inherited + text_blobs?: Record; // inherited + data_blobs?: Record; // inherited experimental_services?: { name: string; service: string;