Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

implement [data_blobs] #232

Merged
merged 1 commit into from Apr 2, 2022
Merged

implement [data_blobs] #232

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
32 changes: 28 additions & 4 deletions packages/core/src/plugins/bindings.ts
View file
Open in desktop
Expand Up @@ -12,6 +12,7 @@ import {
RequestContext,
SetupResult,
getRequestContext,
viewToBuffer,
} from "@miniflare/shared";
import dotenv from "dotenv";
import { MiniflareCoreError } from "../error";
Expand Down Expand Up @@ -46,6 +47,7 @@ export interface BindingsOptions {
globals?: Record<string, any>;
wasmBindings?: Record<string, string>;
textBlobBindings?: Record<string, string>;
dataBlobBindings?: Record<string, string>;
serviceBindings?: ServiceBindingsOptions;
}

Expand Down Expand Up @@ -175,6 +177,16 @@ export class BindingsPlugin
})
textBlobBindings?: Record<string, string>;

@Option({
type: OptionType.OBJECT,
typeFormat: "NAME=PATH",
name: "data-blob",
description: "Data blob to bind",
logName: "Data Blob Bindings",
fromWrangler: ({ data_blobs }) => data_blobs,
})
dataBlobBindings?: Record<string, string>;

@Option({
type: OptionType.OBJECT,
typeFormat: "NAME=MOUNT[@ENV]",
Expand Down Expand Up @@ -258,8 +270,9 @@ export class BindingsPlugin
// 2) .env Variables
// 3) WASM Module Bindings
// 4) Text blob Bindings
// 5) Service Bindings
// 6) Custom Bindings
// 5) Data blob Bindings
// 6) Service Bindings
// 7) Custom Bindings

const bindings: Context = {};
const watch: string[] = [];
Expand Down Expand Up @@ -303,12 +316,23 @@ export class BindingsPlugin
}
}

// 5) Load service bindings
// 5) Load data blobs from files
if (this.dataBlobBindings) {
// eslint-disable-next-line prefer-const
for (let [name, dataPath] of Object.entries(this.dataBlobBindings)) {
dataPath = path.resolve(this.ctx.rootPath, dataPath);
const fileContent = await fs.readFile(dataPath);
bindings[name] = viewToBuffer(fileContent);
watch.push(dataPath);
}
}

// 6) Load service bindings
for (const { name, service } of this.#processedServiceBindings) {
bindings[name] = new Fetcher(service, this.#getServiceFetch);
}

// 6) Copy user's arbitrary bindings
// 7) Copy user's arbitrary bindings
Object.assign(bindings, this.bindings);

return { globals: this.globals, bindings, watch };
Expand Down
56 changes: 48 additions & 8 deletions packages/core/test/plugins/bindings.spec.ts
View file
Open in desktop
Expand Up @@ -16,6 +16,7 @@ import {
NoOpLog,
PluginContext,
getRequestContext,
viewToBuffer,
} from "@miniflare/shared";
import {
getObjectProperties,
Expand All @@ -41,6 +42,8 @@ const addModulePath = path.join(fixturesPath, "add.wasm");
// lorem-ipsum.txt is five paragraphs of lorem ipsum nonsense text
const loremIpsumPath = path.join(fixturesPath, "lorem-ipsum.txt");
const loremIpsum = readFileSync(loremIpsumPath, "utf-8");
// we also make a data version of it to verify aganst data blobs
const loremIpsumData = viewToBuffer(readFileSync(loremIpsumPath));

test("BindingsPlugin: parses options from argv", (t) => {
let options = parsePluginArgv(BindingsPlugin, [
Expand All @@ -62,6 +65,10 @@ test("BindingsPlugin: parses options from argv", (t) => {
"TEXT1=text-blob-1.txt",
"--text-blob",
"TEXT2=text-blob-2.txt",
"--data-blob",
"DATA1=data-blob-1.bin",
"--data-blob",
"DATA2=data-blob-2.bin",
"--service",
"SERVICE1=service1",
"--service",
Expand All @@ -73,6 +80,7 @@ test("BindingsPlugin: parses options from argv", (t) => {
globals: { KEY3: "value3", KEY4: "value4" },
wasmBindings: { MODULE1: "module1.wasm", MODULE2: "module2.wasm" },
textBlobBindings: { TEXT1: "text-blob-1.txt", TEXT2: "text-blob-2.txt" },
dataBlobBindings: { DATA1: "data-blob-1.bin", DATA2: "data-blob-2.bin" },
serviceBindings: {
SERVICE1: "service1",
SERVICE2: { service: "service2", environment: "development" },
Expand Down Expand Up @@ -109,6 +117,10 @@ test("BindingsPlugin: parses options from wrangler config", async (t) => {
TEXT1: "text-blob-1.txt",
TEXT2: "text-blob-2.txt",
},
data_blobs: {
DATA1: "data-blob-1.bin",
DATA2: "data-blob-2.bin",
},
experimental_services: [
{ name: "SERVICE1", service: "service1", environment: "development" },
{ name: "SERVICE2", service: "service2", environment: "production" },
Expand All @@ -123,6 +135,7 @@ test("BindingsPlugin: parses options from wrangler config", async (t) => {
globals: { KEY5: "value5", KEY6: false, KEY7: 10 },
wasmBindings: { MODULE1: "module1.wasm", MODULE2: "module2.wasm" },
textBlobBindings: { TEXT1: "text-blob-1.txt", TEXT2: "text-blob-2.txt" },
dataBlobBindings: { DATA1: "data-blob-1.bin", DATA2: "data-blob-2.bin" },
serviceBindings: {
SERVICE1: { service: "service1", environment: "development" },
SERVICE2: { service: "service2", environment: "production" },
Expand Down Expand Up @@ -156,6 +169,7 @@ test("BindingsPlugin: logs options", (t) => {
globals: { KEY5: "value5", KEY6: "value6" },
wasmBindings: { MODULE1: "module1.wasm", MODULE2: "module2.wasm" },
textBlobBindings: { TEXT1: "text-blob-1.txt", TEXT2: "text-blob-2.txt" },
dataBlobBindings: { DATA1: "data-blob-1.bin", DATA2: "data-blob-2.bin" },
serviceBindings: {
SERVICE1: "service1",
SERVICE2: { service: "service2", environment: "development" },
Expand All @@ -168,6 +182,7 @@ test("BindingsPlugin: logs options", (t) => {
"Custom Globals: KEY5, KEY6",
"WASM Bindings: MODULE1, MODULE2",
"Text Blob Bindings: TEXT1, TEXT2",
"Data Blob Bindings: DATA1, DATA2",
"Service Bindings: SERVICE1, SERVICE2",
]);
logs = logPluginOptions(BindingsPlugin, { envPath: true });
Expand Down Expand Up @@ -292,23 +307,40 @@ test("BindingsPlugin: setup: loads text blob bindings", async (t) => {
t.is(result.bindings?.LOREM_IPSUM, loremIpsum);
});

test("BindingsPlugin: setup: loads data blob bindings", async (t) => {
let plugin = new BindingsPlugin(ctx, {
dataBlobBindings: { BINARY_DATA: loremIpsumPath },
});
let result = await plugin.setup();
t.deepEqual(result.bindings?.BINARY_DATA, loremIpsumData);

// Check resolves data blob bindings path relative to rootPath
plugin = new BindingsPlugin(
{ log, compat, rootPath: path.dirname(loremIpsumPath) },
{ dataBlobBindings: { BINARY_DATA: "lorem-ipsum.txt" } }
);
result = await plugin.setup();
t.deepEqual(result.bindings?.BINARY_DATA, loremIpsumData);
});

test("BindingsPlugin: setup: loads bindings from all sources", async (t) => {
// Bindings should be loaded in this order, from lowest to highest priority:
// 1) Wrangler [vars]
// 2) .env Variables
// 3) WASM Module Bindings
// 4) Text Blob Bindings
// 5) Service Bindings
// 6) Custom Bindings
// 5) Data Blob Bindings
// 6) Service Bindings
// 7) Custom Bindings

// wranglerOptions should contain [kWranglerBindings]
const wranglerOptions = parsePluginWranglerConfig(BindingsPlugin, {
vars: { A: "w", B: "w", C: "w", D: "w", E: "w", F: "w" },
vars: { A: "w", B: "w", C: "w", D: "w", E: "w", F: "w", G: "w" },
});

const tmp = await useTmp(t);
const envPath = path.join(tmp, ".env");
await fs.writeFile(envPath, "A=env\nB=env\nC=env\nD=env\nE=env");
await fs.writeFile(envPath, "A=env\nB=env\nC=env\nD=env\nE=env\nF=env");

const obj = { ping: "pong" };
const throws = () => {
Expand All @@ -321,11 +353,18 @@ test("BindingsPlugin: setup: loads bindings from all sources", async (t) => {
B: addModulePath,
C: addModulePath,
D: addModulePath,
E: addModulePath,
},
textBlobBindings: {
A: loremIpsumPath,
B: loremIpsumPath,
C: loremIpsumPath,
D: loremIpsumPath,
},
dataBlobBindings: {
A: loremIpsumPath,
B: loremIpsumPath,
C: loremIpsumPath,
},
serviceBindings: { A: throws, B: throws },
bindings: { A: obj },
Expand All @@ -334,10 +373,11 @@ test("BindingsPlugin: setup: loads bindings from all sources", async (t) => {
const result = await plugin.setup();
assert(result.bindings);

t.is(result.bindings.F, "w");
t.is(result.bindings.E, "env");
t.true(result.bindings.D instanceof WebAssembly.Module);
t.is(result.bindings.C, loremIpsum);
t.is(result.bindings.G, "w");
t.is(result.bindings.F, "env");
t.true(result.bindings.E instanceof WebAssembly.Module);
t.is(result.bindings.D, loremIpsum);
t.deepEqual(result.bindings.C, loremIpsumData);
t.true(result.bindings.B instanceof Fetcher);
t.is(result.bindings.A, obj);
});
Expand Down
5 changes: 3 additions & 2 deletions packages/shared/src/wrangler.ts
View file
Open in desktop
Expand Up @@ -34,8 +34,9 @@ export interface WranglerEnvironmentConfig {
crons?: string[];
}; // inherited
usage_model?: "bundled" | "unbound"; // inherited
wasm_modules?: Record<string, string>; // (probably) inherited
text_blobs?: Record<string, string>;
wasm_modules?: Record<string, string>; // inherited
text_blobs?: Record<string, string>; // inherited
data_blobs?: Record<string, string>; // inherited
experimental_services?: {
name: string;
service: string;
Expand Down