Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

jsg: dynamic imports #88

Merged
merged 2 commits into from Oct 11, 2022
Merged

jsg: dynamic imports #88

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2046a7b
jsg: dynamic imports
mikea Oct 10, 2022
7179247
review comments
mikea Oct 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/workerd/io/worker.c++
View file
Open in desktop
Expand Up @@ -1329,7 +1329,7 @@ Worker::Worker(kj::Own<const Script> scriptParam,
KJ_CASE_ONEOF(mainModule, kj::Path) {
// const_cast OK because we hold the lock.
auto& registry = KJ_ASSERT_NONNULL(const_cast<Script&>(*script).impl->getModuleRegistry());
KJ_IF_MAYBE(entry, registry.resolve(mainModule)) {
KJ_IF_MAYBE(entry, registry.resolve(lock.v8Isolate, mainModule)) {
JSG_REQUIRE(entry->maybeSynthetic == nullptr, TypeError,
"Main module must be an ES module.");
auto module = entry->module.Get(lock.v8Isolate);
Expand Down
10 changes: 5 additions & 5 deletions src/workerd/jsg/modules.c++
View file
Open in desktop
Expand Up @@ -31,7 +31,7 @@ v8::MaybeLocal<v8::Module> resolveCallback(v8::Local<v8::Context> context,

kj::Path targetPath = referrerPath.parent().eval(kj::str(specifier));

result = JSG_REQUIRE_NONNULL(registry->resolve(targetPath), Error,
result = JSG_REQUIRE_NONNULL(registry->resolve(isolate, targetPath), Error,
"No such module \"", targetPath.toString(),
"\".\n imported from \"", referrerPath.toString(), "\"")
.module.Get(isolate);
Expand All @@ -56,7 +56,7 @@ v8::MaybeLocal<v8::Value> evaluateSyntheticModuleCallback(

KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() {
auto registry = getModulesForResolveCallback(isolate);
auto& entry = KJ_ASSERT_NONNULL(registry->resolve(module),
auto& entry = KJ_ASSERT_NONNULL(registry->resolve(isolate, module),
"module passed to evaluateSyntheticModuleCallback isn't in modules table");

// V8 doc comments say this callback must always return an already-resolved promise... I don't
Expand Down Expand Up @@ -172,7 +172,7 @@ v8::Local<v8::Value> CommonJsModuleContext::require(kj::String specifier, v8::Is

kj::Path targetPath = path.parent().eval(specifier);

auto& info = JSG_REQUIRE_NONNULL(modulesForResolveCallback->resolve(targetPath),
auto& info = JSG_REQUIRE_NONNULL(modulesForResolveCallback->resolve(isolate, targetPath),
Error, "No such module \"", targetPath.toString(), "\".");
// Adding imported from suffix here not necessary like it is for resolveCallback, since we have a
// js stack that will include the parent module's name and location of the failed require().
Expand Down Expand Up @@ -252,7 +252,7 @@ namespace {
v8::Local<v8::Module> compileEsmModule(
v8::Isolate* isolate,
kj::StringPtr name,
kj::StringPtr content) {
kj::ArrayPtr<const char> content) {
// Must pass true for `is_module`, but we can skip everything else.
const int resourceLineOffset = 0;
const int resourceColumnOffset = 0;
Expand Down Expand Up @@ -301,7 +301,7 @@ ModuleRegistry::ModuleInfo::ModuleInfo(
ModuleRegistry::ModuleInfo::ModuleInfo(
v8::Isolate* isolate,
kj::StringPtr name,
kj::StringPtr content)
kj::ArrayPtr<const char> content)
: ModuleInfo(isolate, compileEsmModule(isolate, name, content)) {}

ModuleRegistry::ModuleInfo::ModuleInfo(
Expand Down
64 changes: 51 additions & 13 deletions src/workerd/jsg/modules.h
View file
Open in desktop
Expand Up @@ -161,7 +161,7 @@ class ModuleRegistry {
v8::Local<v8::Module> module,
kj::Maybe<SyntheticModuleInfo> maybeSynthetic = nullptr);

ModuleInfo(v8::Isolate* isolate, kj::StringPtr name, kj::StringPtr content);
ModuleInfo(v8::Isolate* isolate, kj::StringPtr name, kj::ArrayPtr<const char> content);

ModuleInfo(v8::Isolate* isolate, kj::StringPtr name,
kj::Maybe<kj::ArrayPtr<kj::StringPtr>> maybeExports,
Expand All @@ -171,9 +171,9 @@ class ModuleRegistry {
ModuleInfo& operator=(ModuleInfo&&) = default;
};

virtual kj::Maybe<ModuleInfo&> resolve(const kj::Path& specifier) = 0;
virtual kj::Maybe<ModuleInfo&> resolve(v8::Isolate* isolate, const kj::Path& specifier) = 0;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's have resolve take a jsg::Lock& js instead of the isolate

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unfortunately that would mean changing very long call chains, since they don't have lock wired through.

E.g.:

resolve <- evaluateSyntheticModuleCallback <- createSyntheticModule <- ModuleInfo with no lock in sight yet.

Let me know if you want to create locks before calling resolve or tackle passing jsg::Lock everywhere in modules.h

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I do eventually want to update modules.h to use jsg::Lock& throughout so if you don't want to do this now that's fine. That said, it's simple to grab the jsg::Lock for the current isolate without passing it through everywhere using auto& js jsg::Lock::from(isolate).

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll make separate PR with this cleanup today if you don't mind.


virtual kj::Maybe<ModuleInfo&> resolve(v8::Local<v8::Module> module) = 0;
virtual kj::Maybe<ModuleInfo&> resolve(v8::Isolate* isolate, v8::Local<v8::Module> module) = 0;

virtual kj::Maybe<const kj::Path&> resolvePath(v8::Local<v8::Module> referrer)= 0;

Expand All @@ -198,17 +198,26 @@ class ModuleRegistryImpl final: public ModuleRegistry {
entries.insert(Entry(specifier, kj::fwd<ModuleInfo>(info)));
}

kj::Maybe<ModuleInfo&> resolve(const kj::Path& specifier) override {
void addBuiltinModule(const kj::Path& specifier, kj::ArrayPtr<const char> sourceCode) {
// Register new module accessible by a given importPath. The module is instantiated
// after first resolve attempt within application has failed, i.e. it is possible for
// application to override the module.
// sourceCode has to exist while this ModuleRegistry exists.
// The expectation is for this method to be called during the assembly of worker global context.
entries.insert(Entry(specifier, sourceCode));
}

kj::Maybe<ModuleInfo&> resolve(v8::Isolate* isolate, const kj::Path& specifier) override {
// TODO(soon): Soon we will support prefixed imports of Workers built in types.
KJ_IF_MAYBE(entry, entries.find(specifier)) {
return entry->info;
return entry->module(isolate);
}
return nullptr;
}

kj::Maybe<ModuleInfo&> resolve(v8::Local<v8::Module> module) override {
kj::Maybe<ModuleInfo&> resolve(v8::Isolate* isolate, v8::Local<v8::Module> module) override {
KJ_IF_MAYBE(entry, entries.template find<1>(module)) {
return entry->info;
return entry->module(isolate);
}
return nullptr;
}
Expand All @@ -223,7 +232,7 @@ class ModuleRegistryImpl final: public ModuleRegistry {
size_t size() const { return entries.size(); }

Promise<Value> resolveDynamicImport(v8::Isolate* isolate, kj::Path specifier) override {
KJ_IF_MAYBE(info, resolve(specifier)) {
KJ_IF_MAYBE(info, resolve(isolate, specifier)) {
KJ_IF_MAYBE(func, dynamicImportHandler) {
auto handler = [&info = *info, isolate]() -> Value {
auto module = info.module.Get(isolate);
Expand Down Expand Up @@ -252,12 +261,32 @@ class ModuleRegistryImpl final: public ModuleRegistry {
// object by identity. We use a kj::Table!
struct Entry {
kj::Path specifier;
ModuleInfo info;
kj::OneOf<ModuleInfo, kj::ArrayPtr<const char>> info;
// Either instantiated module or module source code.

Entry(kj::Path& specifier, ModuleInfo info)
Entry(const kj::Path& specifier, ModuleInfo info)
: specifier(specifier.clone()), info(kj::mv(info)) {}

Entry(const kj::Path& specifier, kj::ArrayPtr<const char> src)
: specifier(specifier.clone()), info(src) {}

Entry(Entry&&) = default;
Entry& operator=(Entry&&) = default;

ModuleInfo& module(v8::Isolate* isolate) {
// Lazily instantiate module from source code if needed

KJ_SWITCH_ONEOF(info) {
KJ_CASE_ONEOF(moduleInfo, ModuleInfo) {
return moduleInfo;
}
KJ_CASE_ONEOF(src, kj::ArrayPtr<const char>) {
info = ModuleInfo(isolate, specifier.toString(), src);
return KJ_ASSERT_NONNULL(info.tryGet<ModuleInfo>());
}
}
KJ_UNREACHABLE;
}
};

struct SpecifierHashCallbacks {
Expand All @@ -276,19 +305,28 @@ class ModuleRegistryImpl final: public ModuleRegistry {
const Entry& keyForRow(const Entry& row) const { return row; }

bool matches(const Entry& entry, const Entry& other) const {
return entry.info.hash == other.info.hash;
return hashCode(entry) == hashCode(other);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks like it has problems with hash collisions. It looks like previously hash codes were based strictly on V8 identity hashes which might plausibly be unique (I'm not sure), but now it's based on string hashes in some cases which definitely could collide. This function needs to change to actually perform a proper comparison.

}

bool matches(const Entry& entry, v8::Local<v8::Module>& module) const {
return entry.info.hash == module->GetIdentityHash();
return entry.info.template is<ModuleInfo>() &&
entry.info.template get<ModuleInfo>().hash == module->GetIdentityHash();
}

uint hashCode(v8::Local<v8::Module>& module) const {
return kj::hashCode(module->GetIdentityHash());
}

uint hashCode(const Entry& entry) const {
return entry.info.hash;
KJ_SWITCH_ONEOF(entry.info) {
KJ_CASE_ONEOF(moduleInfo, ModuleInfo) {
return moduleInfo.hash;
}
KJ_CASE_ONEOF(src, kj::ArrayPtr<const char>) {
return kj::hashCode(src);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess this is hashing the whole source code, which sounds kind of expensive. Maybe we could key off a pointer to the code instead? This always points to a static constant, right?

}
}
KJ_UNREACHABLE;
}
};

Expand Down