From 3383223f3c7640a32ad98815d96ba41538455d59 Mon Sep 17 00:00:00 2001 From: Kevin Castro Date: Sat, 30 May 2026 13:55:25 -0700 Subject: [PATCH] feat(modules): async loader for dynamic import() Add JS_SetModuleLoaderFuncAsync together with JS_FulfillAsyncModuleLoad and JS_RejectAsyncModuleLoad so embedders can satisfy import(specifier) from an asynchronous source (e.g. a network fetch) without blocking the JS thread. Scope is limited to dynamic import(); static imports keep using the synchronous JSModuleLoaderFunc[2], mirroring the HTML spec split between HostImportModuleDynamically and HostResolveImportedModule. The loader is handed an opaque handle and settles it later. Properties: - the specifier is normalized and a cached module is settled without calling the loader; - concurrent import()s of the same specifier share one load: the loader runs once and the module is evaluated once; - settling a handle more than once is a no-op, and handles still in flight are reclaimed by JS_FreeRuntime. Tested in api-test.c: fulfill, reject, transitive static import, cache hit, in-flight dedup, top-level await, re-entrant settle, attribute and normalizer hooks, nested dynamic import, and shutdown sweeps. --- api-test.c | 694 +++++++++++++++++++++++++++++++++++++++++++++++++++++ quickjs.c | 324 +++++++++++++++++++++++-- quickjs.h | 41 ++++ 3 files changed, 1036 insertions(+), 23 deletions(-) diff --git a/api-test.c b/api-test.c index 7aa7dc8a6..c2f4be5fb 100644 --- a/api-test.c +++ b/api-test.c @@ -391,6 +391,684 @@ static void module_serde(void) JS_FreeRuntime(rt); } +/* Tests for the async dynamic-import loader (JS_SetModuleLoaderFuncAsync etc). + The loader simulates async I/O: it stashes the handle and enqueues a job that + settles it on a later tick; tests pump the queue and check import()'s result. */ + +typedef enum { + DELIVER_FULFILL, /* compile `deliver_src` as a module and fulfill */ + DELIVER_FULFILL_NULL, /* fulfill with a NULL module (generic failure) */ + DELIVER_REJECT, /* reject with an Error whose message is `deliver_src` */ + DELIVER_DOUBLE, /* fulfill, then settle again to exercise the guard */ + DELIVER_NEVER, /* stash the handle but never settle it */ +} deliver_mode; + +typedef struct { + int loader_calls; /* async loader invocations */ + int sync_loader_calls; /* sync loader invocations (transitive deps) */ + int normalize_calls; /* async normalizer invocations */ + int attrs_calls; /* async attribute-check invocations */ + bool attrs_reject; /* if set, the attribute check fails */ + bool reentrant; /* if set, the loader settles synchronously */ + deliver_mode mode; + const char *deliver_src; /* module source, or reject message */ + /* pending hand-off between the async loader and its deliver job */ + JSAsyncModuleLoadHandle handle; + char *module_name; +} async_test_state; + +static async_test_state *g_async; + +/* identity normalizer that counts calls, to prove the hook is wired in */ +static char *async_normalize(JSContext *ctx, const char *base_name, + const char *name, void *opaque) +{ + (void)base_name; (void)opaque; + g_async->normalize_calls++; + return js_strdup(ctx, name); +} + +/* Async attribute checker: counts calls and optionally fails. */ +static int async_check_attrs(JSContext *ctx, void *opaque, JSValueConst attributes) +{ + (void)opaque; (void)attributes; + g_async->attrs_calls++; + if (g_async->attrs_reject) { + JS_ThrowTypeError(ctx, "unsupported import attributes"); + return -1; + } + return 0; +} + +/* sync loader serving a tiny fixed table, for static/transitive imports */ +static JSModuleDef *sync_module_loader(JSContext *ctx, const char *module_name, + void *opaque) +{ + (void)opaque; + const char *src = NULL; + if (!strcmp(module_name, "base")) + src = "export const base = 41;"; + else if (!strcmp(module_name, "cached")) + src = "export const v = 7;"; + if (!src) { + JS_ThrowReferenceError(ctx, "sync loader: unknown module '%s'", + module_name); + return NULL; + } + if (g_async) + g_async->sync_loader_calls++; + JSValue mod_val = JS_Eval(ctx, src, strlen(src), module_name, + JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY); + if (JS_IsException(mod_val)) + return NULL; + JSModuleDef *m = JS_VALUE_GET_PTR(mod_val); + JS_FreeValue(ctx, mod_val); + return m; +} + +static JSValue async_loader_deliver_job(JSContext *ctx, int argc, JSValueConst *argv) +{ + (void)argc; (void)argv; + JSAsyncModuleLoadHandle handle = g_async->handle; + char *name = g_async->module_name; + g_async->handle = NULL; + g_async->module_name = NULL; + assert(handle); + + if (g_async->mode == DELIVER_REJECT) { + JSValue err = JS_NewError(ctx); + JS_DefinePropertyValueStr(ctx, err, "message", + JS_NewString(ctx, g_async->deliver_src), + JS_PROP_C_W_E); + /* JS_RejectAsyncModuleLoad takes ownership of `err`. */ + JS_RejectAsyncModuleLoad(ctx, handle, err); + free(name); + return JS_UNDEFINED; + } + + if (g_async->mode == DELIVER_FULFILL_NULL) { + /* Fulfilling with NULL means "load failed"; the engine synthesizes a + generic rejection. */ + JS_FulfillAsyncModuleLoad(ctx, handle, NULL); + free(name); + return JS_UNDEFINED; + } + + JSValue mod_val = JS_Eval(ctx, g_async->deliver_src, + strlen(g_async->deliver_src), name, + JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY); + assert(!JS_IsException(mod_val)); + JSModuleDef *m = JS_VALUE_GET_PTR(mod_val); + assert(m); + /* The module def is owned by the runtime's module table; the JSValue + wrapper is just a handle, so free it. */ + JS_FreeValue(ctx, mod_val); + free(name); + + JS_FulfillAsyncModuleLoad(ctx, handle, m); + if (g_async->mode == DELIVER_DOUBLE) { + /* Second settle on the same handle must be a safe no-op. Try both a + repeat fulfill and a reject; neither should change the result or + crash. */ + JS_FulfillAsyncModuleLoad(ctx, handle, m); + JS_RejectAsyncModuleLoad(ctx, handle, JS_NewString(ctx, "ignored")); + } + return JS_UNDEFINED; +} + +static void async_loader(JSContext *ctx, const char *module_name, + void *opaque, JSValueConst attributes, + JSAsyncModuleLoadHandle handle) +{ + (void)opaque; (void)attributes; + g_async->loader_calls++; + assert(!g_async->handle); /* one outstanding at a time in these tests */ + g_async->handle = handle; + g_async->module_name = strdup(module_name); + if (g_async->mode == DELIVER_NEVER) + return; /* leave the handle pending; JS_FreeRuntime must reclaim it */ + if (g_async->reentrant) { + /* Settle synchronously, from inside the loader callback itself. */ + async_loader_deliver_job(ctx, 0, NULL); + return; + } + int r = JS_EnqueueJob(ctx, async_loader_deliver_job, 0, NULL); + assert(r == 0); +} + +/* Pump the job queue to quiescence (bounded, to catch run-away loops). */ +static void pump_jobs(JSRuntime *rt) +{ + int max_iters = 4096; + while (JS_IsJobPending(rt) && max_iters-- > 0) { + JSContext *job_ctx; + int r = JS_ExecutePendingJob(rt, &job_ctx); + assert(r >= 0); + } + assert(max_iters > 0); +} + +static int32_t get_int_global(JSContext *ctx, const char *name) +{ + JSValue global = JS_GetGlobalObject(ctx); + JSValue v = JS_GetPropertyStr(ctx, global, name); + assert(JS_IsNumber(v)); + int32_t n = -1; + JS_ToInt32(ctx, &n, v); + JS_FreeValue(ctx, v); + JS_FreeValue(ctx, global); + return n; +} + +static char *get_string_global(JSContext *ctx, const char *name) +{ + JSValue global = JS_GetGlobalObject(ctx); + JSValue v = JS_GetPropertyStr(ctx, global, name); + assert(JS_IsString(v)); + const char *s = JS_ToCString(ctx, v); + char *out = strdup(s); + JS_FreeCString(ctx, s); + JS_FreeValue(ctx, v); + JS_FreeValue(ctx, global); + return out; +} + +static void run_async_eval(JSContext *ctx, const char *code) +{ + JSValue ev = JS_Eval(ctx, code, strlen(code), "", JS_EVAL_TYPE_MODULE); + assert(!JS_IsException(ev)); + JS_FreeValue(ctx, ev); +} + +/* fulfill: import('hello') resolves to { v: 42 }. */ +static void async_module_loader(void) +{ + async_test_state st = { .mode = DELIVER_FULFILL, + .deliver_src = "export const v = 42;" }; + g_async = &st; + + JSRuntime *rt = new_runtime(); + JS_SetModuleLoaderFuncAsync(rt, NULL, async_loader, NULL, NULL); + JSContext *ctx = JS_NewContext(rt); + + run_async_eval(ctx, + "import('hello').then(m => { globalThis.__result = m.v; });"); + pump_jobs(rt); + + assert(st.loader_calls == 1); + assert(!st.handle); + assert(get_int_global(ctx, "__result") == 42); + + JS_FreeContext(ctx); + JS_FreeRuntime(rt); + g_async = NULL; +} + +/* reject: import('boom') is caught with our error message. */ +static void async_module_loader_reject(void) +{ + async_test_state st = { .mode = DELIVER_REJECT, .deliver_src = "kaboom" }; + g_async = &st; + + JSRuntime *rt = new_runtime(); + JS_SetModuleLoaderFuncAsync(rt, NULL, async_loader, NULL, NULL); + JSContext *ctx = JS_NewContext(rt); + + run_async_eval(ctx, + "import('boom').then(() => { globalThis.__err = 'resolved?!'; }," + " e => { globalThis.__err = e.message; });"); + pump_jobs(rt); + + assert(st.loader_calls == 1); + char *err = get_string_global(ctx, "__err"); + assert(!strcmp(err, "kaboom")); + free(err); + + JS_FreeContext(ctx); + JS_FreeRuntime(rt); + g_async = NULL; +} + +/* transitive: an async-loaded module statically imports 'base'; that static + dependency must be resolved through the SYNC loader. */ +static void async_module_loader_transitive(void) +{ + async_test_state st = { .mode = DELIVER_FULFILL, + .deliver_src = "import { base } from 'base';" + "export const v = base + 1;" }; + g_async = &st; + + JSRuntime *rt = new_runtime(); + JS_SetModuleLoaderFunc(rt, NULL, sync_module_loader, NULL); + JS_SetModuleLoaderFuncAsync(rt, NULL, async_loader, NULL, NULL); + JSContext *ctx = JS_NewContext(rt); + + run_async_eval(ctx, + "import('combo').then(m => { globalThis.__result = m.v; });"); + pump_jobs(rt); + + assert(st.loader_calls == 1); /* 'combo' went through the async loader */ + assert(st.sync_loader_calls == 1); /* 'base' went through the sync loader */ + assert(get_int_global(ctx, "__result") == 42); + + JS_FreeContext(ctx); + JS_FreeRuntime(rt); + g_async = NULL; +} + +/* cache hit: a specifier already in the module table (loaded via static + import through the sync loader) is settled from cache, NOT via the loader. */ +static void async_module_loader_cache_hit(void) +{ + async_test_state st = { .mode = DELIVER_FULFILL, + .deliver_src = "export const v = 999;" /* must NOT be used */ }; + g_async = &st; + + JSRuntime *rt = new_runtime(); + JS_SetModuleLoaderFunc(rt, NULL, sync_module_loader, NULL); + JS_SetModuleLoaderFuncAsync(rt, NULL, async_loader, NULL, NULL); + JSContext *ctx = JS_NewContext(rt); + + /* Statically import 'cached' so it's resolved + in the module table, then + dynamically import the same specifier. */ + run_async_eval(ctx, + "import { v } from 'cached';" + "import('cached').then(m => { globalThis.__result = m.v; });"); + pump_jobs(rt); + + assert(st.sync_loader_calls == 1); /* only the static import hit the loader */ + assert(st.loader_calls == 0); /* the dynamic import was a cache hit */ + assert(!st.handle); + assert(get_int_global(ctx, "__result") == 7); + + JS_FreeContext(ctx); + JS_FreeRuntime(rt); + g_async = NULL; +} + +/* double settle: a second fulfill/reject on the same handle is a no-op. */ +static void async_module_loader_double_settle(void) +{ + async_test_state st = { .mode = DELIVER_DOUBLE, + .deliver_src = "export const v = 5;" }; + g_async = &st; + + JSRuntime *rt = new_runtime(); + JS_SetModuleLoaderFuncAsync(rt, NULL, async_loader, NULL, NULL); + JSContext *ctx = JS_NewContext(rt); + + run_async_eval(ctx, + "import('twice').then(m => { globalThis.__result = m.v; }," + " () => { globalThis.__result = -1; });"); + pump_jobs(rt); + + assert(st.loader_calls == 1); + /* The first fulfill wins; the redundant fulfill/reject are ignored. */ + assert(get_int_global(ctx, "__result") == 5); + + JS_FreeContext(ctx); + JS_FreeRuntime(rt); + g_async = NULL; +} + +/* shutdown sweep: the host never settles the handle. JS_FreeRuntime must + reclaim it without leaking (new_runtime aborts on leaks) or crashing. */ +static void async_module_loader_shutdown_pending(void) +{ + async_test_state st = { .mode = DELIVER_NEVER, .deliver_src = NULL }; + g_async = &st; + + JSRuntime *rt = new_runtime(); + JS_SetModuleLoaderFuncAsync(rt, NULL, async_loader, NULL, NULL); + JSContext *ctx = JS_NewContext(rt); + + run_async_eval(ctx, "import('never').then(() => {}, () => {});"); + pump_jobs(rt); + + assert(st.loader_calls == 1); + assert(st.handle); /* still pending, deliberately never settled */ + free(st.module_name); /* the deliver job that would free it never ran */ + st.module_name = NULL; + + /* The pending handle (and the references it owns) must be reclaimed here. */ + JS_FreeContext(ctx); + JS_FreeRuntime(rt); + g_async = NULL; +} + +/* top-level await: the loaded module's top-level `await` must complete before + the import() Promise settles. */ +static void async_module_loader_tla(void) +{ + async_test_state st = { .mode = DELIVER_FULFILL, + .deliver_src = "export const v = await Promise.resolve(123);" }; + g_async = &st; + + JSRuntime *rt = new_runtime(); + JS_SetModuleLoaderFuncAsync(rt, NULL, async_loader, NULL, NULL); + JSContext *ctx = JS_NewContext(rt); + + run_async_eval(ctx, + "import('tla').then(m => { globalThis.__result = m.v; }," + " e => { globalThis.__result = -1; });"); + pump_jobs(rt); + + assert(st.loader_calls == 1); + assert(get_int_global(ctx, "__result") == 123); + + JS_FreeContext(ctx); + JS_FreeRuntime(rt); + g_async = NULL; +} + +/* re-entrant settle: the host may call JS_FulfillAsyncModuleLoad synchronously + from inside the loader callback (load already cached/available). */ +static void async_module_loader_reentrant(void) +{ + async_test_state st = { .mode = DELIVER_FULFILL, .reentrant = true, + .deliver_src = "export const v = 7;" }; + g_async = &st; + + JSRuntime *rt = new_runtime(); + JS_SetModuleLoaderFuncAsync(rt, NULL, async_loader, NULL, NULL); + JSContext *ctx = JS_NewContext(rt); + + run_async_eval(ctx, + "import('re').then(m => { globalThis.__result = m.v; }," + " e => { globalThis.__result = -1; });"); + pump_jobs(rt); + + assert(st.loader_calls == 1); + assert(get_int_global(ctx, "__result") == 7); + + JS_FreeContext(ctx); + JS_FreeRuntime(rt); + g_async = NULL; +} + +/* NULL module: fulfilling with a NULL JSModuleDef rejects the import() Promise + with a synthesized generic error. */ +static void async_module_loader_null_module(void) +{ + async_test_state st = { .mode = DELIVER_FULFILL_NULL }; + g_async = &st; + + JSRuntime *rt = new_runtime(); + JS_SetModuleLoaderFuncAsync(rt, NULL, async_loader, NULL, NULL); + JSContext *ctx = JS_NewContext(rt); + + run_async_eval(ctx, + "import('n').then(() => { globalThis.__err = 'resolved?!'; }," + " e => { globalThis.__err = e.message; });"); + pump_jobs(rt); + + char *err = get_string_global(ctx, "__err"); + assert(!strcmp(err, "async module load returned NULL")); + free(err); + + JS_FreeContext(ctx); + JS_FreeRuntime(rt); + g_async = NULL; +} + +/* evaluation throw: a module that throws at top level rejects import() with + the thrown error (the loader/compile succeeded; evaluation failed). */ +static void async_module_loader_eval_throw(void) +{ + async_test_state st = { .mode = DELIVER_FULFILL, + .deliver_src = "throw new Error('boom-in-module');" + "export const v = 1;" }; + g_async = &st; + + JSRuntime *rt = new_runtime(); + JS_SetModuleLoaderFuncAsync(rt, NULL, async_loader, NULL, NULL); + JSContext *ctx = JS_NewContext(rt); + + run_async_eval(ctx, + "import('t').then(() => { globalThis.__err = 'resolved?!'; }," + " e => { globalThis.__err = e.message; });"); + pump_jobs(rt); + + char *err = get_string_global(ctx, "__err"); + assert(!strcmp(err, "boom-in-module")); + free(err); + + JS_FreeContext(ctx); + JS_FreeRuntime(rt); + g_async = NULL; +} + +/* attribute check: a registered async attribute checker that rejects must + reject the import() BEFORE the loader is ever invoked. */ +static void async_module_loader_attrs_reject(void) +{ + async_test_state st = { .mode = DELIVER_FULFILL, .attrs_reject = true, + .deliver_src = "export const v = 1;" }; + g_async = &st; + + JSRuntime *rt = new_runtime(); + JS_SetModuleLoaderFuncAsync(rt, NULL, async_loader, async_check_attrs, NULL); + JSContext *ctx = JS_NewContext(rt); + + run_async_eval(ctx, + "import('x', { with: { type: 'json' } })" + " .then(() => { globalThis.__err = 'resolved?!'; }," + " e => { globalThis.__err = e.message; });"); + pump_jobs(rt); + + assert(st.attrs_calls == 1); + assert(st.loader_calls == 0); /* rejected before the loader ran */ + char *err = get_string_global(ctx, "__err"); + assert(!strcmp(err, "unsupported import attributes")); + free(err); + + JS_FreeContext(ctx); + JS_FreeRuntime(rt); + g_async = NULL; +} + +/* normalizer: a registered async normalizer must be invoked to resolve the + specifier before the loader is called. */ +static void async_module_loader_normalize(void) +{ + async_test_state st = { .mode = DELIVER_FULFILL, + .deliver_src = "export const v = 55;" }; + g_async = &st; + + JSRuntime *rt = new_runtime(); + JS_SetModuleLoaderFuncAsync(rt, async_normalize, async_loader, NULL, NULL); + JSContext *ctx = JS_NewContext(rt); + + run_async_eval(ctx, + "import('norm').then(m => { globalThis.__result = m.v; }," + " e => { globalThis.__result = -1; });"); + pump_jobs(rt); + + assert(st.normalize_calls >= 1); + assert(st.loader_calls == 1); + assert(get_int_global(ctx, "__result") == 55); + + JS_FreeContext(ctx); + JS_FreeRuntime(rt); + g_async = NULL; +} + +/* nested dynamic import: an async-loaded module's own import() also routes + through the async loader (loader called for 'outer' then 'inner'). */ +#define NESTED_Q 8 +static JSAsyncModuleLoadHandle nested_hq[NESTED_Q]; +static char *nested_nq[NESTED_Q]; +static int nested_qn; +static int nested_loader_calls; + +static JSValue nested_deliver_job(JSContext *ctx, int argc, JSValueConst *argv) +{ + (void)argc; (void)argv; + if (nested_qn == 0) + return JS_UNDEFINED; + JSAsyncModuleLoadHandle handle = nested_hq[0]; + char *name = nested_nq[0]; + for (int i = 1; i < nested_qn; i++) { + nested_hq[i-1] = nested_hq[i]; nested_nq[i-1] = nested_nq[i]; + } + nested_qn--; + const char *src = !strcmp(name, "outer") + ? "const inner = await import('inner'); export const v = inner.w + 1;" + : "export const w = 40;"; + JSValue mv = JS_Eval(ctx, src, strlen(src), name, + JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY); + assert(!JS_IsException(mv)); + JSModuleDef *m = JS_VALUE_GET_PTR(mv); + JS_FreeValue(ctx, mv); + free(name); + JS_FulfillAsyncModuleLoad(ctx, handle, m); + return JS_UNDEFINED; +} + +static void nested_loader(JSContext *ctx, const char *module_name, void *opaque, + JSValueConst attributes, JSAsyncModuleLoadHandle handle) +{ + (void)opaque; (void)attributes; + nested_loader_calls++; + assert(nested_qn < NESTED_Q); + nested_hq[nested_qn] = handle; + nested_nq[nested_qn] = strdup(module_name); + nested_qn++; + int r = JS_EnqueueJob(ctx, nested_deliver_job, 0, NULL); + assert(r == 0); +} + +static void async_module_loader_nested_dynamic(void) +{ + nested_qn = 0; nested_loader_calls = 0; + JSRuntime *rt = new_runtime(); + JS_SetModuleLoaderFuncAsync(rt, NULL, nested_loader, NULL, NULL); + JSContext *ctx = JS_NewContext(rt); + + run_async_eval(ctx, + "import('outer').then(m => { globalThis.__result = m.v; }," + " e => { globalThis.__result = -1; });"); + pump_jobs(rt); + + assert(nested_loader_calls == 2); /* outer + inner both via the async loader */ + assert(get_int_global(ctx, "__result") == 41); + + JS_FreeContext(ctx); + JS_FreeRuntime(rt); +} + +/* concurrency harness: many loads in flight at once. The loaded module bumps a + global eval counter so tests can detect double-evaluation (broken dedup). */ +#define ASYNC_Q 128 +static JSAsyncModuleLoadHandle cc_hq[ASYNC_Q]; +static char *cc_nq[ASYNC_Q]; +static int cc_qn; +static int cc_loader_calls; +static int cc_never; /* if set, never enqueue delivery (leave handles pending) */ + +static JSValue cc_deliver(JSContext *ctx, int argc, JSValueConst *argv) +{ + (void)argc; (void)argv; + if (cc_qn == 0) + return JS_UNDEFINED; + JSAsyncModuleLoadHandle h = cc_hq[0]; + char *name = cc_nq[0]; + for (int i = 1; i < cc_qn; i++) { cc_hq[i-1] = cc_hq[i]; cc_nq[i-1] = cc_nq[i]; } + cc_qn--; + static const char src[] = + "globalThis.__evals = (globalThis.__evals||0) + 1; export const v = 9;"; + JSValue mv = JS_Eval(ctx, src, strlen(src), name, + JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY); + assert(!JS_IsException(mv)); + JSModuleDef *m = JS_VALUE_GET_PTR(mv); + JS_FreeValue(ctx, mv); + free(name); + JS_FulfillAsyncModuleLoad(ctx, h, m); + return JS_UNDEFINED; +} + +static void cc_loader(JSContext *ctx, const char *module_name, void *opaque, + JSValueConst attributes, JSAsyncModuleLoadHandle handle) +{ + (void)opaque; (void)attributes; + cc_loader_calls++; + assert(cc_qn < ASYNC_Q); + cc_hq[cc_qn] = handle; + cc_nq[cc_qn] = strdup(module_name); + cc_qn++; + if (cc_never) + return; + int r = JS_EnqueueJob(ctx, cc_deliver, 0, NULL); + assert(r == 0); +} + +/* in-flight dedup: two concurrent import()s of the same specifier must invoke + the loader once, evaluate once, and resolve both to the same namespace. */ +static void async_module_loader_inflight_dedup(void) +{ + cc_qn = 0; cc_loader_calls = 0; cc_never = 0; + JSRuntime *rt = new_runtime(); + JS_SetModuleLoaderFuncAsync(rt, NULL, cc_loader, NULL, NULL); + JSContext *ctx = JS_NewContext(rt); + + run_async_eval(ctx, + "globalThis.__evals = 0; globalThis.__same = 0;" + "Promise.all([import('dup'), import('dup')]).then(([a, b]) => {" + " globalThis.__same = (a === b) ? 1 : 0; });"); + pump_jobs(rt); + + assert(cc_loader_calls == 1); /* loader invoked once */ + assert(get_int_global(ctx, "__evals") == 1); /* module evaluated once */ + assert(get_int_global(ctx, "__same") == 1); /* identical namespace */ + + JS_FreeContext(ctx); + JS_FreeRuntime(rt); +} + +/* the dual of dedup: many distinct specifiers in flight must each load and + evaluate exactly once (dedup must not be over-eager). */ +static void async_module_loader_concurrent_distinct(void) +{ + cc_qn = 0; cc_loader_calls = 0; cc_never = 0; + JSRuntime *rt = new_runtime(); + JS_SetModuleLoaderFuncAsync(rt, NULL, cc_loader, NULL, NULL); + JSContext *ctx = JS_NewContext(rt); + + run_async_eval(ctx, + "globalThis.__evals = 0; globalThis.__n = 0;" + "{ const a = []; for (let i = 0; i < 64; i++)" + " a.push(import('m' + i).then(m => { globalThis.__n += m.v; })); }"); + pump_jobs(rt); + + assert(cc_loader_calls == 64); + assert(get_int_global(ctx, "__evals") == 64); + assert(get_int_global(ctx, "__n") == 64 * 9); + + JS_FreeContext(ctx); + JS_FreeRuntime(rt); +} + +/* shutdown must reclaim a handle with multiple waiters: several concurrent + imports of one never-settled specifier share a handle JS_FreeRuntime frees. */ +static void async_module_loader_shutdown_multi_waiter(void) +{ + cc_qn = 0; cc_loader_calls = 0; cc_never = 1; + JSRuntime *rt = new_runtime(); + JS_SetModuleLoaderFuncAsync(rt, NULL, cc_loader, NULL, NULL); + JSContext *ctx = JS_NewContext(rt); + + run_async_eval(ctx, + "Promise.all([import('z'), import('z'), import('z')]).then(()=>{},()=>{});"); + pump_jobs(rt); + + assert(cc_loader_calls == 1); /* one handle, three waiters attached to it */ + for (int i = 0; i < cc_qn; i++) + free(cc_nq[i]); /* delivery never ran, so free the stashed names */ + + JS_FreeContext(ctx); + JS_FreeRuntime(rt); +} + static void runtime_cstring_free(void) { JSRuntime *rt = new_runtime(); @@ -1078,6 +1756,22 @@ int main(void) raw_context_global_var(); is_array(); module_serde(); + async_module_loader(); + async_module_loader_reject(); + async_module_loader_transitive(); + async_module_loader_cache_hit(); + async_module_loader_double_settle(); + async_module_loader_shutdown_pending(); + async_module_loader_tla(); + async_module_loader_reentrant(); + async_module_loader_null_module(); + async_module_loader_eval_throw(); + async_module_loader_attrs_reject(); + async_module_loader_normalize(); + async_module_loader_nested_dynamic(); + async_module_loader_inflight_dedup(); + async_module_loader_concurrent_distinct(); + async_module_loader_shutdown_multi_waiter(); runtime_cstring_free(); utf16_string(); weak_map_gc_check(); diff --git a/quickjs.c b/quickjs.c index 38a724fbb..9fabd62a7 100644 --- a/quickjs.c +++ b/quickjs.c @@ -337,6 +337,14 @@ struct JSRuntime { } u; JSModuleCheckSupportedImportAttributes *module_check_attrs; void *module_loader_opaque; + + /* async loader for dynamic import() (see JS_SetModuleLoaderFuncAsync) */ + JSModuleNormalizeFunc *module_normalize_func_async; /* optional, may be NULL */ + JSModuleLoaderFuncAsync *module_loader_func_async; /* NULL => not registered */ + JSModuleCheckSupportedImportAttributes *module_check_attrs_async; + void *module_loader_opaque_async; + struct list_head pending_async_loads; /* outstanding JSAsyncModuleLoadOpaque */ + /* timestamp for internal use in module evaluation */ int64_t module_async_evaluation_next_timestamp; @@ -356,6 +364,26 @@ struct JSRuntime { JSRuntimeFinalizerState *finalizers; }; +/* one import() awaiting an in-flight async load */ +typedef struct JSAsyncModuleWaiter { + JSValue resolving_funcs[2]; +} JSAsyncModuleWaiter; + +/* one outstanding async load of `name` in `ctx`. Concurrent import()s of the + same specifier share a handle (extra waiters, no new loader call), so the + module is fetched and evaluated once. Owned by rt->pending_async_loads and + freed by the JS_FreeRuntime sweep; settling releases the waiters and sets + `settled` so a second fulfill/reject is a no-op. */ +struct JSAsyncModuleLoadOpaque { + struct list_head link; /* in rt->pending_async_loads */ + JSContext *ctx; + JSAtom name; /* normalized specifier; dedup key with ctx */ + JSAsyncModuleWaiter *waiters; /* owned until settled */ + int waiters_count; + int waiters_size; + bool settled; +}; + struct JSClass { uint32_t class_id; /* 0 means free entry */ JSAtom class_name; @@ -1988,6 +2016,7 @@ JSRuntime *JS_NewRuntime2(const JSMallocFunctions *mf, void *opaque) init_list_head(&rt->string_list); #endif init_list_head(&rt->job_list); + init_list_head(&rt->pending_async_loads); if (JS_InitAtoms(rt)) goto fail; @@ -2302,6 +2331,21 @@ void JS_FreeRuntime(JSRuntime *rt) } init_list_head(&rt->job_list); + /* free async loads still in flight; don't run their reject callbacks (no JS + during teardown), just release the references they hold */ + list_for_each_safe(el, el1, &rt->pending_async_loads) { + JSAsyncModuleLoadOpaque *h = + list_entry(el, JSAsyncModuleLoadOpaque, link); + for (i = 0; i < h->waiters_count; i++) { + JS_FreeValueRT(rt, h->waiters[i].resolving_funcs[0]); + JS_FreeValueRT(rt, h->waiters[i].resolving_funcs[1]); + } + js_free_rt(rt, h->waiters); + JS_FreeAtomRT(rt, h->name); + js_free_rt(rt, h); + } + init_list_head(&rt->pending_async_loads); + JS_RunGC(rt); #ifdef ENABLE_DUMPS // JS_DUMP_LEAKS @@ -29911,6 +29955,19 @@ void JS_SetModuleNormalizeFunc2(JSRuntime *rt, rt->normalize_u.module_normalize_func2 = module_normalize; } +/* async loader for dynamic import(); see JS_SetModuleLoaderFuncAsync in quickjs.h */ +void JS_SetModuleLoaderFuncAsync(JSRuntime *rt, + JSModuleNormalizeFunc *module_normalize, + JSModuleLoaderFuncAsync *module_loader_async, + JSModuleCheckSupportedImportAttributes *module_check_attrs, + void *opaque) +{ + rt->module_normalize_func_async = module_normalize; + rt->module_loader_func_async = module_loader_async; + rt->module_check_attrs_async = module_check_attrs; + rt->module_loader_opaque_async = opaque; +} + int JS_SetModulePrivateValue(JSContext *ctx, JSModuleDef *m, JSValue val) { set_value(ctx, &m->private_value, val); @@ -30996,30 +31053,72 @@ static JSValue js_load_module_fulfilled(JSContext *ctx, JSValueConst this_val, return JS_UNDEFINED; } -static void JS_LoadModuleInternal(JSContext *ctx, const char *basename, - const char *filename, - JSValueConst *resolving_funcs, - JSValueConst attributes) +/* link + evaluate `m` once, then settle each of the `count` waiters with the + same result. Waiters' resolving_funcs and `m` are borrowed (not freed). */ +static void js_load_module_continue_n(JSContext *ctx, JSModuleDef *m, + const JSAsyncModuleWaiter *waiters, + int count) { JSValue evaluate_promise; - JSModuleDef *m; JSValue ret, err, func_obj, evaluate_resolving_funcs[2]; JSValueConst func_data[3]; - - m = js_host_resolve_imported_module(ctx, basename, filename, attributes); - if (!m) - goto fail; + int i; if (js_resolve_module(ctx, m) < 0) { js_free_modules(ctx, JS_FREE_MODULE_NOT_RESOLVED); goto fail; } - /* Evaluate the module code */ func_obj = JS_NewModuleValue(ctx, m); evaluate_promise = JS_EvalFunction(ctx, func_obj); if (JS_IsException(evaluate_promise)) { fail: + err = JS_GetException(ctx); + for (i = 0; i < count; i++) { + ret = JS_Call(ctx, waiters[i].resolving_funcs[1], JS_UNDEFINED, 1, vc(&err)); + JS_FreeValue(ctx, ret); /* XXX: what to do if exception ? */ + } + JS_FreeValue(ctx, err); + return; + } + + /* fan the evaluation out: one then-handler pair per waiter */ + for (i = 0; i < count; i++) { + func_obj = JS_NewModuleValue(ctx, m); + func_data[0] = waiters[i].resolving_funcs[0]; + func_data[1] = waiters[i].resolving_funcs[1]; + func_data[2] = func_obj; + evaluate_resolving_funcs[0] = JS_NewCFunctionData(ctx, js_load_module_fulfilled, 0, 0, 3, func_data); + evaluate_resolving_funcs[1] = JS_NewCFunctionData(ctx, js_load_module_rejected, 0, 0, 3, func_data); + JS_FreeValue(ctx, func_obj); + ret = js_promise_then(ctx, evaluate_promise, 2, vc(evaluate_resolving_funcs)); + JS_FreeValue(ctx, ret); + JS_FreeValue(ctx, evaluate_resolving_funcs[0]); + JS_FreeValue(ctx, evaluate_resolving_funcs[1]); + } + JS_FreeValue(ctx, evaluate_promise); +} + +/* single-waiter wrapper (sync loader + cache-hit paths) */ +static void js_load_module_continue(JSContext *ctx, JSModuleDef *m, + JSValueConst *resolving_funcs) +{ + JSAsyncModuleWaiter w; + w.resolving_funcs[0] = resolving_funcs[0]; + w.resolving_funcs[1] = resolving_funcs[1]; + js_load_module_continue_n(ctx, m, &w, 1); +} + +static void JS_LoadModuleInternal(JSContext *ctx, const char *basename, + const char *filename, + JSValueConst *resolving_funcs, + JSValueConst attributes) +{ + JSModuleDef *m; + JSValue ret, err; + + m = js_host_resolve_imported_module(ctx, basename, filename, attributes); + if (!m) { err = JS_GetException(ctx); ret = JS_Call(ctx, resolving_funcs[1], JS_UNDEFINED, 1, vc(&err)); JS_FreeValue(ctx, ret); /* XXX: what to do if exception ? */ @@ -31027,18 +31126,7 @@ static void JS_LoadModuleInternal(JSContext *ctx, const char *basename, return; } - func_obj = JS_NewModuleValue(ctx, m); - func_data[0] = resolving_funcs[0]; - func_data[1] = resolving_funcs[1]; - func_data[2] = func_obj; - evaluate_resolving_funcs[0] = JS_NewCFunctionData(ctx, js_load_module_fulfilled, 0, 0, 3, func_data); - evaluate_resolving_funcs[1] = JS_NewCFunctionData(ctx, js_load_module_rejected, 0, 0, 3, func_data); - JS_FreeValue(ctx, func_obj); - ret = js_promise_then(ctx, evaluate_promise, 2, vc(evaluate_resolving_funcs)); - JS_FreeValue(ctx, ret); - JS_FreeValue(ctx, evaluate_resolving_funcs[0]); - JS_FreeValue(ctx, evaluate_resolving_funcs[1]); - JS_FreeValue(ctx, evaluate_promise); + js_load_module_continue(ctx, m, resolving_funcs); } /* Return a promise or an exception in case of memory error. Used by @@ -31057,6 +31145,69 @@ JSValue JS_LoadModule(JSContext *ctx, const char *basename, return promise; } +/* add a waiter to an in-flight handle (dups resolving_funcs); -1 on OOM */ +static int js_async_module_add_waiter(JSContext *ctx, JSAsyncModuleLoadHandle h, + JSValueConst *resolving_funcs) +{ + if (h->waiters_count >= h->waiters_size) { + int new_size = h->waiters_size * 2; + JSAsyncModuleWaiter *w = js_realloc(ctx, h->waiters, + sizeof(*w) * new_size); + if (!w) + return -1; + h->waiters = w; + h->waiters_size = new_size; + } + h->waiters[h->waiters_count].resolving_funcs[0] = JS_DupValue(ctx, resolving_funcs[0]); + h->waiters[h->waiters_count].resolving_funcs[1] = JS_DupValue(ctx, resolving_funcs[1]); + h->waiters_count++; + return 0; +} + +/* allocate a handle for the first import of `name`, seeded with one waiter, + linked into rt->pending_async_loads (dups name and resolving_funcs) */ +static JSAsyncModuleLoadHandle js_alloc_async_module_load(JSContext *ctx, + JSAtom name, + JSValueConst *resolving_funcs) +{ + JSAsyncModuleLoadHandle h = js_malloc(ctx, sizeof(*h)); + if (!h) + return NULL; + h->waiters = js_malloc(ctx, sizeof(*h->waiters)); + if (!h->waiters) { + js_free(ctx, h); + return NULL; + } + h->ctx = ctx; + h->name = JS_DupAtom(ctx, name); + h->waiters_size = 1; + h->waiters_count = 1; + h->waiters[0].resolving_funcs[0] = JS_DupValue(ctx, resolving_funcs[0]); + h->waiters[0].resolving_funcs[1] = JS_DupValue(ctx, resolving_funcs[1]); + h->settled = false; + list_add_tail(&h->link, &ctx->rt->pending_async_loads); + return h; +} + +/* mark settled and release the waiters; the struct stays linked and is freed by + the JS_FreeRuntime sweep, so a second fulfill/reject is a safe no-op */ +static void js_settle_async_module_load(JSAsyncModuleLoadHandle h) +{ + JSContext *ctx = h->ctx; + int i; + h->settled = true; + for (i = 0; i < h->waiters_count; i++) { + JS_FreeValue(ctx, h->waiters[i].resolving_funcs[0]); + JS_FreeValue(ctx, h->waiters[i].resolving_funcs[1]); + } + js_free(ctx, h->waiters); + h->waiters = NULL; + h->waiters_count = 0; + h->waiters_size = 0; + JS_FreeAtom(ctx, h->name); + h->name = JS_ATOM_NULL; +} + static JSValue js_dynamic_import_job(JSContext *ctx, int argc, JSValueConst *argv) { @@ -31064,8 +31215,9 @@ static JSValue js_dynamic_import_job(JSContext *ctx, JSValueConst basename_val = argv[2]; JSValueConst specifier = argv[3]; JSValueConst attributes = argv[4]; - const char *basename = NULL, *filename; + const char *basename = NULL, *filename = NULL; JSValue ret, err; + JSRuntime *rt = ctx->rt; if (!JS_IsString(basename_val)) { JS_ThrowTypeError(ctx, "no function filename for import()"); @@ -31079,6 +31231,81 @@ static JSValue js_dynamic_import_job(JSContext *ctx, if (!filename) goto exception; + /* async loader: normalize, check attributes and short-circuit a cache hit + (mirroring js_host_resolve_imported_module), then hand off to the host */ + if (rt->module_loader_func_async) { + JSAsyncModuleLoadHandle handle; + JSModuleDef *m; + JSAtom module_name; + char *cname; + struct list_head *el; + + if (rt->module_normalize_func_async) { + cname = rt->module_normalize_func_async(ctx, basename, filename, + rt->module_loader_opaque_async); + } else { + cname = js_default_module_normalize_name(ctx, basename, filename); + } + if (!cname) + goto exception; + + if (rt->module_check_attrs_async && + rt->module_check_attrs_async(ctx, rt->module_loader_opaque_async, + attributes) < 0) { + js_free(ctx, cname); + goto exception; + } + + module_name = JS_NewAtom(ctx, cname); + if (module_name == JS_ATOM_NULL) { + js_free(ctx, cname); + goto exception; + } + + /* already loaded (prior static or dynamic import): settle from cache */ + m = js_find_loaded_module(ctx, module_name); + if (m) { + JS_FreeAtom(ctx, module_name); + js_free(ctx, cname); + js_load_module_continue(ctx, m, resolving_funcs); + JS_FreeCString(ctx, filename); + JS_FreeCString(ctx, basename); + return JS_UNDEFINED; + } + + /* in-flight dedup: an unsettled load of this name is already pending in + this context, so wait on it instead of invoking the loader again */ + list_for_each(el, &rt->pending_async_loads) { + handle = list_entry(el, JSAsyncModuleLoadOpaque, link); + if (!handle->settled && handle->ctx == ctx && + handle->name == module_name) { + int r = js_async_module_add_waiter(ctx, handle, resolving_funcs); + JS_FreeAtom(ctx, module_name); + js_free(ctx, cname); + if (r < 0) + goto exception; + JS_FreeCString(ctx, filename); + JS_FreeCString(ctx, basename); + return JS_UNDEFINED; + } + } + + handle = js_alloc_async_module_load(ctx, module_name, resolving_funcs); + JS_FreeAtom(ctx, module_name); + if (!handle) { + js_free(ctx, cname); + goto exception; + } + /* hand off the normalized name; the runtime owns the handle now */ + rt->module_loader_func_async(ctx, cname, + rt->module_loader_opaque_async, + attributes, handle); + js_free(ctx, cname); + JS_FreeCString(ctx, filename); + JS_FreeCString(ctx, basename); + return JS_UNDEFINED; + } + JS_LoadModuleInternal(ctx, basename, filename, resolving_funcs, attributes); JS_FreeCString(ctx, filename); JS_FreeCString(ctx, basename); @@ -31088,10 +31315,61 @@ static JSValue js_dynamic_import_job(JSContext *ctx, ret = JS_Call(ctx, resolving_funcs[1], JS_UNDEFINED, 1, vc(&err)); JS_FreeValue(ctx, ret); /* XXX: what to do if exception ? */ JS_FreeValue(ctx, err); + JS_FreeCString(ctx, filename); JS_FreeCString(ctx, basename); return JS_UNDEFINED; } +void JS_FulfillAsyncModuleLoad(JSContext *ctx, + JSAsyncModuleLoadHandle handle, + JSModuleDef *module) +{ + assert(ctx == handle->ctx); + if (handle->settled) /* second fulfill/reject is a no-op */ + return; + + if (!module) { + /* NULL means load failed: reject every waiter with a generic error */ + int i; + JSValue err = JS_NewError(ctx); + JS_DefinePropertyValueStr(ctx, err, "message", + JS_NewString(ctx, "async module load returned NULL"), + JS_PROP_C_W_E); + for (i = 0; i < handle->waiters_count; i++) { + JSValue ret = JS_Call(ctx, handle->waiters[i].resolving_funcs[1], + JS_UNDEFINED, 1, vc(&err)); + JS_FreeValue(ctx, ret); + } + JS_FreeValue(ctx, err); + } else { + js_load_module_continue_n(ctx, module, handle->waiters, + handle->waiters_count); + } + + js_settle_async_module_load(handle); +} + +void JS_RejectAsyncModuleLoad(JSContext *ctx, + JSAsyncModuleLoadHandle handle, + JSValue error) +{ + int i; + + assert(ctx == handle->ctx); + if (handle->settled) { /* no-op, but we still own `error` */ + JS_FreeValue(ctx, error); + return; + } + for (i = 0; i < handle->waiters_count; i++) { + JSValue ret = JS_Call(ctx, handle->waiters[i].resolving_funcs[1], + JS_UNDEFINED, 1, vc(&error)); + JS_FreeValue(ctx, ret); + } + JS_FreeValue(ctx, error); + + js_settle_async_module_load(handle); +} + static JSValue js_dynamic_import(JSContext *ctx, JSValueConst specifier, JSValueConst options) { diff --git a/quickjs.h b/quickjs.h index b9ed27560..0dafae280 100644 --- a/quickjs.h +++ b/quickjs.h @@ -1185,6 +1185,47 @@ JS_EXTERN void JS_SetModuleLoaderFunc2(JSRuntime *rt, JS_EXTERN void JS_SetModuleNormalizeFunc2(JSRuntime *rt, JSModuleNormalizeFunc2 *module_normalize); +/* Async loader for dynamic import(). Only import() routes here; static imports + still use JS_SetModuleLoaderFunc[2]. The loader receives a handle and settles + it later (e.g. after async I/O) with JS_FulfillAsyncModuleLoad or + JS_RejectAsyncModuleLoad. Settling more than once is a no-op, and a handle the + host never settles is freed by JS_FreeRuntime. Concurrent import()s of the + same normalized specifier share one load, and a cached module is settled + without calling the loader. */ + +/* opaque handle the host receives for an async load */ +typedef struct JSAsyncModuleLoadOpaque JSAsyncModuleLoadOpaque; +typedef JSAsyncModuleLoadOpaque *JSAsyncModuleLoadHandle; + +/* called when JS evaluates import(specifier); settle `handle` later */ +typedef void JSModuleLoaderFuncAsync(JSContext *ctx, + const char *module_name, + void *opaque, + JSValueConst attributes, + JSAsyncModuleLoadHandle handle); + +/* register an async dynamic-import loader; module_loader_async = NULL + unregisters. Coexists with the sync loader used for static imports. */ +JS_EXTERN void JS_SetModuleLoaderFuncAsync(JSRuntime *rt, + JSModuleNormalizeFunc *module_normalize, + JSModuleLoaderFuncAsync *module_loader_async, + JSModuleCheckSupportedImportAttributes *module_check_attrs, + void *opaque); + +/* settle the import() Promise with a loaded module. `module` is a compiled, + not yet linked/evaluated JSModuleDef (e.g. from JS_Eval with + JS_EVAL_FLAG_COMPILE_ONLY); this drives link + evaluate. NULL rejects with a + generic error. Further calls on the handle are ignored. */ +JS_EXTERN void JS_FulfillAsyncModuleLoad(JSContext *ctx, + JSAsyncModuleLoadHandle handle, + JSModuleDef *module); + +/* reject the import() Promise with `error` (ownership taken). Further calls on + the handle are ignored. */ +JS_EXTERN void JS_RejectAsyncModuleLoad(JSContext *ctx, + JSAsyncModuleLoadHandle handle, + JSValue error); + /* return the import.meta object of a module */ JS_EXTERN JSValue JS_GetImportMeta(JSContext *ctx, JSModuleDef *m); JS_EXTERN JSAtom JS_GetModuleName(JSContext *ctx, JSModuleDef *m);