diff --git a/eslint.config.mjs b/eslint.config.mjs index b2f4843fc4254..59a4208bc6071 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -35,6 +35,7 @@ export default [{ 'src/postamble*.js', 'src/closure-externs/', 'src/embind/', + 'src/pthread_esm_startup.mjs', 'src/emrun_postjs.js', 'src/wasm_worker.js', 'src/audio_worklet.js', diff --git a/package.json b/package.json index 6df0bb77ccd41..05594783a58dd 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ }, "scripts": { "lint": "eslint .", - "fmt": "prettier --write src/*.mjs tools/*.mjs", - "check": "prettier --check src/*.mjs tools/*.mjs" + "fmt": "prettier --write src/*.mjs tools/*.mjs --ignore-path src/pthread_esm_startup.mjs", + "check": "prettier --check src/*.mjs tools/*.mjs --ignore-path src/pthread_esm_startup.mjs" } } diff --git a/site/source/docs/compiling/Modularized-Output.rst b/site/source/docs/compiling/Modularized-Output.rst index e34bf06382a57..4067f8eda65cc 100644 --- a/site/source/docs/compiling/Modularized-Output.rst +++ b/site/source/docs/compiling/Modularized-Output.rst @@ -163,7 +163,7 @@ This setting implicitly enables :ref:`export_es6` and sets :ref:`MODULARIZE` to Some additional limitations are: -* ``-pthread`` / :ref:`wasm_workers` are not yet supported. +* :ref:`wasm_workers` is not yet supported. * :ref:`abort_on_wasm_exceptions` is not supported (requires wrapping wasm exports). diff --git a/src/lib/libpthread.js b/src/lib/libpthread.js index 6d979627efe31..e0fce6d7775f7 100644 --- a/src/lib/libpthread.js +++ b/src/lib/libpthread.js @@ -29,6 +29,13 @@ const MAX_PTR = Number((2n ** 64n) - 1n); #else const MAX_PTR = (2 ** 32) - 1 #endif + +#if WASM_ESM_INTEGRATION +const pthreadWorkerScript = TARGET_BASENAME + '.pthread.mjs'; +#else +const pthreadWorkerScript = TARGET_JS_NAME; +#endif + // Use a macro to avoid duplicating pthread worker options. // We cannot use a normal JS variable since the vite bundler requires that worker // options be inline. @@ -295,7 +302,9 @@ var LibraryPThread = { #if ASSERTIONS assert(wasmMemory instanceof WebAssembly.Memory, 'WebAssembly memory should have been loaded by now!'); +#if !WASM_ESM_INTEGRATION assert(wasmModule instanceof WebAssembly.Module, 'WebAssembly Module should have been loaded by now!'); +#endif #endif // When running on a pthread, none of the incoming parameters on the module @@ -333,7 +342,9 @@ var LibraryPThread = { #else // WASM2JS wasmMemory, #endif // WASM2JS +#if !WASM_ESM_INTEGRATION wasmModule, +#endif #if LOAD_SOURCE_MAP wasmSourceMap, #endif @@ -391,7 +402,7 @@ var LibraryPThread = { #if TRUSTED_TYPES // Use Trusted Types compatible wrappers. if (typeof trustedTypes != 'undefined' && trustedTypes.createPolicy) { - var p = trustedTypes.createPolicy('emscripten#workerPolicy1', { createScriptURL: (ignored) => import.meta.url }); + var p = trustedTypes.createPolicy('emscripten#workerPolicy1', { createScriptURL: (ignored) => new URL('{{{ pthreadWorkerScript }}}', import.meta.url) }); worker = new Worker(p.createScriptURL('ignored'), {{{ pthreadWorkerOptions }}}); } else #endif @@ -409,7 +420,7 @@ var LibraryPThread = { // the first case in their bundling step. The latter ends up producing an invalid // URL to import from the server (e.g., for webpack the file:// path). // See https://github.com/webpack/webpack/issues/12638 - worker = new Worker(new URL('{{{ TARGET_JS_NAME }}}', import.meta.url), {{{ pthreadWorkerOptions }}}); + worker = new Worker(new URL('{{{ pthreadWorkerScript }}}', import.meta.url), {{{ pthreadWorkerOptions }}}); #else // EXPORT_ES6 var pthreadMainJs = _scriptName; #if expectToReceiveOnModule('mainScriptUrlOrBlob') diff --git a/src/modularize.js b/src/modularize.js index d75eba2d97769..ccc326a150adc 100644 --- a/src/modularize.js +++ b/src/modularize.js @@ -8,7 +8,7 @@ // JS program code (INNER_JS_CODE) and wrapping it in a factory function. #if SOURCE_PHASE_IMPORTS -import source wasmModule from './{settings.WASM_BINARY_FILE}'; +import source wasmModule from './{{{ WASM_BINARY_FILE }}}'; #endif #if ENVIRONMENT_MAY_BE_WEB && !EXPORT_ES6 && !(MINIMAL_RUNTIME && !PTHREADS) diff --git a/src/postamble.js b/src/postamble.js index 7df57059ee0e4..88f1588ee14a2 100644 --- a/src/postamble.js +++ b/src/postamble.js @@ -306,6 +306,9 @@ export default async function init(moduleArg = {}) { Object.assign(Module, moduleArg); processModuleArgs(); #if WASM_ESM_INTEGRATION +#if PTHREADS + registerTLSInit(__emscripten_tls_init); +#endif updateMemoryViews(); #if DYNCALLS && '$dynCalls' in addedLibraryItems @@ -318,7 +321,7 @@ export default async function init(moduleArg = {}) { run(); } -#if PTHREADS || WASM_WORKERS +#if (WASM_WORKERS || PTHREADS) && !WASM_ESM_INTEGRATION // When run as a worker thread run `init` immediately. if ({{{ ENVIRONMENT_IS_WORKER_THREAD() }}}) await init() #endif diff --git a/src/preamble.js b/src/preamble.js index d74bd63a5f4ae..6b34051bdef31 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -990,7 +990,7 @@ function getWasmImports() { #endif // WASM_ASYNC_COMPILATION #endif // SOURCE_PHASE_IMPORTS } -#endif +#endif // WASM_ESM_INTEGRATION #if !WASM_BIGINT // Globals used by JS i64 conversions (see makeSetValue) diff --git a/src/pthread_esm_startup.mjs b/src/pthread_esm_startup.mjs new file mode 100644 index 0000000000000..f49f7141547fb --- /dev/null +++ b/src/pthread_esm_startup.mjs @@ -0,0 +1,53 @@ +/** + * @license + * Copyright 2025 The Emscripten Authors + * SPDX-License-Identifier: MIT + */ + +// This file is used as the initial script loaded into pthread workers when +// running in WASM_ESM_INTEGRATION mode. +// Tyhe point of this file is to delay the loading of the main program module +// until the wasm memory has been received via postMessage. + +#if RUNTIME_DEBUG +console.log("Running pthread_esm_startup"); +#endif + +#if ENVIRONMENT_MAY_BE_NODE +// Create as web-worker-like an environment as we can. +var worker_threads = await import('worker_threads'); +global.Worker = worker_threads.Worker; +var parentPort = worker_threads['parentPort']; +parentPort.on('message', (msg) => global.onmessage?.({ data: msg })); +Object.assign(globalThis, { + self: global, + postMessage: (msg) => parentPort['postMessage'](msg), +}); +#endif + +self.onmessage = async (msg) => { +#if RUNTIME_DEBUG + console.log('pthread_esm_startup', msg.data.cmd); +#endif + if (msg.data.cmd == 'load') { + // Until we initialize the runtime, queue up any further incoming messages + // that can arrive while the async import (await import below) is happening. + // For examples the `run` message often arrives right away before the import + // is complete. + let messageQueue = [msg]; + self.onmessage = (e) => messageQueue.push(e); + + // Now that we have the wasmMemory we can import the main program + globalThis.wasmMemory = msg.data.wasmMemory; + const prog = await import('./{{{ TARGET_JS_NAME }}}'); + + // Now that the import is completed the main program will have installed + // its own `onmessage` handler and replaced our handler. + // Now we can dispatch any queued messages to this new handler. + for (let msg of messageQueue) { + await self.onmessage(msg); + } + + await prog.default() + } +}; diff --git a/src/runtime_common.js b/src/runtime_common.js index a664cdb0393a0..724a7b5b88599 100644 --- a/src/runtime_common.js +++ b/src/runtime_common.js @@ -29,7 +29,7 @@ var readyPromiseResolve, readyPromiseReject; var wasmModuleReceived; #endif -#if ENVIRONMENT_MAY_BE_NODE +#if ENVIRONMENT_MAY_BE_NODE && !WASM_ESM_INTEGRATION if (ENVIRONMENT_IS_NODE && {{{ ENVIRONMENT_IS_WORKER_THREAD() }}}) { // Create as web-worker-like an environment as we can. var parentPort = worker_threads['parentPort']; @@ -39,7 +39,7 @@ if (ENVIRONMENT_IS_NODE && {{{ ENVIRONMENT_IS_WORKER_THREAD() }}}) { postMessage: (msg) => parentPort['postMessage'](msg), }); } -#endif // ENVIRONMENT_MAY_BE_NODE +#endif // ENVIRONMENT_MAY_BE_NODE && !WASM_ESM_INTEGRATION #endif #if PTHREADS diff --git a/src/runtime_init_memory.js b/src/runtime_init_memory.js index 549df2ff8051b..5cf26d7b154a3 100644 --- a/src/runtime_init_memory.js +++ b/src/runtime_init_memory.js @@ -12,6 +12,14 @@ // check for full engine support (use string 'subarray' to avoid closure compiler confusion) function initMemory() { +#if WASM_ESM_INTEGRATION && PTHREADS + if (ENVIRONMENT_IS_PTHREAD) { + wasmMemory = globalThis.wasmMemory; + assert(wasmMemory); + updateMemoryViews(); + } +#endif + {{{ runIfWorkerThread('return') }}} #if expectToReceiveOnModule('wasmMemory') diff --git a/src/runtime_pthread.js b/src/runtime_pthread.js index 4d7316c9ea459..5ec47971f13f4 100644 --- a/src/runtime_pthread.js +++ b/src/runtime_pthread.js @@ -104,8 +104,10 @@ if (ENVIRONMENT_IS_PTHREAD) { #endif } +#if !WASM_ESM_INTEGRATION wasmMemory = msgData.wasmMemory; updateMemoryViews(); +#endif #if LOAD_SOURCE_MAP wasmSourceMap = resetPrototype(WasmSourceMap, msgData.wasmSourceMap); @@ -114,6 +116,7 @@ if (ENVIRONMENT_IS_PTHREAD) { wasmOffsetConverter = resetPrototype(WasmOffsetConverter, msgData.wasmOffsetConverter); #endif +#if !WASM_ESM_INTEGRATION #if MINIMAL_RUNTIME // Pass the shared Wasm module in the Module object for MINIMAL_RUNTIME. Module['wasm'] = msgData.wasmModule; @@ -121,6 +124,7 @@ if (ENVIRONMENT_IS_PTHREAD) { #else wasmModuleReceived(msgData.wasmModule); #endif // MINIMAL_RUNTIME +#endif } else if (cmd === 'run') { #if ASSERTIONS assert(msgData.pthread_ptr); diff --git a/test/common.py b/test/common.py index 3136b9c544df4..b3a6641598dca 100644 --- a/test/common.py +++ b/test/common.py @@ -683,6 +683,8 @@ def also_with_modularize(f): @wraps(f) def metafunc(self, modularize, *args, **kwargs): if modularize: + if '-sWASM_ESM_INTEGRATION': + self.skipTest('also_with_modularize is not compatible with WASM_ESM_INTEGRATION') self.emcc_args += ['--extern-post-js', test_file('modularize_post_js.js'), '-sMODULARIZE'] f(self, *args, **kwargs) @@ -1177,8 +1179,6 @@ def setup_node_pthreads(self): self.emcc_args += ['-Wno-pthreads-mem-growth', '-pthread'] if self.get_setting('MINIMAL_RUNTIME'): self.skipTest('node pthreads not yet supported with MINIMAL_RUNTIME') - if self.get_setting('WASM_ESM_INTEGRATION'): - self.skipTest('pthreads not yet supported with WASM_ESM_INTEGRATION') nodejs = self.get_nodejs() self.js_engines = [nodejs] self.node_args += shared.node_pthread_flags(nodejs) diff --git a/test/test_core.py b/test/test_core.py index 154d4bebf38bb..b1fa831e6dca0 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -7761,6 +7761,7 @@ def test_webidl(self, mode, allow_memory_growth): # PTHREAD_POOL_DELAY_LOAD=1 adds a pthreadPoolReady promise that users # can wait on for pthread initialization. @node_pthreads + @no_esm_integration('WASM_ESM_INTEGRATION is not compatible with WASM_ASYNC_COMPILATION') def test_embind_sync_if_pthread_delayed(self): self.set_setting('WASM_ASYNC_COMPILATION', 0) self.set_setting('PTHREAD_POOL_DELAY_LOAD', 1) @@ -9283,6 +9284,7 @@ def test_pthread_unhandledrejection(self): @node_pthreads @no_wasm2js('wasm2js does not support PROXY_TO_PTHREAD (custom section support)') @also_with_modularize + @no_esm_integration('USE_OFFSET_CONVERTER') def test_pthread_offset_converter(self): self.set_setting('PROXY_TO_PTHREAD') self.set_setting('EXIT_RUNTIME') @@ -9703,6 +9705,7 @@ def test_emscripten_async_load_script(self): @no_sanitize('sanitizers do not support WASM_WORKERS') @also_with_minimal_runtime @also_with_modularize + @no_esm_integration('WASM_ESM_INTEGRATION is not compatible with WASM_WORKERS') def test_wasm_worker_hello(self): if self.is_wasm2js() and '-sMODULARIZE' in self.emcc_args: self.skipTest('WASM2JS + MODULARIZE + WASM_WORKERS is not supported') @@ -9711,11 +9714,13 @@ def test_wasm_worker_hello(self): @node_pthreads @no_sanitize('sanitizers do not support WASM_WORKERS') + @no_esm_integration('WASM_ESM_INTEGRATION is not compatible with WASM_WORKERS') def test_wasm_worker_malloc(self): self.do_run_in_out_file_test('wasm_worker/malloc_wasm_worker.c', emcc_args=['-sWASM_WORKERS']) @node_pthreads @no_sanitize('sanitizers do not support WASM_WORKERS') + @no_esm_integration('WASM_ESM_INTEGRATION is not compatible with WASM_WORKERS') def test_wasm_worker_wait_async(self): self.do_runf('atomic/test_wait_async.c', emcc_args=['-sWASM_WORKERS']) diff --git a/tools/link.py b/tools/link.py index b13b05937cd46..1c796af4f4134 100644 --- a/tools/link.py +++ b/tools/link.py @@ -810,8 +810,8 @@ def phase_linker_setup(options, linker_args): # noqa: C901, PLR0912, PLR0915 exit_with_error('WASM_ESM_INTEGRATION is not compatible with dynamic linking') if settings.ASYNCIFY: exit_with_error('WASM_ESM_INTEGRATION is not compatible with -sASYNCIFY') - if settings.WASM_WORKERS or settings.PTHREADS: - exit_with_error('WASM_ESM_INTEGRATION is not compatible with multi-threading') + if settings.WASM_WORKERS: + exit_with_error('WASM_ESM_INTEGRATION is not compatible with WASM_WORKERS') if settings.USE_OFFSET_CONVERTER: exit_with_error('WASM_ESM_INTEGRATION is not compatible with USE_OFFSET_CONVERTER') if not settings.WASM_ASYNC_COMPILATION: @@ -2233,6 +2233,11 @@ def phase_final_emitting(options, target, js_target, wasm_target): support_target = unsuffixed(js_target) + '.support.mjs' move_file(final_js, support_target) create_esm_wrapper(js_target, support_target, wasm_target) + if settings.PTHREADS: + support_target = unsuffixed(js_target) + '.pthread.mjs' + pthread_code = building.read_and_preprocess(utils.path_from_root('src/pthread_esm_startup.mjs'), expand_macros=True) + write_file(support_target, pthread_code) + fix_es6_import_statements(support_target) else: move_file(final_js, js_target)