[esm-integration] Add pthread support

Under ESM integration all dependencies must be satisfied at import time,
but pthreads requires that we have supply the memory via postMessage,
so the memory is, by definition, not available at import time.

On order to work around this issue we create a smaller pthread
stub/loader file that delays the import of the main program until the
initial `postMessage` has been received.  Once the memory is received we
load main program using a dynamic `import` statement.

Author: sbc100

site/source/docs/compiling/Modularized-Output.rst

@@ -163,7 +163,7 @@ This setting implicitly enables :ref:`export_es6` and sets :ref:`MODULARIZE` to
 
 Some additional limitations are:
 
-* ``-pthread`` / :ref:`wasm_workers` are not yet supported.
+* :ref:`wasm_workers` is not yet supported.
 
 * :ref:`abort_on_wasm_exceptions` is not supported (requires wrapping wasm
   exports).

src/lib/libpthread.js

@@ -29,6 +29,13 @@ const MAX_PTR = Number((2n ** 64n) - 1n);
 #else
 const MAX_PTR = (2 ** 32) - 1
 #endif
+
+#if WASM_ESM_INTEGRATION
+const pthreadWorkerScript = TARGET_BASENAME + '.pthread.mjs';
+#else
+const pthreadWorkerScript = TARGET_JS_NAME;
+#endif
+
 // Use a macro to avoid duplicating pthread worker options.
 // We cannot use a normal JS variable since the vite bundler requires that worker
 // options be inline.
@@ -282,6 +289,7 @@ var LibraryPThread = {
           message = `Pthread ${ptrToString(worker.pthread_ptr)} sent an error!`;
         }
 #endif
+        console.error(e.stack);
         err(`${message} ${e.filename}:${e.lineno}: ${e.message}`);
         throw e;
       };
@@ -295,7 +303,9 @@ var LibraryPThread = {
 
 #if ASSERTIONS
       assert(wasmMemory instanceof WebAssembly.Memory, 'WebAssembly memory should have been loaded by now!');
+#if !WASM_ESM_INTEGRATION
       assert(wasmModule instanceof WebAssembly.Module, 'WebAssembly Module should have been loaded by now!');
+#endif
 #endif
 
       // When running on a pthread, none of the incoming parameters on the module
@@ -333,7 +343,9 @@ var LibraryPThread = {
 #else // WASM2JS
         wasmMemory,
 #endif // WASM2JS
+#if !WASM_ESM_INTEGRATION
         wasmModule,
+#endif
 #if LOAD_SOURCE_MAP
         wasmSourceMap,
 #endif
@@ -391,25 +403,25 @@ var LibraryPThread = {
 #if TRUSTED_TYPES
       // Use Trusted Types compatible wrappers.
       if (typeof trustedTypes != 'undefined' && trustedTypes.createPolicy) {
-        var p = trustedTypes.createPolicy('emscripten#workerPolicy1', { createScriptURL: (ignored) => import.meta.url });
+        var p = trustedTypes.createPolicy('emscripten#workerPolicy1', { createScriptURL: (ignored) => new URL('{{{ pthreadWorkerScript }}}', import.meta.url));
         worker = new Worker(p.createScriptURL('ignored'), {{{ pthreadWorkerOptions }}});
       } else
 #endif
 #if expectToReceiveOnModule('mainScriptUrlOrBlob')
-        if (Module['mainScriptUrlOrBlob']) {
-          var pthreadMainJs = Module['mainScriptUrlOrBlob'];
-          if (typeof pthreadMainJs != 'string') {
-            pthreadMainJs = URL.createObjectURL(pthreadMainJs);
-          }
-          worker = new Worker(pthreadMainJs, {{{ pthreadWorkerOptions }}});
-        } else
+      if (Module['mainScriptUrlOrBlob']) {
+        var pthreadMainJs = Module['mainScriptUrlOrBlob'];
+        if (typeof pthreadMainJs != 'string') {
+          pthreadMainJs = URL.createObjectURL(pthreadMainJs);
+        }
+        worker = new Worker(pthreadMainJs, {{{ pthreadWorkerOptions }}});
+      } else
 #endif
       // We need to generate the URL with import.meta.url as the base URL of the JS file
       // instead of just using new URL(import.meta.url) because bundler's only recognize
       // the first case in their bundling step. The latter ends up producing an invalid
       // URL to import from the server (e.g., for webpack the file:// path).
       // See https://github.com/webpack/webpack/issues/12638
-      worker = new Worker(new URL('{{{ TARGET_JS_NAME }}}', import.meta.url), {{{ pthreadWorkerOptions }}});
+      worker = new Worker(new URL('{{{ pthreadWorkerScript }}}', import.meta.url), {{{ pthreadWorkerOptions }}});
 #else // EXPORT_ES6
       var pthreadMainJs = _scriptName;
 #if expectToReceiveOnModule('mainScriptUrlOrBlob')
@@ -601,10 +613,10 @@ var LibraryPThread = {
 
     worker.pthread_ptr = threadParams.pthread_ptr;
     var msg = {
-        cmd: 'run',
-        start_routine: threadParams.startRoutine,
-        arg: threadParams.arg,
-        pthread_ptr: threadParams.pthread_ptr,
+      cmd: 'run',
+      start_routine: threadParams.startRoutine,
+      arg: threadParams.arg,
+      pthread_ptr: threadParams.pthread_ptr,
     };
 #if OFFSCREENCANVAS_SUPPORT
     // Note that we do not need to quote these names because they are only used

src/postamble.js

@@ -6,6 +6,8 @@
 
 // === Auto-generated postamble setup entry stuff ===
 
+#include "runtime_common.js"
+
 #if PROXY_TO_WORKER
 if (ENVIRONMENT_IS_WORKER) {
 #include "webGLWorker.js'
@@ -306,6 +308,9 @@ export default async function init(moduleArg = {}) {
   Object.assign(Module, moduleArg);
   processModuleArgs();
 #if WASM_ESM_INTEGRATION
+#if PTHREADS
+  registerTLSInit(__emscripten_tls_init);
+#endif
   updateMemoryViews();
 #if DYNCALLS && '$dynCalls' in addedLibraryItems
 
@@ -318,9 +323,9 @@ export default async function init(moduleArg = {}) {
   run();
 }
 
-#if PTHREADS || WASM_WORKERS
+#if WASM_WORKERS
 // When run as a worker thread run `init` immediately.
-if ({{{ ENVIRONMENT_IS_WORKER_THREAD() }}}) await init()
+if ({{{ ENVIRONMENT_IS_WORKER_THREAD() }}}) init()
 #endif
 
 #if ENVIRONMENT_MAY_BE_NODE

src/preamble.js

@@ -101,8 +101,6 @@ function _free() {
  */
 var isFileURI = (filename) => filename.startsWith('file://');
 
-#include "runtime_common.js"
-
 #if ASSERTIONS
 assert(typeof Int32Array != 'undefined' && typeof Float64Array !== 'undefined' && Int32Array.prototype.subarray != undefined && Int32Array.prototype.set != undefined,
        'JS engine does not provide full typed array support');
@@ -144,7 +142,7 @@ function initRuntime() {
 #endif
 
 #if PTHREADS
-  if (ENVIRONMENT_IS_PTHREAD) return startWorker(Module);
+  if (ENVIRONMENT_IS_PTHREAD) return startWorker();
 #endif
 
 #if STACK_OVERFLOW_CHECK >= 2
@@ -990,7 +988,7 @@ function getWasmImports() {
 #endif // WASM_ASYNC_COMPILATION
 #endif // SOURCE_PHASE_IMPORTS
 }
-#endif
+#endif // WASM_ESM_INTEGRATION
 
 #if !WASM_BIGINT
 // Globals used by JS i64 conversions (see makeSetValue)

src/pthread_esm_startup.mjs

@@ -0,0 +1,48 @@
+/**
+ * @license
+ * Copyright 2025 The Emscripten Authors
+ * SPDX-License-Identifier: MIT
+ */
+
+// This file is used as the initial script loaded into pthread workers when
+// running in WASM_ESM_INTEGRATION mode.
+// The point of this file is to delay the loading the main program module 
+// until the wasm memory has been received via postMessage.
+
+#if RUNTIME_DEBUG
+console.log("Running pthread_esm_startup");
+#endif
+
+#if ENVIRONMENT_MAY_BE_NODE
+// Create as web-worker-like an environment as we can.
+var worker_threads = await import('worker_threads');
+global.Worker = worker_threads.Worker;
+var parentPort = worker_threads['parentPort'];
+parentPort.on('message', (msg) => global.onmessage?.({ data: msg }));
+Object.assign(globalThis, {
+  self: global,
+  postMessage: (msg) => parentPort['postMessage'](msg),
+});
+#endif
+
+self.onmessage = async (msg) => {
+#if RUNTIME_DEBUG
+  console.log('pthread_esm_startup', msg.data.cmd);
+#endif
+  if (msg.data.cmd == 'load') {
+    // Until we initialize the runtime, queue up any further incoming messages.
+    let messageQueue = [msg];
+    self.onmessage = (e) => messageQueue.push(e);
+
+    // Now that we have the wasmMemory we can import the main program
+    globalThis.wasmMemory = msg.data.wasmMemory;
+    const prog = await import('./{{{ TARGET_JS_NAME }}}');
+
+    // Dispatch queued messages the new handler installed by the main program
+    for (let msg of messageQueue) {
+      await self.onmessage(msg);
+    }
+
+    await prog.default()
+  }
+};

src/runtime_common.js

@@ -29,7 +29,7 @@ var readyPromiseResolve, readyPromiseReject;
 var wasmModuleReceived;
 #endif
 
-#if ENVIRONMENT_MAY_BE_NODE
+#if ENVIRONMENT_MAY_BE_NODE && !WASM_ESM_INTEGRATION
 if (ENVIRONMENT_IS_NODE && {{{ ENVIRONMENT_IS_WORKER_THREAD() }}}) {
   // Create as web-worker-like an environment as we can.
   var parentPort = worker_threads['parentPort'];
@@ -39,7 +39,7 @@ if (ENVIRONMENT_IS_NODE && {{{ ENVIRONMENT_IS_WORKER_THREAD() }}}) {
     postMessage: (msg) => parentPort['postMessage'](msg),
   });
 }
-#endif // ENVIRONMENT_MAY_BE_NODE
+#endif // ENVIRONMENT_MAY_BE_NODE && !WASM_ESM_INTEGRATION
 #endif
 
 #if PTHREADS

src/runtime_init_memory.js

@@ -12,6 +12,14 @@
 // check for full engine support (use string 'subarray' to avoid closure compiler confusion)
 
 function initMemory() {
+#if WASM_ESM_INTEGRATION && PTHREADS
+  if (ENVIRONMENT_IS_PTHREAD) {
+    wasmMemory = globalThis.wasmMemory;
+    assert(wasmMemory);
+    updateMemoryViews();
+  }
+#endif
+
   {{{ runIfWorkerThread('return') }}}
 
 #if expectToReceiveOnModule('wasmMemory')

src/runtime_pthread.js

@@ -45,20 +45,23 @@ if (ENVIRONMENT_IS_PTHREAD) {
       var msgData = e['data'];
       //dbg('msgData: ' + Object.keys(msgData));
       var cmd = msgData.cmd;
+#if PTHREADS_DEBUG
+        dbg('worker: got cmd:', cmd)
+#endif
       if (cmd === 'load') { // Preload command that is called once per worker to parse and load the Emscripten code.
 #if ASSERTIONS
         workerID = msgData.workerID;
 #endif
+
 #if PTHREADS_DEBUG
         dbg('worker: loading module')
 #endif
-
         // Until we initialize the runtime, queue up any further incoming messages.
         let messageQueue = [];
         self.onmessage = (e) => messageQueue.push(e);
 
         // And add a callback for when the runtime is initialized.
-        self.startWorker = (instance) => {
+        self.startWorker = () => {
           // Notify the main thread that this thread has loaded.
           postMessage({ cmd: 'loaded' });
           // Process any messages that were queued before the thread was ready.
@@ -102,8 +105,10 @@ if (ENVIRONMENT_IS_PTHREAD) {
 #endif
         }
 
+#if !WASM_ESM_INTEGRATION
         wasmMemory = msgData.wasmMemory;
         updateMemoryViews();
+#endif
 
 #if LOAD_SOURCE_MAP
         wasmSourceMap = resetPrototype(WasmSourceMap, msgData.wasmSourceMap);
@@ -112,13 +117,15 @@ if (ENVIRONMENT_IS_PTHREAD) {
         wasmOffsetConverter = resetPrototype(WasmOffsetConverter, msgData.wasmOffsetConverter);
 #endif
 
+#if !WASM_ESM_INTEGRATION
 #if MINIMAL_RUNTIME
         // Pass the shared Wasm module in the Module object for MINIMAL_RUNTIME.
         Module['wasm'] = msgData.wasmModule;
         loadModule();
 #else
         wasmModuleReceived(msgData.wasmModule);
 #endif // MINIMAL_RUNTIME
+#endif
       } else if (cmd === 'run') {
 #if ASSERTIONS
         assert(msgData.pthread_ptr);

test/common.py

@@ -683,6 +683,8 @@ def also_with_modularize(f):
   @wraps(f)
   def metafunc(self, modularize, *args, **kwargs):
     if modularize:
+      if '-sWASM_ESM_INTEGRATION':
+        self.skipTest('not compatible with WASM_ESM_INTEGRATION')
       self.emcc_args += ['--extern-post-js', test_file('modularize_post_js.js'), '-sMODULARIZE']
     f(self, *args, **kwargs)
 
@@ -1125,7 +1127,6 @@ def require_jspi(self):
       self.skipTest('JSPI is not currently supported for WASM2JS')
     if self.get_setting('WASM_ESM_INTEGRATION'):
       self.skipTest('WASM_ESM_INTEGRATION is not compatible with JSPI')
-
     if self.is_browser_test():
       if 'EMTEST_SKIP_JSPI' in os.environ:
         self.skipTest('skipping JSPI (EMTEST_SKIP_JSPI is set)')
@@ -1177,8 +1178,6 @@ def setup_node_pthreads(self):
     self.emcc_args += ['-Wno-pthreads-mem-growth', '-pthread']
     if self.get_setting('MINIMAL_RUNTIME'):
       self.skipTest('node pthreads not yet supported with MINIMAL_RUNTIME')
-    if self.get_setting('WASM_ESM_INTEGRATION'):
-      self.skipTest('pthreads not yet supported with WASM_ESM_INTEGRATION')
     nodejs = self.get_nodejs()
     self.js_engines = [nodejs]
     self.node_args += shared.node_pthread_flags(nodejs)

test/core/test_pthread_join_and_asyncify.c

@@ -3,7 +3,8 @@
 // University of Illinois/NCSA Open Source License.  Both these licenses can be
 // found in the LICENSE file.
 
-#include <emscripten.h>
+#include <emscripten/emscripten.h>
+#include <emscripten/console.h>
 #include <pthread.h>
 #include <stdio.h>
 #include <assert.h>
@@ -15,6 +16,7 @@ EM_ASYNC_JS(int, async_call, (), {
 
 void *run_thread(void *args) {
   int ret = async_call();
+  emscripten_outf("done async_call: %d", ret);
   assert(ret == 42);
   return NULL;
 }