WASM test analyzer (#4043)

* wasm * WASM test scanner * Update Makefile * Update Makefile * Configurable heap limit * slightly better error * Update js_parser.zig * Update path.test.js * Update node.mjs --------- Co-authored-by: Jarred Sumner <[email protected]>
oven-sh · Aug 8, 2023 · f2f2277 · f2f2277
1 parent 0b183be
commit f2f2277
Show file tree

Hide file tree

Showing 25 changed files with 966 additions and 170 deletions.
diff --git a/Makefile b/Makefile
@@ -703,44 +703,44 @@ dev-build-obj-wasm:
 
 .PHONY: dev-wasm
 dev-wasm: dev-build-obj-wasm
-	emcc -sEXPORTED_FUNCTIONS="['_bun_free', '_cycleStart', '_cycleEnd', '_bun_malloc', '_scan', '_transform', '_init']" \
-		-g -s ERROR_ON_UNDEFINED_SYMBOLS=0  -DNDEBUG  \
+	emcc -sEXPORTED_FUNCTIONS="['_bun_free', '_cycleStart', '_cycleEnd', '_bun_malloc', '_scan', '_transform', '_init', '_getTests']" \
+		-g2 -s ERROR_ON_UNDEFINED_SYMBOLS=0  -DNDEBUG  \
 		$(BUN_DEPS_DIR)/libmimalloc.a.wasm  \
-		packages/debug-bun-freestanding-wasm32/bun-wasm.o $(OPTIMIZATION_LEVEL) --no-entry --allow-undefined  -s ASSERTIONS=0  -s ALLOW_MEMORY_GROWTH=1 -s WASM_BIGINT=1  \
+		packages/debug-bun-freestanding-wasm32/bun-wasm.o --no-entry --allow-undefined  -s ASSERTIONS=0  -s ALLOW_MEMORY_GROWTH=1 -s WASM_BIGINT=1  \
 		-o packages/debug-bun-freestanding-wasm32/bun-wasm.wasm
-	cp packages/debug-bun-freestanding-wasm32/bun-wasm.wasm src/api/demo/public/bun-wasm.wasm
+	cp packages/debug-bun-freestanding-wasm32/bun-wasm.wasm packages/bun-wasm/bun.wasm
 
 .PHONY: build-obj-wasm
 build-obj-wasm:
 	$(ZIG) build bun-wasm -Doptimize=ReleaseFast -Dtarget=wasm32-freestanding
-	emcc -sEXPORTED_FUNCTIONS="['_bun_free', '_cycleStart', '_cycleEnd', '_bun_malloc', '_scan', '_transform', '_init']" \
-		-g -s ERROR_ON_UNDEFINED_SYMBOLS=0  -DNDEBUG  \
+	emcc -sEXPORTED_FUNCTIONS="['_bun_free', '_cycleStart', '_cycleEnd', '_bun_malloc', '_scan', '_transform', '_init', '_getTests']" \
+		-s ERROR_ON_UNDEFINED_SYMBOLS=0  -DNDEBUG  \
 		$(BUN_DEPS_DIR)/libmimalloc.a.wasm  \
 		packages/bun-freestanding-wasm32/bun-wasm.o $(OPTIMIZATION_LEVEL) --no-entry --allow-undefined  -s ASSERTIONS=0  -s ALLOW_MEMORY_GROWTH=1 -s WASM_BIGINT=1  \
 		-o packages/bun-freestanding-wasm32/bun-wasm.wasm
-	cp packages/bun-freestanding-wasm32/bun-wasm.wasm src/api/demo/public/bun-wasm.wasm
+	cp packages/bun-freestanding-wasm32/bun-wasm.wasm packages/bun-wasm/bun.wasm
 
 .PHONY: build-obj-wasm-small
 build-obj-wasm-small:
-	$(ZIG) build bun-wasm -Doptimize=ReleaseSmall -Dtarget=wasm32-freestanding
-	emcc -sEXPORTED_FUNCTIONS="['_bun_free', '_cycleStart', '_cycleEnd', '_bun_malloc', '_scan', '_transform', '_init']" \
-		-g -s ERROR_ON_UNDEFINED_SYMBOLS=0  -DNDEBUG  \
+	$(ZIG) build bun-wasm -Doptimize=ReleaseFast -Dtarget=wasm32-freestanding
+	emcc -sEXPORTED_FUNCTIONS="['_bun_free', '_cycleStart', '_cycleEnd', '_bun_malloc', '_scan', '_transform', '_init', '_getTests']" \
+		-Oz -s ERROR_ON_UNDEFINED_SYMBOLS=0  -DNDEBUG  \
 		$(BUN_DEPS_DIR)/libmimalloc.a.wasm  \
 		packages/bun-freestanding-wasm32/bun-wasm.o -Oz --no-entry --allow-undefined  -s ASSERTIONS=0  -s ALLOW_MEMORY_GROWTH=1 -s WASM_BIGINT=1  \
 		-o packages/bun-freestanding-wasm32/bun-wasm.wasm
-	cp packages/bun-freestanding-wasm32/bun-wasm.wasm src/api/demo/public/bun-wasm.wasm
+	cp packages/bun-freestanding-wasm32/bun-wasm.wasm packages/bun-wasm/bun.wasm
 
 .PHONY: wasm
-wasm: api build-obj-wasm-small
+wasm: api mimalloc-wasm build-obj-wasm-small
 	@rm -rf packages/bun-wasm/*.{d.ts,js,wasm,cjs,mjs,tsbuildinfo}
 	@cp packages/bun-freestanding-wasm32/bun-wasm.wasm packages/bun-wasm/bun.wasm
 	@cp src/api/schema.d.ts packages/bun-wasm/schema.d.ts
 	@cp src/api/schema.js packages/bun-wasm/schema.js
 	@cd packages/bun-wasm && $(NPM_CLIENT) run tsc -- -p .
-	@$(ESBUILD) --sourcemap=external --external:fs --define:process.env.NODE_ENV='"production"' --outdir=packages/bun-wasm --target=esnext --bundle packages/bun-wasm/index.ts --format=esm --minify 2> /dev/null
+	@bun build --sourcemap=external --external=fs --outdir=packages/bun-wasm --target=browser --minify ./packages/bun-wasm/index.ts
 	@mv packages/bun-wasm/index.js packages/bun-wasm/index.mjs
 	@mv packages/bun-wasm/index.js.map packages/bun-wasm/index.mjs.map
-	@$(ESBUILD) --sourcemap=external --external:fs --define:process.env.NODE_ENV='"production"' --outdir=packages/bun-wasm --target=esnext --bundle packages/bun-wasm/index.ts --format=cjs --minify --platform=node 2> /dev/null
+	@$(ESBUILD) --sourcemap=external --external:fs --outdir=packages/bun-wasm --target=esnext --bundle packages/bun-wasm/index.ts --format=cjs --minify --platform=node 2> /dev/null
 	@mv packages/bun-wasm/index.js packages/bun-wasm/index.cjs
 	@mv packages/bun-wasm/index.js.map packages/bun-wasm/index.cjs.map
 	@rm -rf packages/bun-wasm/*.tsbuildinfo
@@ -1379,7 +1379,8 @@ mimalloc:
 
 
 mimalloc-wasm:
-	cd $(BUN_DEPS_DIR)/mimalloc; emcmake cmake -DMI_BUILD_SHARED=OFF -DMI_BUILD_STATIC=ON -DMI_BUILD_TESTS=OFF -DMI_BUILD_OBJECT=ON ${MIMALLOC_OVERRIDE_FLAG} -DMI_USE_CXX=ON .; emmake make;
+	rm -rf $(BUN_DEPS_DIR)/mimalloc/CMakeCache* $(BUN_DEPS_DIR)/mimalloc/CMakeFiles
+	cd $(BUN_DEPS_DIR)/mimalloc; emcmake cmake -DMI_BUILD_SHARED=OFF -DMI_BUILD_STATIC=ON -DMI_BUILD_TESTS=OFF -GNinja -DMI_BUILD_OBJECT=ON ${MIMALLOC_OVERRIDE_FLAG} -DMI_USE_CXX=OFF .; emmake cmake --build .;
 	cp $(BUN_DEPS_DIR)/mimalloc/$(MIMALLOC_INPUT_PATH) $(BUN_DEPS_OUT_DIR)/$(MIMALLOC_FILE).wasm
 
 # alias for link, incase anyone still types that

diff --git a/build.zig b/build.zig
@@ -182,7 +182,7 @@ pub fn build(b: *Build) !void {
     is_debug_build = optimize == OptimizeMode.Debug;
     const bun_executable_name = if (optimize == std.builtin.OptimizeMode.Debug) "bun-debug" else "bun";
     const root_src = if (target.getOsTag() == std.Target.Os.Tag.freestanding)
-        "src/main_wasm.zig"
+        "root_wasm.zig"
     else
         "root.zig";
 
@@ -322,7 +322,7 @@ pub fn build(b: *Build) !void {
         const wasm = b.step("bun-wasm", "Build WASM");
         var wasm_step = b.addStaticLibrary(.{
             .name = "bun-wasm",
-            .root_source_file = FileSource.relative("src/main_wasm.zig"),
+            .root_source_file = FileSource.relative("root_wasm.zig"),
             .target = target,
             .optimize = optimize,
         });
@@ -332,6 +332,8 @@ pub fn build(b: *Build) !void {
         // wasm_step.link_emit_relocs = true;
         // wasm_step.single_threaded = true;
         try configureObjectStep(b, wasm_step, @TypeOf(target), target, obj.main_pkg_path.?);
+        var build_opts = default_build_options;
+        wasm_step.addOptions("build_options", build_opts.step(b));
     }
 
     {

diff --git a/packages/bun-wasm/index.ts b/packages/bun-wasm/index.ts
@@ -1,11 +1,14 @@
 // @ts-nocheck
 import { ByteBuffer } from "peechy/bb";
 import {
+  Loader as BunLoader,
+  TestKind,
+  decodeGetTestsResponse,
   decodeScanResult,
   decodeTransformResponse,
+  encodeGetTestsRequest,
   encodeScan,
   encodeTransform,
-  Loader as BunLoader,
   type ScanResult,
   type TransformResponse,
 } from "./schema";
@@ -15,9 +18,13 @@ export enum Loader {
   tsx = BunLoader.tsx,
   ts = BunLoader.ts,
 }
-
+const testKindMap = {
+  [TestKind.describe_fn]: "describe",
+  [TestKind.test_fn]: "test",
+};
+const capturedErrors = [];
+let captureErrors = false;
 export type { ScanResult, TransformResponse };
-
 function normalizeLoader(file_name: string, loader?: Loader): BunLoader {
   return (
     (loader
@@ -82,12 +89,22 @@ var scratch2: Uint8Array;
 
 const env = {
   console_log(slice: number) {
+    const text = Bun._wasmPtrLenToString(slice);
+    if (captureErrors) {
+      capturedErrors.push(text);
+      return;
+    }
     //@ts-ignore
-    console.log(Bun._wasmPtrLenToString(slice));
+    console.log(text);
   },
   console_error(slice: number) {
     //@ts-ignore
-    console.error(Bun._wasmPtrLenToString(slice));
+    const text = Bun._wasmPtrLenToString(slice);
+    if (captureErrors) {
+      capturedErrors.push(text);
+      return;
+    }
+    console.error(text);
   },
   console_warn(slice: number) {
     //@ts-ignore
@@ -148,7 +165,6 @@ const env = {
   },
   emscripten_notify_memory_growth() {},
 };
-
 export class Bun {
   private static has_initialized = false;
   // @ts-ignore-line
@@ -179,63 +195,115 @@ export class Bun {
     return Bun._decoder.decode(region);
   }
 
-  static async init(url, fetch = globalThis.fetch) {
-    // globalThis.sucraseTransform = sucraseTransform;
+  static async init(url, heapSize = 64_000_000, fetch = globalThis.fetch) {
     scratch = new Uint8Array(8096);
 
     if (Bun.has_initialized) {
       return;
     }
-
-    if (globalThis?.WebAssembly?.instantiateStreaming) {
-      Bun.wasm_source = await globalThis.WebAssembly.instantiateStreaming(
-        fetch(url),
-        { env: env, wasi_snapshot_preview1: Wasi },
-      );
-    } else if (typeof window !== "undefined") {
-      const resp = await fetch(url);
-      Bun.wasm_source = await globalThis.WebAssembly.instantiate(
-        await resp.arrayBuffer(),
-        {
+    if (typeof process === "undefined") {
+      if (globalThis?.WebAssembly?.instantiateStreaming) {
+        Bun.wasm_source = await globalThis.WebAssembly.instantiateStreaming(fetch(url), {
           env: env,
           wasi_snapshot_preview1: Wasi,
-        },
-      );
-      // is it node?
+        });
+      } else if (typeof window !== "undefined") {
+        const resp = await fetch(url);
+        Bun.wasm_source = await globalThis.WebAssembly.instantiate(await resp.arrayBuffer(), {
+          env: env,
+          wasi_snapshot_preview1: Wasi,
+        });
+        // is it node?
+      }
     } else {
       //@ts-ignore
       const fs = await import("fs");
 
-      Bun.wasm_source = await globalThis.WebAssembly.instantiate(
-        fs.readFileSync(url),
-        {
-          env: env,
-          wasi_snapshot_preview1: Wasi,
-        },
-      );
+      Bun.wasm_source = await globalThis.WebAssembly.instantiate(fs.readFileSync(url), {
+        env: env,
+        wasi_snapshot_preview1: Wasi,
+      });
     }
 
-    const res = Bun.wasm_exports.init();
+    const res = Bun.wasm_exports.init(heapSize);
+
     if (res < 0) {
-      throw `[Bun] Failed to initialize WASM module: code ${res}`;
+      throw new Error(`[Bun] Failed to initialize WASM module: code ${res}`);
     }
 
     Bun.has_initialized = true;
   }
 
-  static transformSync(
-    content: Uint8Array | string,
-    file_name: string,
-    loader?: Loader,
-  ): TransformResponse {
-    if (!Bun.has_initialized) {
-      throw "Please run await Bun.init(wasm_url) before using this.";
+  static getTests(content: Uint8Array | string, filename = "my.test.tsx") {
+    const bb = new ByteBuffer(scratch);
+    bb.length = 0;
+    bb.index = 0;
+    const contents_buffer = content;
+
+    encodeGetTestsRequest(
+      {
+        contents: contents_buffer,
+        path: filename,
+      },
+      bb,
+    );
+
+    const data = bb.toUint8Array();
+
+    const input_ptr = Bun.wasm_exports.bun_malloc(data.length);
+    var buffer = Bun._wasmPtrToSlice(input_ptr);
+    buffer.set(data);
+    captureErrors = true;
+    try {
+      var resp_ptr = Bun.wasm_exports.getTests(input_ptr);
+    } catch (e) {
+      throw e;
+    } finally {
+      captureErrors = false;
+      Bun.wasm_exports.bun_free(input_ptr);
+    }
+
+    if (Number(resp_ptr) === 0) {
+      if (capturedErrors.length) {
+        const err = capturedErrors.slice();
+        capturedErrors.length = 0;
+        throw new Error(err.join("\n").trim());
+      }
+
+      throw new Error("Failed to parse");
+    }
+
+    if (capturedErrors.length) {
+      Bun.wasm_exports.bun_free(resp_ptr);
+      const err = capturedErrors.slice();
+      capturedErrors.length = 0;
+      throw new Error(err.join("\n").trim());
+    }
+
+    var _bb = new ByteBuffer(Bun._wasmPtrToSlice(resp_ptr));
+
+    const response = decodeGetTestsResponse(_bb);
+    var tests = new Array(response.tests.length);
+
+    for (var i = 0; i < response.tests.length; i++) {
+      tests[i] = {
+        name: new TextDecoder().decode(
+          response.contents.subarray(
+            response.tests[i].label.offset,
+            response.tests[i].label.offset + response.tests[i].label.length,
+          ),
+        ),
+        byteOffset: response.tests[i].byteOffset,
+        kind: testKindMap[response.tests[i].kind],
+      };
     }
 
-    // if (process.env.NODE_ENV === "development") {
-    //   console.time("[Bun] Transform " + file_name);
-    // }
+    Bun.wasm_exports.bun_free(resp_ptr);
+
+    return tests;
+  }
 
+  static transformSync(content: Uint8Array | string, file_name: string, loader?: Loader): TransformResponse {
     const bb = new ByteBuffer(scratch);
     bb.length = 0;
     bb.index = 0;
@@ -274,28 +342,14 @@ export class Bun {
     buffer.set(data);
 
     const resp_ptr = Bun.wasm_exports.transform(input_ptr);
-
     var _bb = new ByteBuffer(Bun._wasmPtrToSlice(resp_ptr));
-
     const response = decodeTransformResponse(_bb);
     Bun.wasm_exports.bun_free(input_ptr);
     scratch = bb.data;
     return response;
   }
 
-  static scan(
-    content: Uint8Array | string,
-    file_name: string,
-    loader?: Loader,
-  ): ScanResult {
-    if (!Bun.has_initialized) {
-      throw "Please run await Bun.init(wasm_url) before using this.";
-    }
-
-    // if (process.env.NODE_ENV === "development") {
-    //   console.time("[Bun] Transform " + file_name);
-    // }
-    scratch.fill(0);
+  static scan(content: Uint8Array | string, file_name: string, loader?: Loader): ScanResult {
     const bb = new ByteBuffer(scratch);
     bb.length = 0;
     bb.index = 0;
@@ -337,15 +391,5 @@ export class Bun {
 export const transformSync = Bun.transformSync;
 export const scan = Bun.scan;
 export const init = Bun.init;
+export const getTests = Bun.getTests;
 export default Bun;
-
-if ("window" in globalThis && !("Bun" in globalThis)) {
-  // @ts-ignore-line
-  globalThis.Bun = Bun;
-}
-
-//@ts-ignore
-if (process.env.NODE_ENV === "development") {
-  //@ts-ignore
-  Bun.env = env;
-}
diff --git a/packages/bun-wasm/test/node.mjs b/packages/bun-wasm/test/node.mjs
@@ -1,28 +1,7 @@
-// This is not released yet because there are some memory bugs with the WASM build
-// It causes syntax errors which don't make any sense
-// Most likely, this is an issue with some code expecting 64 bit pointers
-// That's also why this test just prints instead of using a test runner
-import * as Bun from "../index.mjs";
+import { readFileSync } from "fs";
+import { init, getTests } from "../index.mjs";
 
-await Bun.init(new URL("../bun.wasm", import.meta.url));
+const buf = (process.argv.length > 2 ? readFileSync(process.argv.at(-1)) : "") || readFileSync(import.meta.url);
+await init(new URL("../bun.wasm", import.meta.url));
 
-const buf =
-  (process.argv.length > 2 ? process.argv.at(-1) : "") ||
-  new TextEncoder().encode(`
-
-export function hi() {
-    return  <div>Hey</div>;
-}
-
-`);
-const result = Bun.transformSync(buf, "hi.jsx", "jsx");
-if (result.errors?.length) {
-  console.log(JSON.stringify(result.errors, null, 2));
-  throw new Error("Failed");
-}
-
-if (!result.files.length) {
-  throw new Error("unexpectedly empty");
-}
-
-process.stdout.write(result.files[0].data);
+console.log(getTests(buf));
diff --git a/root.zig b/root.zig
@@ -17,3 +17,6 @@ pub const completions = struct {
     pub const zsh = @embedFile("./completions/bun.zsh");
     pub const fish = @embedFile("./completions/bun.fish");
 };
+
+pub const JavaScriptCore = @import("./src/jsc.zig");
+pub const C = @import("./src/c.zig");