Deploy

bundle.js

@@ -16343,33 +16343,22 @@ function isAllowed(mask, i) {
 }
 
 // ts/chtml.ts
-var errElt;
-var progressElt;
-var _rootElt;
-function rootElt() {
-  if (!_rootElt) {
-    _rootElt = document.getElementById("app") || document.body;
-  }
-  return _rootElt;
-}
 function elt(id) {
   const r = document.getElementById(id);
   if (!r) throw new Error(`element ${id} not found`);
   return r;
 }
+function setMessage(id, msg) {
+  id = "msg-" + id;
+  elt(id).style.display = msg ? "block" : "none";
+  if (typeof msg == "object") elt(id).innerHTML = msg.html;
+  else elt(id).textContent = msg;
+}
 function setError(msg) {
-  if (!errElt) {
-    errElt = div("error");
-    rootElt().prepend(errElt);
-  }
-  errElt.textContent = msg;
+  setMessage("error", msg);
 }
 function setProgress(msg) {
-  if (!progressElt) {
-    progressElt = div("progress");
-    rootElt().prepend(progressElt);
-  }
-  progressElt.textContent = msg;
+  setMessage("progress", msg);
 }
 function btn(lbl, cls, onclick) {
   const b = document.createElement("button");
@@ -16382,11 +16371,6 @@ function btn(lbl, cls, onclick) {
   };
   return b;
 }
-function div(cls = "") {
-  const d = document.createElement("div");
-  d.className = cls;
-  return d;
-}
 function text(t, elt2 = "span") {
   const [tag, cls] = elt2.split(".", 2);
   const s = document.createElement(tag);
@@ -16412,6 +16396,10 @@ var WaiSequence = class extends Generation {
   }
   tokens;
   ptr = 0;
+  ffTokens = 0;
+  sampledTokens = 0;
+  genTime = 0;
+  prefillTime = 0;
   async run() {
     if (this.started) throw new Error("Already started");
     this.started = true;
@@ -16425,6 +16413,10 @@ var WaiSequence = class extends Generation {
     this.seq.destroy();
   }
   async advance() {
+    const t0 = performance.now();
+    const isPrefill = this.ffTokens == 0;
+    this.ffTokens += this.tokens.length - this.ptr;
+    this.sampledTokens += 1;
     const adv = this.seq.advance(this.tokens.slice(this.ptr), 0);
     this.ptr = this.tokens.length;
     const mask = this.ll.samplingMask();
@@ -16442,8 +16434,22 @@ var WaiSequence = class extends Generation {
     for (const res of this.ll.getResults()) {
       this.handleParserOutput(res);
     }
+    if (isPrefill) {
+      this.prefillTime += performance.now() - t0;
+    } else {
+      this.genTime += performance.now() - t0;
+    }
     return !r.stop;
   }
+  getStats() {
+    const saved = Math.max(0, this.ffTokens - this.sampledTokens);
+    const seconds = this.genTime / 1e3;
+    const tps = (this.sampledTokens - 1) / seconds;
+    const prefill = this.prefillTime / 1e3;
+    return `Tokens: ${this.sampledTokens} + ${saved} saved, generation: ${tps.toFixed(
+      1
+    )} t/s; prefill ${prefill.toFixed(3)} s`;
+  }
 };
 var WaiModel = class {
   constructor(ai) {
@@ -16474,6 +16480,9 @@ var WaiModel = class {
       }
     );
   }
+  getStats() {
+    return this.ai._engine.pipeline.curRoundRuntimeStatsText();
+  }
   async generation(options) {
     await this.init();
     const c = this.config.new_constraint(
@@ -16537,12 +16546,37 @@ async function generate() {
       append(elt("output"), e);
     };
     await seq.run();
+    elt("stats").textContent = seq.getStats();
   } finally {
     seq.destroy();
   }
 }
+async function checkWebGPU() {
+  const webgpuReport = "Please visit <a href='https://webgpureport.org/'>webgpureport.org</a> to check your system's compatibility.";
+  if (typeof navigator !== "undefined" && navigator.gpu !== void 0) {
+    const adapter = await navigator.gpu.requestAdapter({
+      powerPreference: "high-performance"
+    });
+    if (adapter == null) {
+      setError({
+        html: "Unable to find a compatible GPU. " + webgpuReport
+      });
+      return false;
+    }
+  } else {
+    setError({
+      html: "WebGPU not supported. " + webgpuReport
+    });
+    return false;
+  }
+  return true;
+}
 async function main() {
   loadExample(examples[0]);
+  setError("");
+  setProgress("");
+  if (!await checkWebGPU()) return;
+  setProgress("Press 'Generate' to download model and generate text.");
   for (const ex of examples) {
     const t = ex;
     append(
@@ -16584,6 +16618,11 @@ return grm\`
     name: "Math",
     user: "Let's do some math!",
     grammar: "return grm`2 + 2 = ${gen(/[0-9]+/)}! and 3 + 3 = ${gen(/[0-9]+/)}!`"
+  },
+  {
+    name: "Poem",
+    user: "Write a poem about shouting",
+    grammar: "return grm`${gen(/[a-z \\n]+/, { stop: 'the end', temperature: 0.8 })}`"
   }
 ];
 export {

index.html

@@ -9,6 +9,10 @@
   </head>
   <body>
     <div id="app">
+      <div id="messages">
+        <div id="msg-error"></div>
+        <div id="msg-progress"></div>
+      </div>
       <h1>llguidance-wasm demo</h1>
       <p>
         This website will load phi-3 mini model (2.1GB), store it in browser

style.css

@@ -1,40 +1,57 @@
 body {
-    margin: 20px auto;
-    max-width: 1024px;
-    position: relative;
-    min-height: 100vh;
+  margin: 20px auto;
+  max-width: 1024px;
+  position: relative;
+  min-height: 100vh;
 }
 
-.error {
-    color: red;
+#messages {
+  position: fixed;
+  top: 5px;
 }
 
-.progress {
-    background-color: #6aa1cb;
-    padding: 0.5em;
+#messages > div {
+  margin: 0.5em 0;
+  padding: 0.5em;
+  border-radius: 5px;
+}
+
+#msg-error {
+  background-color: #e58686;
+  border: 1px solid rgb(255, 67, 67);
+}
+
+#msg-progress {
+  background-color: #8fccfa;
+  border: 1px solid rgb(68, 68, 255);
 }
 
 #grammar {
-    font-family: monospace;
+  font-family: monospace;
 }
 
 .pure-control-group {
-    margin: 1em 0;
+  margin: 1em 0;
 }
 
-#output, #stats {
-    margin: 1em 0;
-    padding: 1em;
-    border: 1px solid #ccc;
-    background-color: #f9f9f9;
-    white-space: pre-wrap;
+#app {
+  margin-top: 80px;
+}
+
+#output,
+#stats {
+  margin: 1em 0;
+  padding: 1em;
+  border: 1px solid #ccc;
+  background-color: #f9f9f9;
+  white-space: pre-wrap;
 }
 
 #examples button {
-    display: inline-block;
-    margin: 0.5em 0.5em;
+  display: inline-block;
+  margin: 0.5em 0.5em;
 }
 
 span.generated {
-    background-color: #9dff1c;
-}
+  background-color: #9dff1c;
+}