Allow full html-build runs at new endpoint

Dockerfile

@@ -1,12 +1,21 @@
 FROM node:18.18.0-bookworm-slim
 RUN apt-get update && \
-    apt-get install --yes --no-install-recommends p7zip-full && \
+    apt-get install --yes --no-install-recommends \
+    ca-certificates curl p7zip-full python3 python3-pip pipx && \
     rm -rf /var/lib/apt/lists/*
 
+ENV PIPX_HOME /opt/pipx
+ENV PIPX_BIN_DIR /usr/bin
+RUN pipx install bs-highlighter
+
 COPY --from=ghcr.io/whatwg/wattsi:latest /whatwg/wattsi/bin/wattsi /bin/wattsi
 
 WORKDIR /app
 
+COPY --from=ghcr.io/whatwg/html-build:latest /whatwg/html-build/build.sh /whatwg/html-build/lint.sh ./
+COPY --from=ghcr.io/whatwg/html-build:latest /bin/html-build /bin/
+COPY --from=ghcr.io/whatwg/html-build:latest /whatwg/html-build/entities ./entities/
+
 COPY . .
 
 RUN npm install --omit=dev

README.md

@@ -1,8 +1,8 @@
 # build.whatwg.org
 
-This app is a build server to allow you to run [Wattsi](https://github.com/whatwg/wattsi) without having to actually install it locally. Which is really useful, since not everyone has a Free Pascal compiler lying around.
+This app is a build server to allow you to run [html-build](https://github.com/whatwg/html-build) and [Wattsi](https://github.com/whatwg/wattsi) without having to actually install many dependencies locally.
 
-Currently it is hosted on build.whatwg.org.Currently it is hosted on build.whatwg.org.
+Currently it is hosted on build.whatwg.org.
 
 ## Endpoints
 
@@ -24,6 +24,32 @@ If the resulting status code is 200, the result will be a ZIP file containing th
 
 The response will have a header, `Exit-Code`, which gives the exit code of Wattsi. This will always be `0` for a 200 OK response, but a 400 Bad Request could give a variety of different values, depending on how Wattsi failed.
 
+### `/html-build`
+
+The `/html-build` endpoint accepts POSTs with the following request body fields:
+
+- `html`, a ZIP file, containing your local checkout of [whatwg/html](https://github.com/whatwg/html). We recommend excluding the unneeded `.git/` and `review-drafts/` directories; there are other unneeded files, but those are the large ones.
+
+   ```sh
+   zip -r html.zip . --exclude .\* review-drafts/\*
+   ```
+
+- `sha`, a string, the Git commit hash of the whatwg/html repository
+
+You can send the following query string parameters, which correspond to the same-named html-build options:
+
+- `no-update`
+- `no-lint`
+- `no-highlight`
+- `single-page`
+- `fast`
+- `quiet`
+- `verbose`
+
+If the resulting status code is 200, the result will be a ZIP file containing the output, as well as an `output.txt` containing the stdout/stderr output. If the resulting status code is 400, the body text will be the error message.
+
+The response will have a header, `Exit-Code`, which gives the exit code of html-build. This will always be `0` for a 200 OK response, but a 400 Bad Request could give a variety of different values, depending on how html-build failed.
+
 ### `/version`
 
 This endpoint responds to GET requests so you can check to see if the server is working. It returns a `text/plain` response of the latest-deployed Git commit SHA.
@@ -34,7 +60,12 @@ This server requires the following to run:
 
 - [Node.js](https://nodejs.org/) 18.17.1 or later
 - [7zip](http://www.7-zip.org/) in your path as `7za`
-- And, of course, [Wattsi](https://github.com/whatwg/wattsi), in your `$PATH` as `wattsi`
+- [Wattsi](https://github.com/whatwg/wattsi), in your `$PATH` as `wattsi`
+- Several files from [html-build](https://github.com/whatwg/html-build) in your `$PATH`:
+  - `build.sh`
+  - `lint.sh`
+  - `entities/`
+  - `html-build`, the executable built from the Rust preprocessor portions
 
 It will expose itself on the port given by the `$PORT` environment variable.
 

lib/app.js

@@ -77,6 +77,68 @@ router.post("/wattsi", bodyParser, async ctx => {
   }
 });
 
+router.post("/html-build", bodyParser, async ctx => {
+  const args = booleanArgsFromQuery(
+    ctx.request.query,
+    ["no-update", "no-lint", "no-highlight", "single-page", "fast", "quiet", "verbose"]
+  );
+
+  const sha = ctx.request.body.sha || "(sha not provided)";
+
+  const htmlZipPath = ctx.request.files.html?.filepath ?? ctx.throw(400, "Expected a html file");
+
+  const htmlDirectory = newTempDirectoryName();
+  const outDirectory = newTempDirectoryName();
+  const cacheDirectory = newTempDirectoryName();
+  await mkdir(outDirectory, { recursive: true });
+
+  const env = {
+    HTML_SOURCE: htmlDirectory,
+    HTML_OUTPUT: outDirectory,
+    HTML_CACHE: cacheDirectory,
+    SHA_OVERRIDE: sha,
+    SKIP_BUILD_UPDATE_CHECK: true,
+    PROCESS_WITH_RUST: true
+  };
+
+  try {
+    await execFile("7za", ["x", htmlZipPath, `-o${htmlDirectory}`]);
+
+    try {
+      console.log(`Running build.sh ${args.join(" ")}`);
+      const result = await promisedSpawnWhileCapturingOutput("build.sh", args, { env, shell: "bash" });
+
+      const outputFile = path.join(outDirectory, "output.txt");
+      await writeFile(outputFile, result, { encoding: "utf-8" });
+      console.log(`  build.sh succeeded`);
+    } catch (e) {
+      const errorBody = e.output ?? e.stack;
+      console.log(`  build.sh or file-writing failed:`);
+      console.log(errorBody);
+      const headers = typeof e.code === "number" ? { "Exit-Code": e.code } : {};
+      ctx.throw(400, errorBody, { headers });
+    }
+
+    ctx.response.set("Exit-Code", "0");
+    const zipFilePath = `${outDirectory}.zip`;
+    console.log(`  zipping result`);
+    await execFile("7za", ["a", "-tzip", "-r", zipFilePath, `${outDirectory}/*`]);
+    console.log(`  zipping succeeded`);
+
+    ctx.response.type = "application/zip";
+    ctx.response.body = createReadStream(zipFilePath);
+
+    finished(ctx, () => rm(zipFilePath));
+  } finally {
+    await cleanupLoggingRejections([
+      ...requestFinalRemovalPromises(ctx.request),
+      rm(outDirectory, { recursive: true }),
+      rm(cacheDirectory, { recursive: true }),
+      rm(htmlDirectory, { recursive: true })
+    ]);
+  }
+});
+
 app
   .use(router.routes())
   .use(router.allowedMethods())