From e900fb7771d81568cb764cd611a64d1db63ca474 Mon Sep 17 00:00:00 2001
From: Taras Mankovski <74687+taras@users.noreply.github.com>
Date: Mon, 6 Jul 2026 13:45:25 -0400
Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=9A=91=20report=20failing=20url=20ins?=
=?UTF-8?q?tead=20of=20crashing=20on=20download=20errors?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
A thrown exception during a download (e.g. a gzip/transport decode error,
an HTML parse error, or a filesystem write error) escaped uncaught and
crashed the whole run with a raw stack trace that gave no indication of
which url was being fetched.
Route thrown exceptions into the existing collected-error path so the run
keeps going, writes every page/asset that succeeded, reports each failing
url with its referrer and the underlying reason, and exits non-zero at the
end. Also attach the failing url to sitemap read/parse errors.
---
deno.json | 2 +-
deno.lock | 7 +-
downloader.ts | 145 +++++++++++++++++++++++----------------
main.ts | 6 +-
progress.ts | 7 +-
staticalize.ts | 20 ++++--
test/staticalize.test.ts | 27 ++++++++
7 files changed, 146 insertions(+), 68 deletions(-)
diff --git a/deno.json b/deno.json
index 5dedac7..897957c 100644
--- a/deno.json
+++ b/deno.json
@@ -1,6 +1,6 @@
{
"name": "@frontside/staticalize",
- "version": "0.2.2",
+ "version": "0.2.6",
"exports": {
".": "./mod.ts",
"./cli": "./main.ts"
diff --git a/deno.lock b/deno.lock
index 543c0fe..02699d7 100644
--- a/deno.lock
+++ b/deno.lock
@@ -29,6 +29,7 @@
"jsr:@ts-morph/common@0.27": "0.27.0",
"npm:@effectionx/context-api@~0.5.3": "0.5.3_effection@4.0.2",
"npm:@effectionx/stream-helpers@~0.8.2": "0.8.2_effection@4.0.2",
+ "npm:clayterm@0.6": "0.6.0",
"npm:configliere@~0.2.3": "0.2.3",
"npm:effection@^4.0.2": "4.0.2",
"npm:hast-util-from-html@^2.0.3": "2.0.3",
@@ -211,7 +212,8 @@
"integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q=="
},
"@ungap/structured-clone@1.3.0": {
- "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g=="
+ "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==",
+ "deprecated": true
},
"bcp-47-match@2.0.3": {
"integrity": "sha512-JtTezzbAibu8G0R9op9zb3vcWZd9JF6M0xOYGPn0fNCd7wOpRB1mU2mH9T8gaBGbAAyIIVgB2G7xG0GP98zMAQ=="
@@ -228,6 +230,9 @@
"character-entities-legacy@3.0.0": {
"integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ=="
},
+ "clayterm@0.6.0": {
+ "integrity": "sha512-r6L6o/QRuQP5Ee2S2NAB+0nbJSZDY0mGvuWoLHvT/ioBqiu87rB5pRv7BUHV0qHDRY40qoXrx/g8+B1tceqo8A=="
+ },
"comma-separated-tokens@2.0.3": {
"integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg=="
},
diff --git a/downloader.ts b/downloader.ts
index a9def54..1377ab1 100644
--- a/downloader.ts
+++ b/downloader.ts
@@ -25,7 +25,7 @@ export interface DownloaderOptions {
export type DownloadResult =
| { ok: true; bytes: number }
- | { ok: false; url: string; referrer: string };
+ | { ok: false; url: string; referrer: string; error?: string };
export const DownloadApi = createApi("@staticalize/download", {
*download(
@@ -38,79 +38,104 @@ export const DownloadApi = createApi("@staticalize/download", {
let signal = yield* useAbortSignal();
let path = normalize(join(outdir, source.pathname));
- let response = yield* until(fetch(source.toString(), { signal }));
- if (response.ok) {
- if (response.headers.get("Content-Type")?.includes("html")) {
- let destpath = join(path, "index.html");
- let content = yield* until(response.text());
- let html = fromHtml(content);
-
- let links = selectAll("link[href]", html);
-
- for (let link of links) {
- let href = link.properties.href as string;
- yield* downloader.download(href, source);
-
- // replace self-referencing absolute urls with the destination site
- if (href.startsWith(host.origin)) {
- let url = new URL(href);
- url.host = base.host;
- url.port = base.port;
- url.protocol = base.protocol;
- link.properties.href = url.href;
+ try {
+ let response = yield* until(fetch(source.toString(), { signal }));
+ if (response.ok) {
+ if (response.headers.get("Content-Type")?.includes("html")) {
+ let destpath = join(path, "index.html");
+ let content = yield* until(response.text());
+ let html = fromHtml(content);
+
+ let links = selectAll("link[href]", html);
+
+ for (let link of links) {
+ let href = link.properties.href as string;
+ yield* downloader.download(href, source);
+
+ // replace self-referencing absolute urls with the destination site
+ if (href.startsWith(host.origin)) {
+ let url = new URL(href);
+ url.host = base.host;
+ url.port = base.port;
+ url.protocol = base.protocol;
+ link.properties.href = url.href;
+ }
}
- }
- let assets = selectAll("[src]", html);
+ let assets = selectAll("[src]", html);
- for (let element of assets) {
- let src = element.properties.src as string;
- yield* downloader.download(src, source);
+ for (let element of assets) {
+ let src = element.properties.src as string;
+ yield* downloader.download(src, source);
- // replace self-referencing absolute urls with the destination site
- if (src.startsWith(host.origin)) {
- let url = new URL(src);
- url.host = base.host;
- url.port = base.port;
- url.protocol = base.protocol;
- element.properties.src = url.href;
+ // replace self-referencing absolute urls with the destination site
+ if (src.startsWith(host.origin)) {
+ let url = new URL(src);
+ url.host = base.host;
+ url.port = base.port;
+ url.protocol = base.protocol;
+ element.properties.src = url.href;
+ }
}
- }
- let withContents = selectAll("[content]", html);
- for (let element of withContents) {
- let attr = String(element.properties.content);
- if (attr.startsWith(host.origin)) {
- yield* downloader.download(attr, source);
- let url = new URL(attr);
- url.host = base.host;
- url.port = base.port;
- url.protocol = base.protocol;
- element.properties.content = url.href;
+ let withContents = selectAll("[content]", html);
+ for (let element of withContents) {
+ let attr = String(element.properties.content);
+ if (attr.startsWith(host.origin)) {
+ yield* downloader.download(attr, source);
+ let url = new URL(attr);
+ url.host = base.host;
+ url.port = base.port;
+ url.protocol = base.protocol;
+ element.properties.content = url.href;
+ }
}
- }
- let output = toHtml(html);
- yield* call(async () => {
- let destdir = dirname(destpath);
- await ensureDir(destdir);
- await Deno.writeTextFile(destpath, output);
- });
- return { ok: true, bytes: new TextEncoder().encode(output).byteLength };
+ let output = toHtml(html);
+ yield* call(async () => {
+ let destdir = dirname(destpath);
+ await ensureDir(destdir);
+ await Deno.writeTextFile(destpath, output);
+ });
+ return {
+ ok: true,
+ bytes: new TextEncoder().encode(output).byteLength,
+ };
+ } else {
+ let size = Number(response.headers.get("Content-Length") ?? 0);
+ yield* call(async () => {
+ let destdir = dirname(path);
+ await ensureDir(destdir);
+ await Deno.writeFile(path, response.body!);
+ });
+ return { ok: true, bytes: size };
+ }
} else {
- let size = Number(response.headers.get("Content-Length") ?? 0);
- yield* call(async () => {
- let destdir = dirname(path);
- await ensureDir(destdir);
- await Deno.writeFile(path, response.body!);
- });
- return { ok: true, bytes: size };
+ return {
+ ok: false,
+ url: source.toString(),
+ referrer: referrer.toString(),
+ error: `GET ${source} ${response.status} ${response.statusText}`,
+ };
+ }
+ } catch (error) {
+ // A thrown exception here (e.g. a gzip/transport decode error, an HTML
+ // parse error, or a filesystem write error) would otherwise escape
+ // uncaught and crash the whole run with no indication of which URL was
+ // being downloaded. Route it into the collected-error path instead, so
+ // the run continues, every failing URL is reported, and the output
+ // directory still gets every page/asset that succeeded.
+ if (signal.aborted) {
+ // The run is being torn down — preserve cancellation semantics.
+ throw error;
}
- } else {
return {
ok: false,
url: source.toString(),
referrer: referrer.toString(),
+ error: error instanceof Error
+ ? `${error.name}: ${error.message}`
+ : String(error),
};
}
},
diff --git a/main.ts b/main.ts
index e623141..ac460bb 100644
--- a/main.ts
+++ b/main.ts
@@ -44,7 +44,11 @@ await main(function* (args) {
console.error(`\n${errors.length} failed downloads:\n`);
for (let error of errors) {
console.error(` ${error.url}`);
- console.error(` referrer: ${error.referrer}\n`);
+ console.error(` referrer: ${error.referrer}`);
+ if (error.error) {
+ console.error(` error: ${error.error}`);
+ }
+ console.error("");
}
yield* exit(1);
}
diff --git a/progress.ts b/progress.ts
index 1d6543c..e00873d 100644
--- a/progress.ts
+++ b/progress.ts
@@ -7,6 +7,7 @@ import type { Staticalizer } from "./staticalize.ts";
export interface DownloadError {
url: string;
referrer: string;
+ error?: string;
}
export interface ProgressResult {
@@ -66,7 +67,11 @@ export function* withProgress(
}
} else {
model.errors++;
- errors.push({ url: result.url, referrer: result.referrer });
+ errors.push({
+ url: result.url,
+ referrer: result.referrer,
+ error: result.error,
+ });
}
yield* render(progress(model));
return result;
diff --git a/staticalize.ts b/staticalize.ts
index 259118d..2778759 100644
--- a/staticalize.ts
+++ b/staticalize.ts
@@ -40,10 +40,22 @@ export function useStaticalizer(
error.name = `SitemapError`;
throw error;
}
- let text = await response.text();
- let xml = parse(text, {
- flatten: { attributes: false, empty: false, text: true },
- }) as unknown as SitemapXML;
+ let xml: SitemapXML;
+ try {
+ let text = await response.text();
+ xml = parse(text, {
+ flatten: { attributes: false, empty: false, text: true },
+ }) as unknown as SitemapXML;
+ } catch (cause) {
+ // Reading or parsing the sitemap body can fail on its own (e.g. a
+ // gzip/transport decode error, or malformed XML); attach the URL so the
+ // failure is traceable rather than a bare stack trace.
+ let error = new Error(`GET ${url} could not be read as a sitemap`, {
+ cause,
+ });
+ error.name = `SitemapError`;
+ throw error;
+ }
let entries = xml.urlset.url ?? xml.urlset.urls ?? [];
let list = Array.isArray(entries) ? entries : [entries];
diff --git a/test/staticalize.test.ts b/test/staticalize.test.ts
index a316ae1..8954170 100644
--- a/test/staticalize.test.ts
+++ b/test/staticalize.test.ts
@@ -138,6 +138,33 @@ describe("staticalize", () => {
);
});
+ it("keeps going and does not crash when a page fails to download", async () => {
+ app.get("/", (c) => c.html("Index
"));
+ // advertises gzip but the body is not valid gzip — decoding this response
+ // throws `Invalid gzip header`, which used to escape uncaught and crash the
+ // whole run with no indication of which url was at fault.
+ app.get("/bad", (c) =>
+ c.body("this is definitely not gzip", 200, {
+ "Content-Type": "text/html",
+ "Content-Encoding": "gzip",
+ }));
+ app.get("/good", (c) => c.html("Good
"));
+ app.get(...sitemap(["/", "/bad", "/good"]));
+
+ // the run resolves rather than rejecting with the decode error
+ await staticalize({
+ host,
+ base: new URL("https://frontside.com"),
+ dir: "test/dist",
+ });
+
+ // the healthy pages were still written to disk
+ await expect(exists("test/dist/index.html")).resolves.toEqual(true);
+ await expect(exists("test/dist/good/index.html")).resolves.toEqual(true);
+ // the page that could not be decoded was skipped, not written
+ await expect(exists("test/dist/bad/index.html")).resolves.toEqual(false);
+ });
+
it("does not download assets that are in a different domain", async () => {
app.get("/spa", (c) =>
c.html(`
From a345a5c606f45ebeba6a76714a9d89af0160a325 Mon Sep 17 00:00:00 2001
From: Taras Mankovski <74687+taras@users.noreply.github.com>
Date: Mon, 6 Jul 2026 14:03:28 -0400
Subject: [PATCH 2/3] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20model=20download=20fai?=
=?UTF-8?q?lures=20as=20Error=20with=20cause=20chain?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Address review: carry a real Error (not a string) on failed downloads,
wrap the underlying error as `cause`, and print the full cause hierarchy
in the summary. Drop the unreachable signal.aborted re-throw guard —
effection unwinds a halt via generator return, so cancellation never
reaches this catch.
---
downloader.ts | 19 ++++++++-----------
main.ts | 8 ++++++--
progress.ts | 2 +-
3 files changed, 15 insertions(+), 14 deletions(-)
diff --git a/downloader.ts b/downloader.ts
index 1377ab1..95d087f 100644
--- a/downloader.ts
+++ b/downloader.ts
@@ -25,7 +25,7 @@ export interface DownloaderOptions {
export type DownloadResult =
| { ok: true; bytes: number }
- | { ok: false; url: string; referrer: string; error?: string };
+ | { ok: false; url: string; referrer: string; error: Error };
export const DownloadApi = createApi("@staticalize/download", {
*download(
@@ -115,27 +115,24 @@ export const DownloadApi = createApi("@staticalize/download", {
ok: false,
url: source.toString(),
referrer: referrer.toString(),
- error: `GET ${source} ${response.status} ${response.statusText}`,
+ error: new Error(
+ `GET ${source} responded ${response.status} ${response.statusText}`,
+ ),
};
}
- } catch (error) {
+ } catch (cause) {
// A thrown exception here (e.g. a gzip/transport decode error, an HTML
// parse error, or a filesystem write error) would otherwise escape
// uncaught and crash the whole run with no indication of which URL was
// being downloaded. Route it into the collected-error path instead, so
// the run continues, every failing URL is reported, and the output
- // directory still gets every page/asset that succeeded.
- if (signal.aborted) {
- // The run is being torn down — preserve cancellation semantics.
- throw error;
- }
+ // directory still gets every page/asset that succeeded. The underlying
+ // error is preserved as `cause` for the reported hierarchy.
return {
ok: false,
url: source.toString(),
referrer: referrer.toString(),
- error: error instanceof Error
- ? `${error.name}: ${error.message}`
- : String(error),
+ error: new Error(`could not download ${source}`, { cause }),
};
}
},
diff --git a/main.ts b/main.ts
index ac460bb..85a6509 100644
--- a/main.ts
+++ b/main.ts
@@ -45,8 +45,12 @@ await main(function* (args) {
for (let error of errors) {
console.error(` ${error.url}`);
console.error(` referrer: ${error.referrer}`);
- if (error.error) {
- console.error(` error: ${error.error}`);
+ let cause: unknown = error.error;
+ let indent = " ";
+ while (cause instanceof Error) {
+ console.error(`${indent}${cause.name}: ${cause.message}`);
+ cause = cause.cause;
+ indent += " ";
}
console.error("");
}
diff --git a/progress.ts b/progress.ts
index e00873d..a5642da 100644
--- a/progress.ts
+++ b/progress.ts
@@ -7,7 +7,7 @@ import type { Staticalizer } from "./staticalize.ts";
export interface DownloadError {
url: string;
referrer: string;
- error?: string;
+ error: Error;
}
export interface ProgressResult {
From d82bf18dd5e31e1ea00184b5249ad7240f6c4300 Mon Sep 17 00:00:00 2001
From: Taras Mankovski <74687+taras@users.noreply.github.com>
Date: Tue, 7 Jul 2026 16:07:43 -0400
Subject: [PATCH 3/3] =?UTF-8?q?=E2=9C=A8=20add=20--strict=20option=20to=20?=
=?UTF-8?q?fail=20fast=20on=20download=20errors?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
By default a failed download is collected and reported at the end while
the run continues. Add a --strict flag, threaded through useStaticalizer,
that re-throws on the first download failure so the run aborts.
---
README.md | 20 +++++++++++++-------
config.ts | 5 +++++
downloader.ts | 40 ++++++++++++++++++++--------------------
main.ts | 3 ++-
staticalize.ts | 5 +++--
test/staticalize.test.ts | 22 ++++++++++++++++++++++
6 files changed, 65 insertions(+), 30 deletions(-)
diff --git a/README.md b/README.md
index b3d49ad..dcb685d 100644
--- a/README.md
+++ b/README.md
@@ -28,16 +28,22 @@ file server running at `frontside.com`.
### CLI
```
-Usage: staticalize [options]
+Usage: staticalize [OPTIONS]
-Create a static version of a website by traversing a dynamically evaluated sitemap.xml
+Arguments:
+ URL of the website to staticalize. E.g. http://localhost:8000
Options:
- -h, --help Show help
- -V, --version Show version
- -s, --site URL of the website to staticalize. E.g. http://localhost:8000 [required]
- -o, --output Directory to place the downloaded site (default: "dist")
- --base Base URL of the public website. E.g. http://frontside.com [required]
+ --output