diff --git a/README.md b/README.md index b3d49ad..dcb685d 100644 --- a/README.md +++ b/README.md @@ -28,16 +28,22 @@ file server running at `frontside.com`. ### CLI ``` -Usage: staticalize [options] +Usage: staticalize [OPTIONS] -Create a static version of a website by traversing a dynamically evaluated sitemap.xml +Arguments: + URL of the website to staticalize. E.g. http://localhost:8000 Options: - -h, --help Show help - -V, --version Show version - -s, --site URL of the website to staticalize. E.g. http://localhost:8000 [required] - -o, --output Directory to place the downloaded site (default: "dist") - --base Base URL of the public website. E.g. http://frontside.com [required] + --output Directory to place the downloaded site [default: dist] + --base Base URL of the public website. E.g. http://frontside.com + --strict Fail on the first download error instead of collecting all failures and continuing [default: false] + -h, --help show help + -v, --version show version ``` +By default, staticalize downloads as much of the site as it can: any page or +asset that fails is reported at the end and the process exits non-zero, but the +run continues so you get every page that _did_ work. Pass `--strict` to fail +fast and abort the whole run on the first download error. + [sitemap]: https://sitemaps.org diff --git a/config.ts b/config.ts index 7afe430..5156cd9 100644 --- a/config.ts +++ b/config.ts @@ -24,5 +24,10 @@ export const config = program({ description: "Base URL of the public website. E.g. http://frontside.com", ...field(url()), }, + strict: { + description: + "Fail on the first download error instead of collecting all failures and continuing", + ...field(z.boolean(), field.default(false)), + }, }), }); diff --git a/deno.json b/deno.json index 5dedac7..897957c 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "@frontside/staticalize", - "version": "0.2.2", + "version": "0.2.6", "exports": { ".": "./mod.ts", "./cli": "./main.ts" diff --git a/deno.lock b/deno.lock index 543c0fe..02699d7 100644 --- a/deno.lock +++ b/deno.lock @@ -29,6 +29,7 @@ "jsr:@ts-morph/common@0.27": "0.27.0", "npm:@effectionx/context-api@~0.5.3": "0.5.3_effection@4.0.2", "npm:@effectionx/stream-helpers@~0.8.2": "0.8.2_effection@4.0.2", + "npm:clayterm@0.6": "0.6.0", "npm:configliere@~0.2.3": "0.2.3", "npm:effection@^4.0.2": "4.0.2", "npm:hast-util-from-html@^2.0.3": "2.0.3", @@ -211,7 +212,8 @@ "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==" }, "@ungap/structured-clone@1.3.0": { - "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==" + "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", + "deprecated": true }, "bcp-47-match@2.0.3": { "integrity": "sha512-JtTezzbAibu8G0R9op9zb3vcWZd9JF6M0xOYGPn0fNCd7wOpRB1mU2mH9T8gaBGbAAyIIVgB2G7xG0GP98zMAQ==" @@ -228,6 +230,9 @@ "character-entities-legacy@3.0.0": { "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==" }, + "clayterm@0.6.0": { + "integrity": "sha512-r6L6o/QRuQP5Ee2S2NAB+0nbJSZDY0mGvuWoLHvT/ioBqiu87rB5pRv7BUHV0qHDRY40qoXrx/g8+B1tceqo8A==" + }, "comma-separated-tokens@2.0.3": { "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==" }, diff --git a/downloader.ts b/downloader.ts index a9def54..9b517cd 100644 --- a/downloader.ts +++ b/downloader.ts @@ -21,11 +21,12 @@ export interface DownloaderOptions { host: URL; base: URL; outdir: string; + strict?: boolean; } export type DownloadResult = | { ok: true; bytes: number } - | { ok: false; url: string; referrer: string }; + | { ok: false; url: string; referrer: string; error: Error }; export const DownloadApi = createApi("@staticalize/download", { *download( @@ -34,84 +35,105 @@ export const DownloadApi = createApi("@staticalize/download", { source: URL, referrer: URL, ): Operation { - let { host, base, outdir } = opts; + let { host, base, outdir, strict } = opts; let signal = yield* useAbortSignal(); let path = normalize(join(outdir, source.pathname)); - let response = yield* until(fetch(source.toString(), { signal })); - if (response.ok) { - if (response.headers.get("Content-Type")?.includes("html")) { - let destpath = join(path, "index.html"); - let content = yield* until(response.text()); - let html = fromHtml(content); - - let links = selectAll("link[href]", html); - - for (let link of links) { - let href = link.properties.href as string; - yield* downloader.download(href, source); - - // replace self-referencing absolute urls with the destination site - if (href.startsWith(host.origin)) { - let url = new URL(href); - url.host = base.host; - url.port = base.port; - url.protocol = base.protocol; - link.properties.href = url.href; + let fail = (error: Error): DownloadResult => { + if (strict) { + throw error; + } + return { + ok: false, + url: source.toString(), + referrer: referrer.toString(), + error, + }; + }; + + try { + let response = yield* until(fetch(source.toString(), { signal })); + if (response.ok) { + if (response.headers.get("Content-Type")?.includes("html")) { + let destpath = join(path, "index.html"); + let content = yield* until(response.text()); + let html = fromHtml(content); + + let links = selectAll("link[href]", html); + + for (let link of links) { + let href = link.properties.href as string; + yield* downloader.download(href, source); + + // replace self-referencing absolute urls with the destination site + if (href.startsWith(host.origin)) { + let url = new URL(href); + url.host = base.host; + url.port = base.port; + url.protocol = base.protocol; + link.properties.href = url.href; + } } - } - let assets = selectAll("[src]", html); + let assets = selectAll("[src]", html); - for (let element of assets) { - let src = element.properties.src as string; - yield* downloader.download(src, source); + for (let element of assets) { + let src = element.properties.src as string; + yield* downloader.download(src, source); - // replace self-referencing absolute urls with the destination site - if (src.startsWith(host.origin)) { - let url = new URL(src); - url.host = base.host; - url.port = base.port; - url.protocol = base.protocol; - element.properties.src = url.href; + // replace self-referencing absolute urls with the destination site + if (src.startsWith(host.origin)) { + let url = new URL(src); + url.host = base.host; + url.port = base.port; + url.protocol = base.protocol; + element.properties.src = url.href; + } } - } - let withContents = selectAll("[content]", html); - for (let element of withContents) { - let attr = String(element.properties.content); - if (attr.startsWith(host.origin)) { - yield* downloader.download(attr, source); - let url = new URL(attr); - url.host = base.host; - url.port = base.port; - url.protocol = base.protocol; - element.properties.content = url.href; + let withContents = selectAll("[content]", html); + for (let element of withContents) { + let attr = String(element.properties.content); + if (attr.startsWith(host.origin)) { + yield* downloader.download(attr, source); + let url = new URL(attr); + url.host = base.host; + url.port = base.port; + url.protocol = base.protocol; + element.properties.content = url.href; + } } - } - let output = toHtml(html); - yield* call(async () => { - let destdir = dirname(destpath); - await ensureDir(destdir); - await Deno.writeTextFile(destpath, output); - }); - return { ok: true, bytes: new TextEncoder().encode(output).byteLength }; + let output = toHtml(html); + yield* call(async () => { + let destdir = dirname(destpath); + await ensureDir(destdir); + await Deno.writeTextFile(destpath, output); + }); + return { + ok: true, + bytes: new TextEncoder().encode(output).byteLength, + }; + } else { + let size = Number(response.headers.get("Content-Length") ?? 0); + yield* call(async () => { + let destdir = dirname(path); + await ensureDir(destdir); + await Deno.writeFile(path, response.body!); + }); + return { ok: true, bytes: size }; + } } else { - let size = Number(response.headers.get("Content-Length") ?? 0); - yield* call(async () => { - let destdir = dirname(path); - await ensureDir(destdir); - await Deno.writeFile(path, response.body!); - }); - return { ok: true, bytes: size }; + return fail( + new Error( + `GET ${source} responded ${response.status} ${response.statusText}`, + ), + ); } - } else { - return { - ok: false, - url: source.toString(), - referrer: referrer.toString(), - }; + } catch (cause) { + // e.g. a gzip/transport decode error, an HTML parse error, or a + // filesystem write error — none of which name the url on their own. + return fail(new Error(`could not download ${source}`, { cause })); } }, }); diff --git a/main.ts b/main.ts index e623141..4a23ccd 100644 --- a/main.ts +++ b/main.ts @@ -18,7 +18,7 @@ await main(function* (args) { case "main": { let result = parser.parse(); if (result.ok) { - let { base, site, output } = result.value; + let { base, site, output, strict } = result.value; let stdin = yield* initStdin; @@ -32,6 +32,7 @@ await main(function* (args) { base: new URL(base), host: new URL(site), dir: output, + strict, }); let { errors } = yield* withProgress({ @@ -44,7 +45,15 @@ await main(function* (args) { console.error(`\n${errors.length} failed downloads:\n`); for (let error of errors) { console.error(` ${error.url}`); - console.error(` referrer: ${error.referrer}\n`); + console.error(` referrer: ${error.referrer}`); + let cause: unknown = error.error; + let indent = " "; + while (cause instanceof Error) { + console.error(`${indent}${cause.name}: ${cause.message}`); + cause = cause.cause; + indent += " "; + } + console.error(""); } yield* exit(1); } diff --git a/progress.ts b/progress.ts index 1d6543c..a5642da 100644 --- a/progress.ts +++ b/progress.ts @@ -7,6 +7,7 @@ import type { Staticalizer } from "./staticalize.ts"; export interface DownloadError { url: string; referrer: string; + error: Error; } export interface ProgressResult { @@ -66,7 +67,11 @@ export function* withProgress( } } else { model.errors++; - errors.push({ url: result.url, referrer: result.referrer }); + errors.push({ + url: result.url, + referrer: result.referrer, + error: result.error, + }); } yield* render(progress(model)); return result; diff --git a/staticalize.ts b/staticalize.ts index 259118d..04f234d 100644 --- a/staticalize.ts +++ b/staticalize.ts @@ -15,6 +15,7 @@ export interface StaticalizeOptions { host: URL; base: URL; dir: string; + strict?: boolean; } export interface Staticalizer { @@ -25,7 +26,7 @@ export interface Staticalizer { export function useStaticalizer( options: StaticalizeOptions, ): Operation { - let { host, base, dir } = options; + let { host, base, dir, strict } = options; return resource(function* (provide) { let signal = yield* useAbortSignal(); @@ -40,10 +41,22 @@ export function useStaticalizer( error.name = `SitemapError`; throw error; } - let text = await response.text(); - let xml = parse(text, { - flatten: { attributes: false, empty: false, text: true }, - }) as unknown as SitemapXML; + let xml: SitemapXML; + try { + let text = await response.text(); + xml = parse(text, { + flatten: { attributes: false, empty: false, text: true }, + }) as unknown as SitemapXML; + } catch (cause) { + // Reading or parsing the sitemap body can fail on its own (e.g. a + // gzip/transport decode error, or malformed XML); attach the URL so the + // failure is traceable rather than a bare stack trace. + let error = new Error(`GET ${url} could not be read as a sitemap`, { + cause, + }); + error.name = `SitemapError`; + throw error; + } let entries = xml.urlset.url ?? xml.urlset.urls ?? []; let list = Array.isArray(entries) ? entries : [entries]; @@ -56,7 +69,7 @@ export function useStaticalizer( ); }); - let downloader = yield* useDownloader({ host, base, outdir: dir }); + let downloader = yield* useDownloader({ host, base, outdir: dir, strict }); yield* provide({ urls, diff --git a/test/staticalize.test.ts b/test/staticalize.test.ts index a316ae1..470ce37 100644 --- a/test/staticalize.test.ts +++ b/test/staticalize.test.ts @@ -138,6 +138,55 @@ describe("staticalize", () => { ); }); + it("keeps going and does not crash when a page fails to download", async () => { + app.get("/", (c) => c.html("

Index

")); + // advertises gzip but the body is not valid gzip — decoding this response + // throws `Invalid gzip header`, which used to escape uncaught and crash the + // whole run with no indication of which url was at fault. + app.get("/bad", (c) => + c.body("this is definitely not gzip", 200, { + "Content-Type": "text/html", + "Content-Encoding": "gzip", + })); + app.get("/good", (c) => c.html("

Good

")); + app.get(...sitemap(["/", "/bad", "/good"])); + + // the run resolves rather than rejecting with the decode error + await staticalize({ + host, + base: new URL("https://frontside.com"), + dir: "test/dist", + }); + + // the healthy pages were still written to disk + await expect(exists("test/dist/index.html")).resolves.toEqual(true); + await expect(exists("test/dist/good/index.html")).resolves.toEqual(true); + // the page that could not be decoded was skipped, not written + await expect(exists("test/dist/bad/index.html")).resolves.toEqual(false); + }); + + it("fails fast on the first download error when strict", async () => { + app.get("/", (c) => c.html("

Index

")); + // same bad-gzip fixture as above: decoding this response throws. + app.get("/bad", (c) => + c.body("this is definitely not gzip", 200, { + "Content-Type": "text/html", + "Content-Encoding": "gzip", + })); + app.get("/good", (c) => c.html("

Good

")); + app.get(...sitemap(["/", "/bad", "/good"])); + + // in strict mode the failure aborts the run rather than being collected + await expect( + staticalize({ + host, + base: new URL("https://frontside.com"), + dir: "test/dist", + strict: true, + }), + ).rejects.toThrow(/could not download/); + }); + it("does not download assets that are in a different domain", async () => { app.get("/spa", (c) => c.html(`