🚑 report failing url instead of crashing on download errors

README.md

@@ -28,16 +28,22 @@ file server running at `frontside.com`.
 ### CLI
 
 ```
-Usage: staticalize [options]
+Usage: staticalize [OPTIONS] <site>
 
-Create a static version of a website by traversing a dynamically evaluated sitemap.xml
+Arguments:
+   <site>                    URL of the website to staticalize. E.g. http://localhost:8000
 
 Options:
-  -h, --help                Show help
-  -V, --version             Show version
-  -s, --site <string>       URL of the website to staticalize. E.g. http://localhost:8000  [required]
-  -o, --output <string>  Directory to place the downloaded site (default: "dist")
-      --base <string>   Base URL of the public website. E.g. http://frontside.com      [required]
+   --output <OUTPUT>         Directory to place the downloaded site [default: dist]
+   --base <BASE>             Base URL of the public website. E.g. http://frontside.com
+   --strict                  Fail on the first download error instead of collecting all failures and continuing [default: false]
+   -h, --help                show help
+   -v, --version             show version
 ```
 
+By default, staticalize downloads as much of the site as it can: any page or
+asset that fails is reported at the end and the process exits non-zero, but the
+run continues so you get every page that _did_ work. Pass `--strict` to fail
+fast and abort the whole run on the first download error.
+
 [sitemap]: https://sitemaps.org

config.ts

@@ -24,5 +24,10 @@ export const config = program({
       description: "Base URL of the public website. E.g. http://frontside.com",
       ...field(url()),
     },
+    strict: {
+      description:
+        "Fail on the first download error instead of collecting all failures and continuing",
+      ...field(z.boolean(), field.default(false)),
+    },
   }),
 });

deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@frontside/staticalize",
-  "version": "0.2.2",
+  "version": "0.2.6",
   "exports": {
     ".": "./mod.ts",
     "./cli": "./main.ts"

downloader.ts

@@ -21,11 +21,12 @@ export interface DownloaderOptions {
   host: URL;
   base: URL;
   outdir: string;
+  strict?: boolean;
 }
 
 export type DownloadResult =
   | { ok: true; bytes: number }
-  | { ok: false; url: string; referrer: string };
+  | { ok: false; url: string; referrer: string; error: Error };
 
 export const DownloadApi = createApi("@staticalize/download", {
   *download(
@@ -34,84 +35,105 @@ export const DownloadApi = createApi("@staticalize/download", {
     source: URL,
     referrer: URL,
   ): Operation<DownloadResult> {
-    let { host, base, outdir } = opts;
+    let { host, base, outdir, strict } = opts;
     let signal = yield* useAbortSignal();
     let path = normalize(join(outdir, source.pathname));
 
-    let response = yield* until(fetch(source.toString(), { signal }));
-    if (response.ok) {
-      if (response.headers.get("Content-Type")?.includes("html")) {
-        let destpath = join(path, "index.html");
-        let content = yield* until(response.text());
-        let html = fromHtml(content);
-
-        let links = selectAll("link[href]", html);
-
-        for (let link of links) {
-          let href = link.properties.href as string;
-          yield* downloader.download(href, source);
-
-          // replace self-referencing absolute urls with the destination site
-          if (href.startsWith(host.origin)) {
-            let url = new URL(href);
-            url.host = base.host;
-            url.port = base.port;
-            url.protocol = base.protocol;
-            link.properties.href = url.href;
+    let fail = (error: Error): DownloadResult => {
+      if (strict) {
+        throw error;
+      }
+      return {
+        ok: false,
+        url: source.toString(),
+        referrer: referrer.toString(),
+        error,
+      };
+    };
+
+    try {
+      let response = yield* until(fetch(source.toString(), { signal }));
+      if (response.ok) {
+        if (response.headers.get("Content-Type")?.includes("html")) {
+          let destpath = join(path, "index.html");
+          let content = yield* until(response.text());
+          let html = fromHtml(content);
+
+          let links = selectAll("link[href]", html);
+
+          for (let link of links) {
+            let href = link.properties.href as string;
+            yield* downloader.download(href, source);
+
+            // replace self-referencing absolute urls with the destination site
+            if (href.startsWith(host.origin)) {
+              let url = new URL(href);
+              url.host = base.host;
+              url.port = base.port;
+              url.protocol = base.protocol;
+              link.properties.href = url.href;
+            }
           }
-        }
 
-        let assets = selectAll("[src]", html);
+          let assets = selectAll("[src]", html);
 
-        for (let element of assets) {
-          let src = element.properties.src as string;
-          yield* downloader.download(src, source);
+          for (let element of assets) {
+            let src = element.properties.src as string;
+            yield* downloader.download(src, source);
 
-          // replace self-referencing absolute urls with the destination site
-          if (src.startsWith(host.origin)) {
-            let url = new URL(src);
-            url.host = base.host;
-            url.port = base.port;
-            url.protocol = base.protocol;
-            element.properties.src = url.href;
+            // replace self-referencing absolute urls with the destination site
+            if (src.startsWith(host.origin)) {
+              let url = new URL(src);
+              url.host = base.host;
+              url.port = base.port;
+              url.protocol = base.protocol;
+              element.properties.src = url.href;
+            }
           }
-        }
 
-        let withContents = selectAll("[content]", html);
-        for (let element of withContents) {
-          let attr = String(element.properties.content);
-          if (attr.startsWith(host.origin)) {
-            yield* downloader.download(attr, source);
-            let url = new URL(attr);
-            url.host = base.host;
-            url.port = base.port;
-            url.protocol = base.protocol;
-            element.properties.content = url.href;
+          let withContents = selectAll("[content]", html);
+          for (let element of withContents) {
+            let attr = String(element.properties.content);
+            if (attr.startsWith(host.origin)) {
+              yield* downloader.download(attr, source);
+              let url = new URL(attr);
+              url.host = base.host;
+              url.port = base.port;
+              url.protocol = base.protocol;
+              element.properties.content = url.href;
+            }
           }
-        }
 
-        let output = toHtml(html);
-        yield* call(async () => {
-          let destdir = dirname(destpath);
-          await ensureDir(destdir);
-          await Deno.writeTextFile(destpath, output);
-        });
-        return { ok: true, bytes: new TextEncoder().encode(output).byteLength };
+          let output = toHtml(html);
+          yield* call(async () => {
+            let destdir = dirname(destpath);
+            await ensureDir(destdir);
+            await Deno.writeTextFile(destpath, output);
+          });
+          return {
+            ok: true,
+            bytes: new TextEncoder().encode(output).byteLength,
+          };
+        } else {
+          let size = Number(response.headers.get("Content-Length") ?? 0);
+          yield* call(async () => {
+            let destdir = dirname(path);
+            await ensureDir(destdir);
+            await Deno.writeFile(path, response.body!);
+          });
+          return { ok: true, bytes: size };
+        }
       } else {
-        let size = Number(response.headers.get("Content-Length") ?? 0);
-        yield* call(async () => {
-          let destdir = dirname(path);
-          await ensureDir(destdir);
-          await Deno.writeFile(path, response.body!);
-        });
-        return { ok: true, bytes: size };
+        return fail(
+          new Error(
+            `GET ${source} responded ${response.status} ${response.statusText}`,
+          ),
+        );
       }
-    } else {
-      return {
-        ok: false,
-        url: source.toString(),
-        referrer: referrer.toString(),
-      };
+    } catch (cause) {
+      // e.g. a gzip/transport decode error, an HTML parse error, or a
+      // filesystem write error — none of which name the url on their own.
+      return fail(new Error(`could not download ${source}`, { cause }));
     }
   },
 });

main.ts

@@ -18,7 +18,7 @@ await main(function* (args) {
     case "main": {
       let result = parser.parse();
       if (result.ok) {
-        let { base, site, output } = result.value;
+        let { base, site, output, strict } = result.value;
 
         let stdin = yield* initStdin;
 
@@ -32,6 +32,7 @@ await main(function* (args) {
           base: new URL(base),
           host: new URL(site),
           dir: output,
+          strict,
         });
 
         let { errors } = yield* withProgress({
@@ -44,7 +45,15 @@ await main(function* (args) {
           console.error(`\n${errors.length} failed downloads:\n`);
           for (let error of errors) {
             console.error(`  ${error.url}`);
-            console.error(`    referrer: ${error.referrer}\n`);
+            console.error(`    referrer: ${error.referrer}`);
+            let cause: unknown = error.error;
+            let indent = "    ";
+            while (cause instanceof Error) {
+              console.error(`${indent}${cause.name}: ${cause.message}`);
+              cause = cause.cause;
+              indent += "  ";
+            }
+            console.error("");
           }
           yield* exit(1);
         }

progress.ts

@@ -7,6 +7,7 @@ import type { Staticalizer } from "./staticalize.ts";
 export interface DownloadError {
   url: string;
   referrer: string;
+  error: Error;
 }
 
 export interface ProgressResult {
@@ -66,7 +67,11 @@ export function* withProgress(
         }
       } else {
         model.errors++;
-        errors.push({ url: result.url, referrer: result.referrer });
+        errors.push({
+          url: result.url,
+          referrer: result.referrer,
+          error: result.error,
+        });
       }
       yield* render(progress(model));
       return result;

staticalize.ts

@@ -15,6 +15,7 @@ export interface StaticalizeOptions {
   host: URL;
   base: URL;
   dir: string;
+  strict?: boolean;
 }
 
 export interface Staticalizer {
@@ -25,7 +26,7 @@ export interface Staticalizer {
 export function useStaticalizer(
   options: StaticalizeOptions,
 ): Operation<Staticalizer> {
-  let { host, base, dir } = options;
+  let { host, base, dir, strict } = options;
 
   return resource(function* (provide) {
     let signal = yield* useAbortSignal();
@@ -40,10 +41,22 @@ export function useStaticalizer(
         error.name = `SitemapError`;
         throw error;
       }
-      let text = await response.text();
-      let xml = parse(text, {
-        flatten: { attributes: false, empty: false, text: true },
-      }) as unknown as SitemapXML;
+      let xml: SitemapXML;
+      try {
+        let text = await response.text();
+        xml = parse(text, {
+          flatten: { attributes: false, empty: false, text: true },
+        }) as unknown as SitemapXML;
+      } catch (cause) {
+        // Reading or parsing the sitemap body can fail on its own (e.g. a
+        // gzip/transport decode error, or malformed XML); attach the URL so the
+        // failure is traceable rather than a bare stack trace.
+        let error = new Error(`GET ${url} could not be read as a sitemap`, {
+          cause,
+        });
+        error.name = `SitemapError`;
+        throw error;
+      }
 
       let entries = xml.urlset.url ?? xml.urlset.urls ?? [];
       let list = Array.isArray(entries) ? entries : [entries];
@@ -56,7 +69,7 @@ export function useStaticalizer(
       );
     });
 
-    let downloader = yield* useDownloader({ host, base, outdir: dir });
+    let downloader = yield* useDownloader({ host, base, outdir: dir, strict });
 
     yield* provide({
       urls,