diff --git a/config.ts b/config.ts index 5156cd9..7e357b9 100644 --- a/config.ts +++ b/config.ts @@ -29,5 +29,15 @@ export const config = program({ "Fail on the first download error instead of collecting all failures and continuing", ...field(z.boolean(), field.default(false)), }, + concurrency: { + description: + "Maximum number of concurrent downloads. Lower values avoid upstream rate limiting.", + ...field(z.number(), field.default(75)), + }, + retries: { + description: + "Number of times to retry a failed download before giving up. Defaults to 0 in strict mode.", + ...field(z.number()), + }, }), }); diff --git a/downloader.ts b/downloader.ts index 9b517cd..550e375 100644 --- a/downloader.ts +++ b/downloader.ts @@ -1,7 +1,7 @@ import { - call, type Operation, resource, + sleep, until, useAbortSignal, } from "effection"; @@ -22,6 +22,8 @@ export interface DownloaderOptions { base: URL; outdir: string; strict?: boolean; + concurrency?: number; + retries?: number; } export type DownloadResult = @@ -35,7 +37,7 @@ export const DownloadApi = createApi("@staticalize/download", { source: URL, referrer: URL, ): Operation { - let { host, base, outdir, strict } = opts; + let { host, base, outdir, strict, retries = 3 } = opts; let signal = yield* useAbortSignal(); let path = normalize(join(outdir, source.pathname)); @@ -52,100 +54,118 @@ export const DownloadApi = createApi("@staticalize/download", { }; try { - let response = yield* until(fetch(source.toString(), { signal })); - if (response.ok) { - if (response.headers.get("Content-Type")?.includes("html")) { - let destpath = join(path, "index.html"); - let content = yield* until(response.text()); - let html = fromHtml(content); - - let links = selectAll("link[href]", html); - - for (let link of links) { - let href = link.properties.href as string; - yield* downloader.download(href, source); - - // replace self-referencing absolute urls with the destination site - if (href.startsWith(host.origin)) { - let url = new URL(href); - url.host = base.host; - url.port = base.port; - url.protocol = base.protocol; - link.properties.href = url.href; - } + let response = yield* fetchWithRetry(source.toString(), signal, retries); + + if (response.headers.get("Content-Type")?.includes("html")) { + let destpath = join(path, "index.html"); + let content = yield* until(response.text()); + let html = fromHtml(content); + + let links = selectAll("link[href]", html); + + for (let link of links) { + let href = link.properties.href as string; + yield* downloader.download(href, source); + + // replace self-referencing absolute urls with the destination site + if (href.startsWith(host.origin)) { + let url = new URL(href); + url.host = base.host; + url.port = base.port; + url.protocol = base.protocol; + link.properties.href = url.href; } + } - let assets = selectAll("[src]", html); + let assets = selectAll("[src]", html); - for (let element of assets) { - let src = element.properties.src as string; - yield* downloader.download(src, source); + for (let element of assets) { + let src = element.properties.src as string; + yield* downloader.download(src, source); - // replace self-referencing absolute urls with the destination site - if (src.startsWith(host.origin)) { - let url = new URL(src); - url.host = base.host; - url.port = base.port; - url.protocol = base.protocol; - element.properties.src = url.href; - } + // replace self-referencing absolute urls with the destination site + if (src.startsWith(host.origin)) { + let url = new URL(src); + url.host = base.host; + url.port = base.port; + url.protocol = base.protocol; + element.properties.src = url.href; } + } - let withContents = selectAll("[content]", html); - for (let element of withContents) { - let attr = String(element.properties.content); - if (attr.startsWith(host.origin)) { - yield* downloader.download(attr, source); - let url = new URL(attr); - url.host = base.host; - url.port = base.port; - url.protocol = base.protocol; - element.properties.content = url.href; - } + let withContents = selectAll("[content]", html); + for (let element of withContents) { + let attr = String(element.properties.content); + if (attr.startsWith(host.origin)) { + yield* downloader.download(attr, source); + let url = new URL(attr); + url.host = base.host; + url.port = base.port; + url.protocol = base.protocol; + element.properties.content = url.href; } - - let output = toHtml(html); - yield* call(async () => { - let destdir = dirname(destpath); - await ensureDir(destdir); - await Deno.writeTextFile(destpath, output); - }); - return { - ok: true, - bytes: new TextEncoder().encode(output).byteLength, - }; - } else { - let size = Number(response.headers.get("Content-Length") ?? 0); - yield* call(async () => { - let destdir = dirname(path); - await ensureDir(destdir); - await Deno.writeFile(path, response.body!); - }); - return { ok: true, bytes: size }; } + + let output = toHtml(html); + let destdir = dirname(destpath); + yield* until(ensureDir(destdir)); + yield* until(Deno.writeTextFile(destpath, output)); + return { + ok: true, + bytes: new TextEncoder().encode(output).byteLength, + }; } else { - return fail( - new Error( - `GET ${source} responded ${response.status} ${response.statusText}`, - ), - ); + let size = Number(response.headers.get("Content-Length") ?? 0); + let destdir = dirname(path); + yield* until(ensureDir(destdir)); + yield* until(Deno.writeFile(path, response.body!)); + return { ok: true, bytes: size }; } - } catch (cause) { - // e.g. a gzip/transport decode error, an HTML parse error, or a - // filesystem write error — none of which name the url on their own. - return fail(new Error(`could not download ${source}`, { cause })); + } catch (cause: unknown) { + return fail( + cause instanceof Error + ? new Error(`could not download ${source}`, { cause }) + : new Error(`could not download ${source}`), + ); } }, }); +function* fetchWithRetry( + url: string, + signal: AbortSignal, + retries: number, +): Operation { + let lastError: Error | undefined; + for (let attempt = 0; attempt <= retries; attempt++) { + if (attempt > 0) { + // exponential backoff: 1s, 2s, 4s, ... + yield* sleep(1000 * 2 ** (attempt - 1)); + } + + try { + let response = yield* until(fetch(url, { signal })); + if (response.ok) { + return response; + } + lastError = new Error( + `GET ${url} responded ${response.status} ${response.statusText}`, + ); + } catch (cause) { + lastError = new Error(`could not download ${url}`, { cause }); + } + } + throw lastError!; +} + const { download } = DownloadApi.operations; export function useDownloader(opts: DownloaderOptions): Operation { let seen = new Map(); return resource(function* (provide) { - let { host } = opts; + let { host, concurrency = 75 } = opts; - let buffer = yield* useTaskBuffer(75); + let buffer = yield* useTaskBuffer(concurrency); let downloader: Downloader = { *download(loc, context = host) { diff --git a/main.ts b/main.ts index 4a23ccd..6bacde3 100644 --- a/main.ts +++ b/main.ts @@ -18,7 +18,10 @@ await main(function* (args) { case "main": { let result = parser.parse(); if (result.ok) { - let { base, site, output, strict } = result.value; + let { base, site, output, strict, concurrency, retries: retriesRaw } = + result.value; + // don't have a great way to default dynamically based on strict mode + let retries = retriesRaw ?? (strict ? 0 : 3); let stdin = yield* initStdin; @@ -33,6 +36,8 @@ await main(function* (args) { host: new URL(site), dir: output, strict, + concurrency, + retries, }); let { errors } = yield* withProgress({ diff --git a/staticalize.ts b/staticalize.ts index 04f234d..3a4ebbe 100644 --- a/staticalize.ts +++ b/staticalize.ts @@ -16,6 +16,8 @@ export interface StaticalizeOptions { base: URL; dir: string; strict?: boolean; + concurrency?: number; + retries?: number; } export interface Staticalizer { @@ -26,7 +28,7 @@ export interface Staticalizer { export function useStaticalizer( options: StaticalizeOptions, ): Operation { - let { host, base, dir, strict } = options; + let { host, base, dir, strict, concurrency, retries } = options; return resource(function* (provide) { let signal = yield* useAbortSignal(); @@ -69,7 +71,14 @@ export function useStaticalizer( ); }); - let downloader = yield* useDownloader({ host, base, outdir: dir, strict }); + let downloader = yield* useDownloader({ + host, + base, + outdir: dir, + strict, + concurrency, + retries, + }); yield* provide({ urls,