diff --git a/.eslintrc.cjs b/.eslintrc.cjs index a5d40eee5..9a42e2b9c 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -30,7 +30,7 @@ module.exports = { }, ], "@typescript-eslint/no-floating-promises": "error", - "@typescript-eslint/await-thenable": "error" + "@typescript-eslint/await-thenable": "error", }, reportUnusedDisableDirectives: true, }; diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 35c1ba3a8..b5ee25feb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -87,4 +87,4 @@ jobs: run: | sudo rm -rf ./test-crawls mkdir test-crawls - sudo CI=true yarn test ./tests/saved-state.test.js ./tests/qa_compare.test.js + sudo CI=true yarn test ./tests/saved-state.test.ts ./tests/qa_compare.test.ts diff --git a/jest.config.js b/jest.config.js new file mode 100644 index 000000000..985ab80fd --- /dev/null +++ b/jest.config.js @@ -0,0 +1,18 @@ +/** @type {import("ts-jest").JestConfigWithTsJest} **/ +export default { + testEnvironment: "node", + extensionsToTreatAsEsm: [".ts"], + moduleNameMapper: { + "^(\\.{1,2}/.*)\\.js$": "$1", + }, + transform: { + "^.+\\.tsx?$": [ + "ts-jest", + { + useESM: true, + tsconfig: "tsconfig.test.json", + }, + ], + }, + testTimeout: 90000, +}; diff --git a/package.json b/package.json index 26a49cb48..2e35c09bd 100644 --- a/package.json +++ b/package.json @@ -48,9 +48,11 @@ }, "devDependencies": { "@types/get-folder-size": "^3.0.4", + "@types/jest": "^30.0.0", "@types/js-levenshtein": "^1.1.3", "@types/js-yaml": "^4.0.8", - "@types/node": "^20.8.7", + "@types/md5": "^2.3.6", + "@types/node": "^25.5.0", "@types/pixelmatch": "^5.2.6", "@types/pngjs": "^6.0.4", "@types/sax": "^1.2.7", @@ -67,12 +69,10 @@ "md5": "^2.3.0", "prettier": "3.0.3", "puppeteer": "^24.4.0", + "ts-jest": "^29.4.9", + "type-fest": "^5.5.0", "typescript": "^5.5.4" }, - "jest": { - "transform": {}, - "testTimeout": 90000 - }, "resolutions": { "wrap-ansi": "7.0.0", "warcio": "^2.4.9", diff --git a/src/util/storage.ts b/src/util/storage.ts index 15439e121..772a7b200 100644 --- a/src/util/storage.ts +++ b/src/util/storage.ts @@ -276,7 +276,7 @@ export async function checkDiskUtilization( collDir: string, params: CrawlerArgs, archiveDirSize: number, - dfOutput = null, + dfOutput: string | null = null, doLog = true, ) { const diskUsage: Record = await getDiskUsage( @@ -350,7 +350,10 @@ export async function getDFOutput(path: string) { return res.stdout; } -export async function getDiskUsage(path = "/crawls", dfOutput = null) { +export async function getDiskUsage( + path = "/crawls", + dfOutput: string | null = null, +) { const result = dfOutput || (await getDFOutput(path)); const lines = result.split("\n"); const keys = lines[0].split(/\s+/gi); diff --git a/tests/.DS_Store b/tests/.DS_Store deleted file mode 100644 index e0497c783..000000000 Binary files a/tests/.DS_Store and /dev/null differ diff --git a/tests/adblockrules.test.js b/tests/adblockrules.test.ts similarity index 83% rename from tests/adblockrules.test.js rename to tests/adblockrules.test.ts index c10d1ac27..5801b3c46 100644 --- a/tests/adblockrules.test.js +++ b/tests/adblockrules.test.ts @@ -2,7 +2,11 @@ import child_process from "child_process"; import fs from "fs"; import yaml from "js-yaml"; -function runCrawl(name, config, commandExtra = "") { +function runCrawl( + name: string, + config: Record, + commandExtra = "", +) { config.generateCDX = true; config.depth = 0; config.collection = name; @@ -10,9 +14,9 @@ function runCrawl(name, config, commandExtra = "") { const configYaml = yaml.dump(config); try { - const proc = child_process.execSync( + child_process.execSync( `docker run -i -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --config stdin ${commandExtra}`, - { input: configYaml, stdin: "inherit", encoding: "utf8" }, + { input: configYaml, stdio: ["pipe", "ignore", "ignore"], encoding: "utf8" }, ); //console.log(proc); @@ -21,7 +25,7 @@ function runCrawl(name, config, commandExtra = "") { } } -function doesCDXContain(coll, value) { +function doesCDXContain(coll: string, value: string) { const data = fs.readFileSync( `test-crawls/collections/${coll}/indexes/index.cdxj`, ); diff --git a/tests/add-exclusion.test.js b/tests/add-exclusion.test.ts similarity index 59% rename from tests/add-exclusion.test.js rename to tests/add-exclusion.test.ts index 861aa9072..bc1f7e64c 100644 --- a/tests/add-exclusion.test.js +++ b/tests/add-exclusion.test.ts @@ -1,24 +1,35 @@ -import { exec } from "child_process"; +import { exec, ExecException } from "child_process"; import Redis from "ioredis"; - -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} +import { sleep } from "./utils"; test("dynamically add exclusion while crawl is running", async () => { - let callback = null; + let callback: + | (( + error: ExecException | null, + stdout: NonSharedBuffer, + stderr: NonSharedBuffer, + ) => void) + | null = null; - const p = new Promise((resolve) => { - callback = (error, stdout, stderr) => { - resolve({ error, stdout, stderr }); + const p = new Promise<{ + error: ExecException | null; + stdout: NonSharedBuffer; + stderr: NonSharedBuffer; + }>((resolve) => { + callback = ( + error: ExecException | null, + stdout: NonSharedBuffer, + stderr: NonSharedBuffer, + ) => { + resolve({ error, stdout, stderr } as const); }; }); try { exec( "docker run -p 36382:6379 -e CRAWL_ID=test -v $PWD/test-crawls:/crawls -v $PWD/tests/fixtures:/tests/fixtures webrecorder/browsertrix-crawler crawl --collection add-exclusion --url https://old.webrecorder.net/ --scopeType prefix --limit 20 --logging debug --debugAccessRedis", - { shell: "/bin/bash" }, - callback, + { shell: "/bin/bash", encoding: "buffer" }, + callback!, ); } catch (error) { console.log(error); @@ -26,7 +37,10 @@ test("dynamically add exclusion while crawl is running", async () => { await sleep(3000); - const redis = new Redis("redis://127.0.0.1:36382/0", { lazyConnect: true, retryStrategy: () => null }) + const redis = new Redis("redis://127.0.0.1:36382/0", { + lazyConnect: true, + retryStrategy: () => null, + }); await redis.connect(); @@ -53,4 +67,3 @@ test("dynamically add exclusion while crawl is running", async () => { expect(stdout.indexOf("Removing excluded URL") > 0).toBe(true); }); - diff --git a/tests/basic_crawl.test.js b/tests/basic_crawl.test.ts similarity index 93% rename from tests/basic_crawl.test.js rename to tests/basic_crawl.test.ts index 940967ab8..4da0a78f6 100644 --- a/tests/basic_crawl.test.js +++ b/tests/basic_crawl.test.ts @@ -2,9 +2,7 @@ import child_process from "child_process"; import fs from "fs"; import path from "path"; import md5 from "md5"; - -const doValidate = process.argv.filter((x) => x.startsWith('-validate'))[0]; -const testIf = (condition, ...args) => condition ? test(...args) : test.skip(...args); +import { doValidate, testIf } from "./utils"; test("ensure basic crawl run with docker run passes", async () => { child_process.execSync( @@ -38,9 +36,9 @@ test("check that individual WARCs have correct prefix and are under rollover siz test("check that a combined warc file exists in the archive folder", () => { const warcLists = fs.readdirSync("test-crawls/collections/wr-net"); - var captureFound = 0; + let captureFound = 0; - for (var i = 0; i < warcLists.length; i++) { + for (let i = 0; i < warcLists.length; i++) { if (warcLists[i].endsWith("_0.warc.gz")) { captureFound = 1; } @@ -54,7 +52,7 @@ test("check that a combined warc file is under the rolloverSize", () => { ); let rolloverSize = 0; - function getFileSize(filename) { + function getFileSize(filename: string) { return fs.statSync(filename).size; } diff --git a/tests/blockrules.test.js b/tests/blockrules.test.ts similarity index 52% rename from tests/blockrules.test.js rename to tests/blockrules.test.ts index cef378010..cf8a10f09 100644 --- a/tests/blockrules.test.js +++ b/tests/blockrules.test.ts @@ -1,11 +1,9 @@ import child_process from "child_process"; import fs from "fs"; import yaml from "js-yaml"; +import { type TestConfig, isCI, testIf } from "./utils"; -const isCI = !!process.env.CI; -const testIf = (condition, ...args) => condition ? test(...args) : test.skip(...args); - -function runCrawl(name, config, commandExtra = "") { +function runCrawl(name: string, config: TestConfig, commandExtra = "") { config.generateCDX = true; config.depth = 0; config.collection = name; @@ -13,9 +11,9 @@ function runCrawl(name, config, commandExtra = "") { const configYaml = yaml.dump(config); try { - const proc = child_process.execSync( + child_process.execSync( `docker run -i -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --config stdin ${commandExtra}`, - { input: configYaml, stdin: "inherit", encoding: "utf8" }, + { input: configYaml, stdio: ["pipe", "ignore", "ignore"], encoding: "utf8" }, ); //console.log(proc); @@ -24,14 +22,14 @@ function runCrawl(name, config, commandExtra = "") { } } -function doesCDXContain(coll, value) { +function doesCDXContain(coll: string, value: string) { const data = fs.readFileSync( `test-crawls/collections/${coll}/indexes/index.cdxj`, ); return data.indexOf(value) >= 0; } -function checkVideo(coll) { +function checkVideo(coll: string) { return doesCDXContain(coll, '"video/mp4"'); } @@ -66,23 +64,27 @@ test("test block rule on specific URL", () => { ).toBe(false); }); -testIf(!isCI, "test block rule based on iframe text, content included due to match", () => { - const config = { - url: "https://oembed.link/https://www.youtube.com/watch?v=aT-Up5Y4uRI", - blockRules: [ - { - url: "https://www.youtube.com/embed/", - frameTextMatch: - '\\\\"channelId\\\\":\\\\"UCrQElMF25VP-1JjhBuFsW_Q\\\\"', - type: "allowOnly", - }, - ], - }; - - runCrawl("block-2", config); - - expect(checkVideo("block-2")).toBe(true); -}); +testIf( + !isCI, + "test block rule based on iframe text, content included due to match", + () => { + const config = { + url: "https://oembed.link/https://www.youtube.com/watch?v=aT-Up5Y4uRI", + blockRules: [ + { + url: "https://www.youtube.com/embed/", + frameTextMatch: + '\\\\"channelId\\\\":\\\\"UCrQElMF25VP-1JjhBuFsW_Q\\\\"', + type: "allowOnly", + }, + ], + }; + + runCrawl("block-2", config); + + expect(checkVideo("block-2")).toBe(true); + }, +); test("test block rule based on iframe text, wrong text, content should be excluded", () => { const config = { @@ -119,28 +121,32 @@ test("test block rule based on iframe text, block matched", () => { expect(checkVideo("block-4")).toBe(false); }); -testIf(!isCI, "test rule based on iframe text not matching, plus allowOnly iframe", () => { - const config = { - url: "https://oembed.link/https://www.youtube.com/watch?v=aT-Up5Y4uRI", - blockRules: [ - { - url: "example.com/embed/", - frameTextMatch: - '\\\\"channelId\\\\":\\\\"UCrQElMF25VP-1JjhBuFsW_Q\\\\"', - type: "block", - }, - { - url: "(youtube.com|example.com)/embed/", - type: "allowOnly", - inFrameUrl: "oembed.link/", - }, - ], - }; - - runCrawl("non-block-5", config); - - expect(checkVideo("non-block-5")).toBe(true); -}); +testIf( + !isCI, + "test rule based on iframe text not matching, plus allowOnly iframe", + () => { + const config = { + url: "https://oembed.link/https://www.youtube.com/watch?v=aT-Up5Y4uRI", + blockRules: [ + { + url: "example.com/embed/", + frameTextMatch: + '\\\\"channelId\\\\":\\\\"UCrQElMF25VP-1JjhBuFsW_Q\\\\"', + type: "block", + }, + { + url: "(youtube.com|example.com)/embed/", + type: "allowOnly", + inFrameUrl: "oembed.link/", + }, + ], + }; + + runCrawl("non-block-5", config); + + expect(checkVideo("non-block-5")).toBe(true); + }, +); test("test block url in frame url", () => { const config = { @@ -164,40 +170,44 @@ test("test block url in frame url", () => { ).toBe(false); }); -testIf(!isCI, "test block rules complex example, block external urls on main frame, but not on youtube", () => { - const config = { - seeds: ["https://archiveweb.page/en/troubleshooting/errors/"], - depth: "0", - blockRules: [ - { - url: "(archiveweb.page|www.youtube.com)", - type: "allowOnly", - inFrameUrl: "archiveweb.page", - }, - { - url: "https://archiveweb.page/assets/js/vendor/lunr.min.js", - inFrameUrl: "archiveweb.page", - }, - { - url: "https://www.youtube.com/embed/", - type: "allowOnly", - frameTextMatch: - '(\\\\"channelId\\\\":\\\\"UCOHO8gYUWpDYFWHXmIwE02g\\\\")', - }, - ], - - combineWARC: true, - - logging: "stats,debug", - }; - - runCrawl("block-7", config); - - expect( - doesCDXContain( - "block-7", - '"https://archiveweb.page/assets/js/vendor/lunr.min.js"', - ), - ).toBe(false); - expect(checkVideo("block-7")).toBe(true); -}); +testIf( + !isCI, + "test block rules complex example, block external urls on main frame, but not on youtube", + () => { + const config = { + seeds: ["https://archiveweb.page/en/troubleshooting/errors/"], + depth: "0", + blockRules: [ + { + url: "(archiveweb.page|www.youtube.com)", + type: "allowOnly", + inFrameUrl: "archiveweb.page", + }, + { + url: "https://archiveweb.page/assets/js/vendor/lunr.min.js", + inFrameUrl: "archiveweb.page", + }, + { + url: "https://www.youtube.com/embed/", + type: "allowOnly", + frameTextMatch: + '(\\\\"channelId\\\\":\\\\"UCOHO8gYUWpDYFWHXmIwE02g\\\\")', + }, + ], + + combineWARC: true, + + logging: "stats,debug", + }; + + runCrawl("block-7", config); + + expect( + doesCDXContain( + "block-7", + '"https://archiveweb.page/assets/js/vendor/lunr.min.js"', + ), + ).toBe(false); + expect(checkVideo("block-7")).toBe(true); + }, +); diff --git a/tests/brave-query-redir.test.js b/tests/brave-query-redir.test.ts similarity index 71% rename from tests/brave-query-redir.test.js rename to tests/brave-query-redir.test.ts index ff5d9d721..9e2f77c45 100644 --- a/tests/brave-query-redir.test.js +++ b/tests/brave-query-redir.test.ts @@ -1,13 +1,13 @@ import fs from "fs"; -import { execSync } from "child_process"; +import { ExecException, execSync } from "child_process"; test("check that gclid query URL is automatically redirected to remove it", async () => { try { execSync( - "docker run --rm -v $PWD/test-crawls:/crawls -i webrecorder/browsertrix-crawler crawl --url 'https://old.webrecorder.net/about?gclid=abc' --collection test-brave-redir --behaviors \"\" --limit 1 --generateCDX"); - + "docker run --rm -v $PWD/test-crawls:/crawls -i webrecorder/browsertrix-crawler crawl --url 'https://old.webrecorder.net/about?gclid=abc' --collection test-brave-redir --behaviors \"\" --limit 1 --generateCDX", + ); } catch (error) { - console.log(error.stderr); + console.log((error as ExecException).stderr); } const filedata = fs.readFileSync( @@ -23,9 +23,15 @@ test("check that gclid query URL is automatically redirected to remove it", asyn for (const line of lines) { const json = line.split(" ").slice(2).join(" "); const data = JSON.parse(json); - if (data.url === "https://old.webrecorder.net/about?gclid=abc" && data.status === "307") { + if ( + data.url === "https://old.webrecorder.net/about?gclid=abc" && + data.status === "307" + ) { redirectFound = true; - } else if (data.url === "https://old.webrecorder.net/about" && data.status === "200") { + } else if ( + data.url === "https://old.webrecorder.net/about" && + data.status === "200" + ) { responseFound = true; } if (responseFound && redirectFound) { diff --git a/tests/collection_name.test.js b/tests/collection_name.test.ts similarity index 91% rename from tests/collection_name.test.js rename to tests/collection_name.test.ts index 1ad493681..ab9b9c26c 100644 --- a/tests/collection_name.test.js +++ b/tests/collection_name.test.ts @@ -4,7 +4,7 @@ import { exec as execCallback } from "child_process"; const exec = util.promisify(execCallback); test("check that the collection name is properly validated", async () => { - let passed = ""; + let passed: string | boolean = ""; try { await exec( @@ -18,7 +18,7 @@ test("check that the collection name is properly validated", async () => { }); test("check that the collection name is not accepted if it doesn't meets our standards", async () => { - let passed = ""; + let passed: string | boolean = ""; try { await exec( diff --git a/tests/config_file.test.js b/tests/config_file.test.ts similarity index 94% rename from tests/config_file.test.js rename to tests/config_file.test.ts index cfae66d68..6de77b819 100644 --- a/tests/config_file.test.js +++ b/tests/config_file.test.ts @@ -3,6 +3,7 @@ import yaml from "js-yaml"; import util from "util"; import { exec as execCallback } from "child_process"; +import { CrawlerArgs } from "../src/util/argParser"; const exec = util.promisify(execCallback); @@ -30,12 +31,12 @@ test("check yaml config file with seed list is used", async () => { const config = yaml.load( fs.readFileSync("tests/fixtures/crawl-1.yaml", "utf8"), - ); + ) as CrawlerArgs; let foundAllSeeds = true; for (const seed of config.seeds) { - const url = new URL(seed).href; + const url = new URL(seed as string).href; if (!pages.has(url)) { foundAllSeeds = false; } diff --git a/tests/config_stdin.test.js b/tests/config_stdin.test.ts similarity index 80% rename from tests/config_stdin.test.js rename to tests/config_stdin.test.ts index bf705b124..a5d41aefe 100644 --- a/tests/config_stdin.test.js +++ b/tests/config_stdin.test.ts @@ -1,15 +1,16 @@ import child_process from "child_process"; import fs from "fs"; import yaml from "js-yaml"; +import { CrawlerArgs } from "../src/util/argParser"; test("pass config file via stdin", async () => { const configYaml = fs.readFileSync("tests/fixtures/crawl-2.yaml", "utf8"); - const config = yaml.load(configYaml); + const config = yaml.load(configYaml) as CrawlerArgs; try { - const proc = child_process.execSync( + child_process.execSync( "docker run -i -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --config stdin --scopeExcludeRx webrecorder.net/202", - { input: configYaml, stdin: "inherit", encoding: "utf8" }, + { input: configYaml, stdio: ["pipe", "ignore", "ignore"], encoding: "utf8" }, ); //console.log(proc); @@ -35,7 +36,7 @@ test("pass config file via stdin", async () => { let foundAllSeeds = true; for (const seed of config.seeds) { - const url = new URL(seed).href; + const url = new URL(seed as string).href; if (!pages.has(url)) { foundAllSeeds = false; } diff --git a/tests/crawl_overwrite.test.js b/tests/crawl_overwrite.test.ts similarity index 100% rename from tests/crawl_overwrite.test.js rename to tests/crawl_overwrite.test.ts diff --git a/tests/custom-behavior-flow.test.js b/tests/custom-behavior-flow.test.ts similarity index 51% rename from tests/custom-behavior-flow.test.js rename to tests/custom-behavior-flow.test.ts index efd8b0441..ab317c38f 100644 --- a/tests/custom-behavior-flow.test.js +++ b/tests/custom-behavior-flow.test.ts @@ -1,13 +1,11 @@ import child_process from "child_process"; import Redis from "ioredis"; - - -async function sleep(time) { - await new Promise((resolve) => setTimeout(resolve, time)); -} +import { sleep } from "./utils"; test("test pushing behavior logs to redis", async () => { - const child = child_process.exec("docker run -p 36398:6379 -v $PWD/test-crawls:/crawls -v $PWD/tests/custom-behaviors/:/custom-behaviors/ -e CRAWL_ID=behavior-logs-flow-test --rm webrecorder/browsertrix-crawler crawl --debugAccessRedis --url https://webrecorder.net/ --customBehaviors /custom-behaviors/custom-flow.json --scopeType page --logBehaviorsToRedis --pageExtraDelay 20"); + const child = child_process.exec( + "docker run -p 36398:6379 -v $PWD/test-crawls:/crawls -v $PWD/tests/custom-behaviors/:/custom-behaviors/ -e CRAWL_ID=behavior-logs-flow-test --rm webrecorder/browsertrix-crawler crawl --debugAccessRedis --url https://webrecorder.net/ --customBehaviors /custom-behaviors/custom-flow.json --scopeType page --logBehaviorsToRedis --pageExtraDelay 20", + ); let crawlFinished = false; @@ -15,11 +13,16 @@ test("test pushing behavior logs to redis", async () => { crawlFinished = true; }); - const redis = new Redis("redis://127.0.0.1:36398/0", { lazyConnect: true, retryStrategy: () => null }); + const redis = new Redis("redis://127.0.0.1:36398/0", { + lazyConnect: true, + retryStrategy: () => null, + }); await sleep(3000); - await redis.connect({ maxRetriesPerRequest: 50 }); + redis.options.maxRetriesPerRequest = 50; + + await redis.connect(); let customLogLineCount = 0; let done = false; @@ -27,7 +30,7 @@ test("test pushing behavior logs to redis", async () => { while (!crawlFinished) { let res = null; try { - res = await redis.rpop("behavior-logs-flow-test:b"); + res = await redis.rpop("behavior-logs-flow-test:b"); } catch (e) { break; } diff --git a/tests/custom-behavior.test.js b/tests/custom-behavior.test.ts similarity index 68% rename from tests/custom-behavior.test.js rename to tests/custom-behavior.test.ts index 2a7f5fc70..e10ec481a 100644 --- a/tests/custom-behavior.test.js +++ b/tests/custom-behavior.test.ts @@ -1,13 +1,18 @@ -import child_process from "child_process"; +import child_process, { ExecException, type ChildProcess } from "child_process"; import Redis from "ioredis"; +import { ErrorWithStatus, sleep } from "./utils"; -let proc = null; +let proc: ChildProcess | null = null; const DOCKER_HOST_NAME = process.env.DOCKER_HOST_NAME || "host.docker.internal"; const TEST_HOST = `http://${DOCKER_HOST_NAME}:31503`; beforeAll(() => { - proc = child_process.spawn("../../node_modules/.bin/http-server", ["-p", "31503"], {cwd: "tests/custom-behaviors/"}); + proc = child_process.spawn( + "../../node_modules/.bin/http-server", + ["-p", "31503"], + { cwd: "tests/custom-behaviors/" }, + ); }); afterAll(() => { @@ -16,18 +21,22 @@ afterAll(() => { } }); - -async function sleep(time) { - await new Promise((resolve) => setTimeout(resolve, time)); -} - +const spawnWithLoggingOnError = (cmd: string) => { + try { + return child_process.execSync(cmd); + } catch (e) { + // Show log lines + console.error((e as ExecException).stdout?.toString()); + throw e; + } +}; test("test custom behaviors from local filepath", async () => { - const res = child_process.execSync( + const res = spawnWithLoggingOnError( "docker run -v $PWD/test-crawls:/crawls -v $PWD/tests/custom-behaviors/:/custom-behaviors/ webrecorder/browsertrix-crawler crawl --url https://specs.webrecorder.net/ --url https://example-com.webrecorder.net/page --url https://old.webrecorder.net/ --customBehaviors /custom-behaviors/ --scopeType page", ); - const log = res.toString(); + const log = res!.toString(); // custom behavior ran for specs.webrecorder.net expect( @@ -52,7 +61,9 @@ test("test custom behaviors from local filepath", async () => { }); test("test custom behavior from URL", async () => { - const res = child_process.execSync(`docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://old.webrecorder.net/ --customBehaviors ${TEST_HOST}/custom-2.js --scopeType page`); + const res = spawnWithLoggingOnError( + `docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://old.webrecorder.net/ --customBehaviors ${TEST_HOST}/custom-2.js --scopeType page`, + ); const log = res.toString(); @@ -66,7 +77,9 @@ test("test custom behavior from URL", async () => { }); test("test mixed custom behavior sources", async () => { - const res = child_process.execSync(`docker run -v $PWD/test-crawls:/crawls -v $PWD/tests/custom-behaviors/:/custom-behaviors/ webrecorder/browsertrix-crawler crawl --url https://specs.webrecorder.net/ --url https://old.webrecorder.net/ --customBehaviors ${TEST_HOST}/custom-2.js --customBehaviors /custom-behaviors/custom.js --scopeType page`); + const res = spawnWithLoggingOnError( + `docker run -v $PWD/test-crawls:/crawls -v $PWD/tests/custom-behaviors/:/custom-behaviors/ webrecorder/browsertrix-crawler crawl --url https://specs.webrecorder.net/ --url https://old.webrecorder.net/ --customBehaviors ${TEST_HOST}/custom-2.js --customBehaviors /custom-behaviors/custom.js --scopeType page`, + ); const log = res.toString(); @@ -88,8 +101,8 @@ test("test mixed custom behavior sources", async () => { }); test("test custom behaviors from git repo", async () => { - const res = child_process.execSync( - "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://specs.webrecorder.net/ --url https://example-com.webrecorder.net/ --url https://old.webrecorder.net/ --customBehaviors \"git+https://github.com/webrecorder/browsertrix-crawler.git?branch=main&path=tests/custom-behaviors\" --scopeType page", + const res = spawnWithLoggingOnError( + 'docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://specs.webrecorder.net/ --url https://example-com.webrecorder.net/ --url https://old.webrecorder.net/ --customBehaviors "git+https://github.com/webrecorder/browsertrix-crawler.git?branch=main&path=tests/custom-behaviors" --scopeType page', ); const log = res.toString(); @@ -124,7 +137,7 @@ test("test invalid behavior exit", async () => { "docker run -v $PWD/test-crawls:/crawls -v $PWD/tests/invalid-behaviors/:/custom-behaviors/ webrecorder/browsertrix-crawler crawl --url https://example-com.webrecorder.net.webrecorder.net/ --url https://example-com.webrecorder.net/ --url https://old.webrecorder.net/ --customBehaviors /custom-behaviors/invalid-export.js --scopeType page", ); } catch (e) { - status = e.status; + status = (e as ErrorWithStatus).status; } // logger fatal exit code @@ -139,7 +152,7 @@ test("test crawl exits if behavior not fetched from url", async () => { "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://example-com.webrecorder.net --customBehaviors https://webrecorder.net/doesntexist/custombehavior.js --scopeType page", ); } catch (e) { - status = e.status; + status = (e as ErrorWithStatus).status; } // logger fatal exit code @@ -154,7 +167,7 @@ test("test crawl exits if behavior not fetched from git repo", async () => { "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://example-com.webrecorder.net --customBehaviors git+https://github.com/webrecorder/doesntexist --scopeType page", ); } catch (e) { - status = e.status; + status = (e as ErrorWithStatus).status; } // logger fatal exit code @@ -169,7 +182,7 @@ test("test crawl exits if not custom behaviors collected from local path", async "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://example-com.webrecorder.net --customBehaviors /custom-behaviors/doesntexist --scopeType page", ); } catch (e) { - status = e.status; + status = (e as ErrorWithStatus).status; } // logger fatal exit code @@ -177,26 +190,35 @@ test("test crawl exits if not custom behaviors collected from local path", async }); test("test pushing behavior logs to redis", async () => { - child_process.execSync("docker network create crawl"); + spawnWithLoggingOnError("docker network create crawl"); - const redisId = child_process.execSync("docker run --rm --network=crawl -p 36399:6379 --name redis -d redis"); + const redisId = spawnWithLoggingOnError( + "docker run --rm --network=crawl -p 36399:6379 --name redis -d redis", + ); - const child = child_process.exec(`docker run -v $PWD/test-crawls:/crawls -v $PWD/tests/custom-behaviors/:/custom-behaviors/ -e CRAWL_ID=behavior-logs-redis-test --network=crawl --rm webrecorder/browsertrix-crawler crawl --debugAccessRedis --redisStoreUrl redis://redis:6379 --url https://specs.webrecorder.net/ --url https://old.webrecorder.net/ --customBehaviors ${TEST_HOST}/custom-2.js --customBehaviors /custom-behaviors/custom.js --scopeType page --logBehaviorsToRedis`); + const child = child_process.exec( + `docker run -v $PWD/test-crawls:/crawls -v $PWD/tests/custom-behaviors/:/custom-behaviors/ -e CRAWL_ID=behavior-logs-redis-test --network=crawl --rm webrecorder/browsertrix-crawler crawl --debugAccessRedis --redisStoreUrl redis://redis:6379 --url https://specs.webrecorder.net/ --url https://old.webrecorder.net/ --customBehaviors ${TEST_HOST}/custom-2.js --customBehaviors /custom-behaviors/custom.js --scopeType page --logBehaviorsToRedis`, + ); - let resolve = null; - const crawlFinished = new Promise(r => resolve = r); + let resolve: ((value: void | PromiseLike) => void) | null = null; + const crawlFinished = new Promise((r) => (resolve = r)); child.on("exit", function () { - resolve(); + resolve!(); }); await crawlFinished; - const redis = new Redis("redis://127.0.0.1:36399/0", { lazyConnect: true, retryStrategy: () => null }); + const redis = new Redis("redis://127.0.0.1:36399/0", { + lazyConnect: true, + retryStrategy: () => null, + }); await sleep(3000); - await redis.connect({ maxRetriesPerRequest: 50 }); + redis.options.maxRetriesPerRequest = 50; + + await redis.connect(); let customLogLineCount = 0; @@ -208,11 +230,18 @@ test("test pushing behavior logs to redis", async () => { const json = JSON.parse(res); expect(json).toHaveProperty("timestamp"); expect(json.logLevel).toBe("info"); - expect(["behavior", "behaviorScript", "behaviorScriptCustom"]).toContain(json.context) + expect(["behavior", "behaviorScript", "behaviorScriptCustom"]).toContain( + json.context, + ); if (json.context === "behaviorScriptCustom") { - expect(["TestBehavior", "TestBehavior2"]).toContain(json.details.behavior); - expect(["https://specs.webrecorder.net/", "https://old.webrecorder.net/"]).toContain(json.details.page); + expect(["TestBehavior", "TestBehavior2"]).toContain( + json.details.behavior, + ); + expect([ + "https://specs.webrecorder.net/", + "https://old.webrecorder.net/", + ]).toContain(json.details.page); customLogLineCount++; } } diff --git a/tests/custom_driver.test.js b/tests/custom_driver.test.ts similarity index 100% rename from tests/custom_driver.test.js rename to tests/custom_driver.test.ts diff --git a/tests/custom_selector.test.js b/tests/custom_selector.test.ts similarity index 80% rename from tests/custom_selector.test.js rename to tests/custom_selector.test.ts index fff714272..aa7fed8e9 100644 --- a/tests/custom_selector.test.js +++ b/tests/custom_selector.test.ts @@ -1,10 +1,11 @@ import child_process from "child_process"; import fs from "fs"; +import { ErrorWithStatus } from "./utils"; test("test custom selector crawls JS files as pages", async () => { try { child_process.execSync( - "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --collection custom-sel-1 --selectLinks \"script[src]->src\"", + 'docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --collection custom-sel-1 --selectLinks "script[src]->src"', ); } catch (error) { console.log(error); @@ -38,9 +39,7 @@ test("test custom selector crawls JS files as pages", async () => { extraPages.add(url); } - const expectedPages = new Set([ - "https://www.iana.org/", - ]); + const expectedPages = new Set(["https://www.iana.org/"]); const expectedExtraPages = new Set([ "https://www.iana.org/static/_js/jquery.js", @@ -51,15 +50,14 @@ test("test custom selector crawls JS files as pages", async () => { expect(extraPages).toEqual(expectedExtraPages); }); - test("test invalid selector, crawl fails", async () => { let status = 0; try { child_process.execSync( - "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --collection custom-sel-invalid --selectLinks \"script[\"", + 'docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --collection custom-sel-invalid --selectLinks "script["', ); } catch (e) { - status = e.status; + status = (e as ErrorWithStatus).status; } // logger fatal exit code @@ -81,19 +79,17 @@ test("test valid autoclick selector passes validation", async () => { expect(failed).toBe(false); }); - test("test invalid autoclick selector fails validation, crawl fails", async () => { let status = 0; try { child_process.execSync( - "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://example-com.webrecorder.net/ --clickSelector \",\" --scopeType page", + 'docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://example-com.webrecorder.net/ --clickSelector "," --scopeType page', ); } catch (e) { - status = e.status; + status = (e as ErrorWithStatus).status; } // logger fatal exit code expect(status).toBe(17); }); - diff --git a/tests/dedupe-basic.test.js b/tests/dedupe-basic.test.ts similarity index 61% rename from tests/dedupe-basic.test.js rename to tests/dedupe-basic.test.ts index f5e27f512..168507e2e 100644 --- a/tests/dedupe-basic.test.js +++ b/tests/dedupe-basic.test.ts @@ -1,15 +1,11 @@ -import {exec, execSync} from "child_process"; +import { exec, execSync } from "child_process"; import fs from "fs"; import path from "path"; import Redis from "ioredis"; import { WARCParser } from "warcio"; +import { sleep } from "./utils"; -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - - -let redisId; +let redisId: NonSharedBuffer; let numResponses = 0; let sizeSaved = 0; @@ -17,7 +13,9 @@ let sizeSaved = 0; beforeAll(() => { execSync("docker network create dedupe"); - redisId = execSync("docker run --rm --network=dedupe -p 37379:6379 --name dedupe-redis -d redis"); + redisId = execSync( + "docker run --rm --network=dedupe -p 37379:6379 --name dedupe-redis -d redis", + ); }); afterAll(async () => { @@ -28,10 +26,17 @@ afterAll(async () => { execSync("docker network rm dedupe"); }); -function runCrawl(name, {db = 0, limit = 4, wacz = true} = {}) { - fs.rmSync(`./test-crawls/collections/${name}`, { recursive: true, force: true }); +function runCrawl(name: string, { db = 0, limit = 4, wacz = true } = {}) { + fs.rmSync(`./test-crawls/collections/${name}`, { + recursive: true, + force: true, + }); - const crawler = exec(`docker run -v $PWD/test-crawls:/crawls --network=dedupe -e CRAWL_ID=${name} webrecorder/browsertrix-crawler crawl --url https://old.webrecorder.net/ --limit ${limit} --exclude community --collection ${name} --redisDedupeUrl redis://dedupe-redis:6379/${db} ${wacz ? "--generateWACZ" : ""}`); + const crawler = exec( + `docker run -v $PWD/test-crawls:/crawls --network=dedupe -e CRAWL_ID=${name} webrecorder/browsertrix-crawler crawl --url https://old.webrecorder.net/ --limit ${limit} --exclude community --collection ${name} --redisDedupeUrl redis://dedupe-redis:6379/${db} ${ + wacz ? "--generateWACZ" : "" + }`, + ); return new Promise((resolve) => { crawler.on("exit", (code) => { @@ -40,31 +45,37 @@ function runCrawl(name, {db = 0, limit = 4, wacz = true} = {}) { }); } -function loadFirstWARC(name) { +function loadFirstWARC(name: string) { const archiveWarcLists = fs.readdirSync( `test-crawls/collections/${name}/archive`, ); - const warcName = path.join(`test-crawls/collections/${name}/archive`, archiveWarcLists[0]); + const warcName = path.join( + `test-crawls/collections/${name}/archive`, + archiveWarcLists[0], + ); const nodeStream = fs.createReadStream(warcName); const parser = new WARCParser(nodeStream); - return parser; + return parser; } -function deleteFirstWARC(name) { +function deleteFirstWARC(name: string) { const archiveWarcLists = fs.readdirSync( `test-crawls/collections/${name}/archive`, ); - const warcName = path.join(`test-crawls/collections/${name}/archive`, archiveWarcLists[0]); + const warcName = path.join( + `test-crawls/collections/${name}/archive`, + archiveWarcLists[0], + ); fs.unlinkSync(warcName); } -function loadDataPackageRelated(name) { +function loadDataPackageRelated(name: string) { execSync( `unzip test-crawls/collections/${name}/${name}.wacz -d test-crawls/collections/${name}/wacz`, ); @@ -77,15 +88,25 @@ function loadDataPackageRelated(name) { return dataPackageJSON.relation; } -async function redisGetHash(key, db=0) { - const redis = new Redis(`redis://127.0.0.1:37379/${db}`, { lazyConnect: true, retryStrategy: () => null }); +async function redisGetHash(key: string, db = 0) { + const redis = new Redis(`redis://127.0.0.1:37379/${db}`, { + lazyConnect: true, + retryStrategy: () => null, + }); + + redis.options.maxRetriesPerRequest = 50; - await redis.connect({maxRetriesPerRequest: 50}); + await redis.connect(); return await redis.hgetall(key); } -async function checkSizeStats(numUniq, key, db, minSize) { +async function checkSizeStats( + numUniq: number, + key: string, + db: number, + minSize: number, +) { const result = await redisGetHash(key, db); console.log(result); expect(numUniq).toBeLessThan(Number(result.totalUrls)); @@ -96,12 +117,12 @@ async function checkSizeStats(numUniq, key, db, minSize) { } test("check revisit records written on duplicate crawl, same collection, no wacz", async () => { - const collName = "dedupe-test-same-coll"; - expect(await runCrawl(collName, {limit: 1, wacz: false})).toBe(0); + expect(await runCrawl(collName, { limit: 1, wacz: false })).toBe(0); - let statusCode = -1; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const statusCode = -1; let response = 0; let revisit = 0; @@ -120,7 +141,7 @@ test("check revisit records written on duplicate crawl, same collection, no wacz deleteFirstWARC(collName); - expect(await runCrawl(collName, {limit: 1, wacz: false})).toBe(0); + expect(await runCrawl(collName, { limit: 1, wacz: false })).toBe(0); const dupeOrig = loadFirstWARC(collName); @@ -144,7 +165,6 @@ test("check revisit records written on duplicate crawl, same collection, no wacz await checkSizeStats(numResponses, "allcounts", 0, 77000); }); - test("dedupe same collection, with wacz, no external waczs referenced", async () => { const collName = "dedupe-test-same-coll"; @@ -157,13 +177,12 @@ test("dedupe same collection, with wacz, no external waczs referenced", async () }); - test("check revisit records written on duplicate crawl, different collections, with wacz", async () => { + expect(await runCrawl("dedupe-test-orig", { db: 1 })).toBe(0); + expect(await runCrawl("dedupe-test-dupe", { db: 1 })).toBe(0); - expect(await runCrawl("dedupe-test-orig", {db: 1})).toBe(0); - expect(await runCrawl("dedupe-test-dupe", {db: 1})).toBe(0); - - let statusCode = -1; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const statusCode = -1; let response = 0; let revisit = 0; @@ -203,18 +222,23 @@ test("check revisit records written on duplicate crawl, different collections, w sizeSaved = await checkSizeStats(numResponses, "allcounts", 1, 48400000); }); - test("import dupe index, orig then revisits, from single wacz", async () => { - execSync(`docker run -v $PWD/test-crawls:/crawls --network=dedupe webrecorder/browsertrix-crawler indexer --sourceUrl /crawls/collections/dedupe-test-orig/dedupe-test-orig.wacz --sourceCrawlId dedupe-test-orig --redisDedupeUrl redis://dedupe-redis:6379/2`); + execSync( + `docker run -v $PWD/test-crawls:/crawls --network=dedupe webrecorder/browsertrix-crawler indexer --sourceUrl /crawls/collections/dedupe-test-orig/dedupe-test-orig.wacz --sourceCrawlId dedupe-test-orig --redisDedupeUrl redis://dedupe-redis:6379/2`, + ); - const redis = new Redis("redis://127.0.0.1:37379/2", { lazyConnect: true, retryStrategy: () => null }); + const redis = new Redis("redis://127.0.0.1:37379/2", { + lazyConnect: true, + retryStrategy: () => null, + }); - await redis.connect({maxRetriesPerRequest: 50}); -}); + redis.options.maxRetriesPerRequest = 50; + await redis.connect(); +}); test("verify new crawl against imported dupe index has same dupes as dedupe against original", async () => { - expect(await runCrawl("dedupe-test-dupe-2", {db: 2})).toBe(0); + expect(await runCrawl("dedupe-test-dupe-2", { db: 2 })).toBe(0); const dupeOrig = loadFirstWARC("dedupe-test-dupe-2"); @@ -238,44 +262,57 @@ test("verify new crawl against imported dupe index has same dupes as dedupe agai }); test("import dupe index from json, reverse, revisits than orig, from wacz", async () => { - const importJson = { resources: [ { - "name": "dedupe-test-dupe", - "path": "/crawls/collections/dedupe-test-dupe/dedupe-test-dupe.wacz", - "crawlId": "dedupe-test-dupe" + name: "dedupe-test-dupe", + path: "/crawls/collections/dedupe-test-dupe/dedupe-test-dupe.wacz", + crawlId: "dedupe-test-dupe", }, { - "name": "invalid-file", - "path": "/crawls/invalid-file", - "crawlId": "dedupe-test-dupe" + name: "invalid-file", + path: "/crawls/invalid-file", + crawlId: "dedupe-test-dupe", }, { - "name": "dedupe-test-orig", - "path": "/crawls/collections/dedupe-test-orig/dedupe-test-orig.wacz", - "crawlId": "dedupe-test-orig" - } - ] + name: "dedupe-test-orig", + path: "/crawls/collections/dedupe-test-orig/dedupe-test-orig.wacz", + crawlId: "dedupe-test-orig", + }, + ], }; - fs.writeFileSync("./test-crawls/collections/dedupe-test-dupe/import-1.json", JSON.stringify(importJson), "utf-8"); + fs.writeFileSync( + "./test-crawls/collections/dedupe-test-dupe/import-1.json", + JSON.stringify(importJson), + "utf-8", + ); - execSync(`docker run -v $PWD/test-crawls:/crawls --network=dedupe webrecorder/browsertrix-crawler indexer --sourceUrl /crawls/collections/dedupe-test-dupe/import-1.json --redisDedupeUrl redis://dedupe-redis:6379/3`); + execSync( + `docker run -v $PWD/test-crawls:/crawls --network=dedupe webrecorder/browsertrix-crawler indexer --sourceUrl /crawls/collections/dedupe-test-dupe/import-1.json --redisDedupeUrl redis://dedupe-redis:6379/3`, + ); + + const redis = new Redis("redis://127.0.0.1:37379/3", { + lazyConnect: true, + retryStrategy: () => null, + }); - const redis = new Redis("redis://127.0.0.1:37379/3", { lazyConnect: true, retryStrategy: () => null }); + redis.options.maxRetriesPerRequest = 50; - await redis.connect({maxRetriesPerRequest: 50}); + await redis.connect(); expect(await redis.hlen("alldupes")).toBe(numResponses); - const sizeSavedImport = await checkSizeStats(numResponses, "allcounts", 3, 48400000); + const sizeSavedImport = await checkSizeStats( + numResponses, + "allcounts", + 3, + 48400000, + ); expect(sizeSavedImport).toBe(sizeSaved); }); - - test("test requires in datapackage.json of wacz deduped against previous crawl", () => { const res1 = loadDataPackageRelated("dedupe-test-dupe"); @@ -297,5 +334,3 @@ test("test requires in datapackage.json of wacz deduped against import from wacz expect(entry2.size).toBeUndefined(); expect(entry2.hash).toBeUndefined(); }); - - diff --git a/tests/dryrun.test.js b/tests/dryrun.test.ts similarity index 100% rename from tests/dryrun.test.js rename to tests/dryrun.test.ts diff --git a/tests/exclude-redirected.test.js b/tests/exclude-redirected.test.ts similarity index 100% rename from tests/exclude-redirected.test.js rename to tests/exclude-redirected.test.ts diff --git a/tests/extra_hops_depth.test.js b/tests/extra_hops_depth.test.ts similarity index 100% rename from tests/extra_hops_depth.test.js rename to tests/extra_hops_depth.test.ts diff --git a/tests/file_stats.test.js b/tests/file_stats.test.ts similarity index 91% rename from tests/file_stats.test.js rename to tests/file_stats.test.ts index 61042e380..e92808aee 100644 --- a/tests/file_stats.test.js +++ b/tests/file_stats.test.ts @@ -1,5 +1,6 @@ import child_process from "child_process"; import fs from "fs"; +import { sleep } from "./utils"; test("ensure that stats file is modified", async () => { const child = child_process.exec( @@ -12,9 +13,6 @@ test("ensure that stats file is modified", async () => { crawler_exited = true; }); - // helper function to sleep - const sleep = (ms) => new Promise((res) => setTimeout(res, ms)); - // wait for stats file creation up to 30 secs (to not wait indefinitely) let counter = 0; while (!fs.existsSync("test-crawls/progress.json")) { @@ -39,7 +37,9 @@ test("ensure that stats file is modified", async () => { ).mtime; // compare initial and final modification time - const diff = Math.abs(final_mtime - initial_mtime); + const diff = Math.abs( + (final_mtime as unknown as number) - (initial_mtime as unknown as number), + ); expect(diff > 0).toBe(true); }); diff --git a/tests/http-auth.test.js b/tests/http-auth.test.js deleted file mode 100644 index 8c53b2e01..000000000 --- a/tests/http-auth.test.js +++ /dev/null @@ -1,101 +0,0 @@ -import { execSync, spawn } from "child_process"; -import fs from "fs"; -import yaml from "js-yaml"; - -let proc = null; - -const DOCKER_HOST_NAME = process.env.DOCKER_HOST_NAME || "host.docker.internal"; - -beforeAll(() => { - proc = spawn("../../node_modules/.bin/http-server", ["-p", "31501", "--username", "user", "--password", "pass"], {cwd: "./docs/site"}); -}); - -afterAll(() => { - if (proc) { - proc.kill(); - } -}); - -test("run crawl without auth", () => { - let status = 0; - try { - execSync(`docker run --rm webrecorder/browsertrix-crawler crawl --url http://${DOCKER_HOST_NAME}:31501 --limit 2 --failOnFailedSeed`); - } catch (e) { - status = e.status; - } - expect(status).toBe(1); -}); - -test("run crawl with auth", () => { - let status = 0; - try { - execSync(`docker run --rm -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://user:pass@${DOCKER_HOST_NAME}:31501 --limit 2 --failOnFailedSeed --collection http-auth-test`); - } catch (e) { - status = e.status; - } - - expect(status).toBe(0); - - expect(fs - .readFileSync( - "test-crawls/collections/http-auth-test/pages/pages.jsonl", - "utf8", - ) - .trim() - .split("\n") - .length).toBe(2); - - expect(fs - .readFileSync( - "test-crawls/collections/http-auth-test/pages/extraPages.jsonl", - "utf8", - ) - .trim() - .split("\n") - .length).toBe(2); - -}); - -test("run crawl with auth config.yaml", () => { - const config = { - seeds: [{ - url: `http://${DOCKER_HOST_NAME}:31501`, - auth: "user:pass" - }], - limit: "2", - collection: "http-auth-test-2", - failOnFailedSeed: "true" - } - - const configYaml = yaml.dump(config); - - let status = 0; - try { - execSync("docker run -i --rm -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --config stdin", - { input: configYaml, stdin: "inherit", encoding: "utf8" }); - - } catch (e) { - console.log(e); - status = e.status; - } - - expect(status).toBe(0); - - expect(fs - .readFileSync( - "test-crawls/collections/http-auth-test-2/pages/pages.jsonl", - "utf8", - ) - .trim() - .split("\n") - .length).toBe(2); - - expect(fs - .readFileSync( - "test-crawls/collections/http-auth-test-2/pages/extraPages.jsonl", - "utf8", - ) - .trim() - .split("\n") - .length).toBe(2); -}); diff --git a/tests/http-auth.test.ts b/tests/http-auth.test.ts new file mode 100644 index 000000000..98f1ee289 --- /dev/null +++ b/tests/http-auth.test.ts @@ -0,0 +1,116 @@ +import { ChildProcessWithoutNullStreams, execSync, spawn } from "child_process"; +import fs from "fs"; +import yaml from "js-yaml"; +import { ErrorWithStatus } from "./utils"; + +let proc: ChildProcessWithoutNullStreams | null = null; + +const DOCKER_HOST_NAME = process.env.DOCKER_HOST_NAME || "host.docker.internal"; + +beforeAll(() => { + proc = spawn( + "../../node_modules/.bin/http-server", + ["-p", "31501", "--username", "user", "--password", "pass"], + { cwd: "./docs/site" }, + ); +}); + +afterAll(() => { + if (proc) { + proc.kill(); + } +}); + +test("run crawl without auth", () => { + let status = 0; + try { + execSync( + `docker run --rm webrecorder/browsertrix-crawler crawl --url http://${DOCKER_HOST_NAME}:31501 --limit 2 --failOnFailedSeed`, + ); + } catch (e) { + status = (e as ErrorWithStatus).status; + } + expect(status).toBe(1); +}); + +test("run crawl with auth", () => { + let status = 0; + try { + execSync( + `docker run --rm -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url http://user:pass@${DOCKER_HOST_NAME}:31501 --limit 2 --failOnFailedSeed --collection http-auth-test`, + ); + } catch (e) { + status = (e as ErrorWithStatus).status; + } + + expect(status).toBe(0); + + expect( + fs + .readFileSync( + "test-crawls/collections/http-auth-test/pages/pages.jsonl", + "utf8", + ) + .trim() + .split("\n").length, + ).toBe(2); + + expect( + fs + .readFileSync( + "test-crawls/collections/http-auth-test/pages/extraPages.jsonl", + "utf8", + ) + .trim() + .split("\n").length, + ).toBe(2); +}); + +test("run crawl with auth config.yaml", () => { + const config = { + seeds: [ + { + url: `http://${DOCKER_HOST_NAME}:31501`, + auth: "user:pass", + }, + ], + limit: "2", + collection: "http-auth-test-2", + failOnFailedSeed: "true", + }; + + const configYaml = yaml.dump(config); + + let status = 0; + try { + execSync( + "docker run -i --rm -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --config stdin", + { input: configYaml, stdio: ["pipe", "ignore", "ignore"], encoding: "utf8" }, + ); + } catch (e) { + console.log(e); + status = (e as ErrorWithStatus).status; + } + + expect(status).toBe(0); + + expect( + fs + .readFileSync( + "test-crawls/collections/http-auth-test-2/pages/pages.jsonl", + "utf8", + ) + .trim() + .split("\n").length, + ).toBe(2); + + expect( + fs + .readFileSync( + "test-crawls/collections/http-auth-test-2/pages/extraPages.jsonl", + "utf8", + ) + .trim() + .split("\n").length, + ).toBe(2); +}); diff --git a/tests/lang-code.test.js b/tests/lang-code.test.js deleted file mode 100644 index a99387fe3..000000000 --- a/tests/lang-code.test.js +++ /dev/null @@ -1,23 +0,0 @@ -import { execSync } from "child_process"; - -test("run crawl with invalid lang", () => { - let status = 0; - try { - execSync(`docker run --rm webrecorder/browsertrix-crawler crawl --url https://webrecorder.net/feed.xml --lang e --limit 1`); - } catch (e) { - status = e.status; - } - expect(status).toBe(17); -}); - -test("run crawl with valid lang", () => { - let status = 0; - try { - execSync(`docker run --rm webrecorder/browsertrix-crawler crawl --url https://webrecorder.net/feed.xml --lang en --limit 1`); - } catch (e) { - status = e.status; - } - expect(status).toBe(0); -}); - - diff --git a/tests/lang-code.test.ts b/tests/lang-code.test.ts new file mode 100644 index 000000000..716997837 --- /dev/null +++ b/tests/lang-code.test.ts @@ -0,0 +1,26 @@ +import { execSync } from "child_process"; +import { ErrorWithStatus } from "./utils"; + +test("run crawl with invalid lang", () => { + let status = 0; + try { + execSync( + `docker run --rm webrecorder/browsertrix-crawler crawl --url https://webrecorder.net/feed.xml --lang e --limit 1`, + ); + } catch (e) { + status = (e as ErrorWithStatus).status; + } + expect(status).toBe(17); +}); + +test("run crawl with valid lang", () => { + let status = 0; + try { + execSync( + `docker run --rm webrecorder/browsertrix-crawler crawl --url https://webrecorder.net/feed.xml --lang en --limit 1`, + ); + } catch (e) { + status = (e as ErrorWithStatus).status; + } + expect(status).toBe(0); +}); diff --git a/tests/limit_reached.test.js b/tests/limit_reached.test.ts similarity index 88% rename from tests/limit_reached.test.js rename to tests/limit_reached.test.ts index 27ec20ead..2d7f331d5 100644 --- a/tests/limit_reached.test.js +++ b/tests/limit_reached.test.ts @@ -1,6 +1,6 @@ import fs from "fs"; import util from "util"; -import { exec as execCallback, execSync } from "child_process"; +import { exec as execCallback, ExecException, execSync } from "child_process"; const exec = util.promisify(execCallback); @@ -25,7 +25,7 @@ test("ensure crawl fails if failOnFailedLimit is reached", async () => { "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://old.webrecorder.net/will404 --url https://specs.webrecorder.net --failOnInvalidStatus --failOnFailedLimit 1 --limit 10 --collection faillimitreached", ); } catch (error) { - expect(error.code).toEqual(12); + expect((error as ExecException).code).toEqual(12); passed = false; } expect(passed).toBe(false); @@ -38,7 +38,7 @@ test("ensure crawl fails if timeLimit is reached", async () => { "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://specs.webrecorder.net --failOnInvalidStatus --timeLimit 1 --limit 2 --collection failontimelimitreached", ); } catch (error) { - expect(error.code).toEqual(15); + expect((error as ExecException).code).toEqual(15); passed = false; } expect(passed).toBe(false); @@ -51,7 +51,7 @@ test("ensure crawl fails if sizeLimit is reached", async () => { "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://specs.webrecorder.net --failOnInvalidStatus --sizeLimit 1 --limit 2 --collection failonsizelimitreached", ); } catch (error) { - expect(error.code).toEqual(14); + expect((error as ExecException).code).toEqual(14); passed = false; } expect(passed).toBe(false); @@ -64,7 +64,7 @@ test("ensure crawl fails if diskUtilizationLimit is reached", async () => { "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://specs.webrecorder.net --failOnInvalidStatus --diskUtilization 1 --limit 2 --collection failonsizelimitreached", ); } catch (error) { - expect(error.code).toEqual(16); + expect((error as ExecException).code).toEqual(16); passed = false; } expect(passed).toBe(false); diff --git a/tests/log_filtering.test.js b/tests/log_filtering.test.ts similarity index 94% rename from tests/log_filtering.test.js rename to tests/log_filtering.test.ts index 09d2b14e5..b0b33f478 100644 --- a/tests/log_filtering.test.js +++ b/tests/log_filtering.test.ts @@ -2,7 +2,7 @@ import child_process from "child_process"; import fs from "fs"; import path from "path"; -function jsonLinesToArray(string) { +function jsonLinesToArray(string: string) { return string .split("\n") .filter((line) => { @@ -24,7 +24,7 @@ test("ensure crawl run with log options passes", async () => { test("check that log files exist and were filtered according to options", () => { const logDir = "test-crawls/collections/wr-specs-logs/logs/"; - const logFiles = []; + const logFiles: string[] = []; fs.readdirSync(logDir).forEach((file) => { if (file.endsWith(".log")) { logFiles.push(path.join(logDir, file)); diff --git a/tests/mult_url_crawl_with_favicon.test.js b/tests/mult_url_crawl_with_favicon.test.ts similarity index 90% rename from tests/mult_url_crawl_with_favicon.test.js rename to tests/mult_url_crawl_with_favicon.test.ts index ac60a6c71..5abcf9914 100644 --- a/tests/mult_url_crawl_with_favicon.test.js +++ b/tests/mult_url_crawl_with_favicon.test.ts @@ -1,8 +1,6 @@ import child_process from "child_process"; import fs from "fs"; - -const doValidate = process.argv.filter((x) => x.startsWith('-validate'))[0]; -const testIf = (condition, ...args) => condition ? test(...args) : test.skip(...args); +import { doValidate, testIf } from "./utils"; test("ensure multi url crawl run with docker run passes", async () => { child_process.execSync( diff --git a/tests/multi-instance-crawl.test.js b/tests/multi-instance-crawl.test.ts similarity index 61% rename from tests/multi-instance-crawl.test.js rename to tests/multi-instance-crawl.test.ts index 9728554c9..25b70cc63 100644 --- a/tests/multi-instance-crawl.test.js +++ b/tests/multi-instance-crawl.test.ts @@ -1,22 +1,26 @@ -import {exec, execSync} from "child_process"; +import { exec, execSync } from "child_process"; import fs from "fs"; import { Redis } from "ioredis"; +import { sleep } from "./utils"; -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - - -let redisId; -let crawler1, crawler2; +let redisId: NonSharedBuffer; +let crawler1: Promise, crawler2: Promise; beforeAll(() => { - fs.rmSync("./test-crawls/collections/shared-crawler-1", { recursive: true, force: true }); - fs.rmSync("./test-crawls/collections/shared-crawler-2", { recursive: true, force: true }); + fs.rmSync("./test-crawls/collections/shared-crawler-1", { + recursive: true, + force: true, + }); + fs.rmSync("./test-crawls/collections/shared-crawler-2", { + recursive: true, + force: true, + }); execSync("docker network create crawl"); - redisId = execSync("docker run --rm --network=crawl -p 37379:6379 --name redis -d redis"); + redisId = execSync( + "docker run --rm --network=crawl -p 37379:6379 --name redis -d redis", + ); crawler1 = runCrawl("crawler-1"); crawler2 = runCrawl("crawler-2"); @@ -32,10 +36,12 @@ afterAll(async () => { execSync("docker network rm crawl"); }); -function runCrawl(name) { - const crawler = exec(`docker run --rm -v $PWD/test-crawls:/crawls --network=crawl --hostname=${name} webrecorder/browsertrix-crawler crawl --url https://old.webrecorder.net/ --limit 4 --exclude community --collection shared-${name} --crawlId testcrawl --redisStoreUrl redis://redis:6379`); +function runCrawl(name: string) { + const crawler = exec( + `docker run --rm -v $PWD/test-crawls:/crawls --network=crawl --hostname=${name} webrecorder/browsertrix-crawler crawl --url https://old.webrecorder.net/ --limit 4 --exclude community --collection shared-${name} --crawlId testcrawl --redisStoreUrl redis://redis:6379`, + ); - return new Promise((resolve) => { + return new Promise((resolve) => { crawler.on("exit", (code) => { resolve(code); }); @@ -43,11 +49,16 @@ function runCrawl(name) { } test("run crawlers with external redis", async () => { - const redis = new Redis("redis://127.0.0.1:37379/0", { lazyConnect: true, retryStrategy: () => null }); + const redis = new Redis("redis://127.0.0.1:37379/0", { + lazyConnect: true, + retryStrategy: () => null, + }); await sleep(3000); - await redis.connect({ maxRetriesPerRequest: 50 }); + redis.options.maxRetriesPerRequest = 50; + + await redis.connect(); let count = 0; @@ -66,18 +77,15 @@ test("run crawlers with external redis", async () => { throw e; } } - }); - test("finish crawls successfully", async () => { const res = await Promise.allSettled([crawler1, crawler2]); - expect(res[0].value).toBe(0); - expect(res[1].value).toBe(0); + expect(res[0].status === "fulfilled" ? res[0].value : null).toBe(0); + expect(res[1].status === "fulfilled" ? res[1].value : null).toBe(0); }, 180000); test("ensure correct number of pages", () => { - expect( fs.existsSync("test-crawls/collections/shared-crawler-1/pages/pages.jsonl"), ).toBe(true); @@ -107,13 +115,16 @@ test("ensure correct number of pages", () => { }); test("ensure correct number of extraPages", () => { - expect( - fs.existsSync("test-crawls/collections/shared-crawler-1/pages/extraPages.jsonl"), + fs.existsSync( + "test-crawls/collections/shared-crawler-1/pages/extraPages.jsonl", + ), ).toBe(true); expect( - fs.existsSync("test-crawls/collections/shared-crawler-2/pages/extraPages.jsonl"), + fs.existsSync( + "test-crawls/collections/shared-crawler-2/pages/extraPages.jsonl", + ), ).toBe(true); const pages_1 = fs diff --git a/tests/non-html-crawl.test.js b/tests/non-html-crawl.test.ts similarity index 85% rename from tests/non-html-crawl.test.js rename to tests/non-html-crawl.test.ts index b015cc0e3..765f4d68c 100644 --- a/tests/non-html-crawl.test.js +++ b/tests/non-html-crawl.test.ts @@ -11,7 +11,7 @@ const XML_REDIR = "https://www.webrecorder.net/feed.xml"; test("PDF: ensure pdf is crawled", () => { child_process.execSync( - `docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url "${PDF}" --collection crawl-pdf` + `docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url "${PDF}" --collection crawl-pdf`, ); }); @@ -20,13 +20,16 @@ test("PDF: check that individual WARCs have PDF written as 200 response", async "test-crawls/collections/crawl-pdf/archive", ); - const warcName = path.join("test-crawls/collections/crawl-pdf/archive", archiveWarcLists[0]); + const warcName = path.join( + "test-crawls/collections/crawl-pdf/archive", + archiveWarcLists[0], + ); const nodeStream = fs.createReadStream(warcName); const parser = new WARCParser(nodeStream); - let statusCode = -1; + let statusCode: string | number | undefined = -1; for await (const record of parser) { if (record.warcType !== "response") { @@ -34,7 +37,7 @@ test("PDF: check that individual WARCs have PDF written as 200 response", async } if (record.warcTargetURI === PDF) { - statusCode = record.httpHeaders.statusCode; + statusCode = record.httpHeaders!.statusCode; } } @@ -43,7 +46,7 @@ test("PDF: check that individual WARCs have PDF written as 200 response", async test("PDF: ensure pdf with redirect is crawled", () => { child_process.execSync( - `docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url "${PDF_HTTP}" --collection crawl-pdf --generateCDX` + `docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url "${PDF_HTTP}" --collection crawl-pdf --generateCDX`, ); }); @@ -52,12 +55,8 @@ test("PDF: check that the pages.jsonl file entry contains status code and mime t fs.existsSync("test-crawls/collections/crawl-pdf/pages/pages.jsonl"), ).toBe(true); - const pages = fs - .readFileSync( - "test-crawls/collections/crawl-pdf/pages/pages.jsonl", - "utf8", - ) + .readFileSync("test-crawls/collections/crawl-pdf/pages/pages.jsonl", "utf8") .trim() .split("\n"); @@ -83,7 +82,9 @@ test("PDF: check that CDX contains data from two crawls: one pdf 200, one 301 an ); const lines = filedata.trim().split("\n"); - const cdxj = lines.map(line => JSON.parse(line.split(" ").slice(2).join(" "))).sort((a, b) => a.url < b.url ? -1 : 1); + const cdxj = lines + .map((line) => JSON.parse(line.split(" ").slice(2).join(" "))) + .sort((a, b) => (a.url < b.url ? -1 : 1)); expect(cdxj.length).toBe(5); @@ -108,7 +109,7 @@ test("PDF: check that CDX contains data from two crawls: one pdf 200, one 301 an test("XML: ensure with and without redirect is crawled", () => { child_process.execSync( - `docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url "${XML}" --url "${XML_REDIR}" --collection crawl-xml --generateCDX` + `docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url "${XML}" --url "${XML_REDIR}" --collection crawl-xml --generateCDX`, ); }); @@ -117,12 +118,8 @@ test("XML: check pages.jsonl file entry contains status code and mime type", () fs.existsSync("test-crawls/collections/crawl-xml/pages/pages.jsonl"), ).toBe(true); - const pages = fs - .readFileSync( - "test-crawls/collections/crawl-xml/pages/pages.jsonl", - "utf8", - ) + .readFileSync("test-crawls/collections/crawl-xml/pages/pages.jsonl", "utf8") .trim() .split("\n"); @@ -148,7 +145,9 @@ test("XML: check that CDX contains one xml 200, one 301 and one 200, two pageinf ); const lines = filedata.trim().split("\n"); - const cdxj = lines.map(line => JSON.parse(line.split(" ").slice(2).join(" "))).sort((a, b) => a.url < b.url ? -1 : 1); + const cdxj = lines + .map((line) => JSON.parse(line.split(" ").slice(2).join(" "))) + .sort((a, b) => (a.url < b.url ? -1 : 1)); expect(cdxj.length).toBe(5); @@ -167,5 +166,3 @@ test("XML: check that CDX contains one xml 200, one 301 and one 200, two pageinf expect(cdxj[4].url).toBe("urn:pageinfo:" + XML_REDIR); expect(cdxj[4].mime).toBe("application/json"); }); - - diff --git a/tests/norm-test.test.js b/tests/norm-test.test.ts similarity index 56% rename from tests/norm-test.test.js rename to tests/norm-test.test.ts index 9534df7bd..7daf8c5c6 100644 --- a/tests/norm-test.test.js +++ b/tests/norm-test.test.ts @@ -3,31 +3,36 @@ import fs from "fs"; import path from "path"; import { WARCParser } from "warcio"; -function loadFirstWARC(name) { +function loadFirstWARC(name: string) { const archiveWarcLists = fs.readdirSync( `test-crawls/collections/${name}/archive`, ); - const warcName = path.join(`test-crawls/collections/${name}/archive`, archiveWarcLists[0]); + const warcName = path.join( + `test-crawls/collections/${name}/archive`, + archiveWarcLists[0], + ); const nodeStream = fs.createReadStream(warcName); const parser = new WARCParser(nodeStream); - return parser; + return parser; } - test("same URL with same query args, but different sort order", async () => { - fs.rmSync("./test-crawls/collections/norm-test-1", { recursive: true, force: true }); + fs.rmSync("./test-crawls/collections/norm-test-1", { + recursive: true, + force: true, + }); - execSync("docker run --rm -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url 'https://example-com.webrecorder.net/?B=1&A=2' --url 'https://example-com.webrecorder.net/?A=2&B=1' --collection norm-test-1"); + execSync( + "docker run --rm -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url 'https://example-com.webrecorder.net/?B=1&A=2' --url 'https://example-com.webrecorder.net/?A=2&B=1' --collection norm-test-1", + ); const parser = loadFirstWARC("norm-test-1"); - const uris = [ - "https://example-com.webrecorder.net/?B=1&A=2", - ]; + const uris = ["https://example-com.webrecorder.net/?B=1&A=2"]; let count = 0; diff --git a/tests/pageinfo-records.test.js b/tests/pageinfo-records.test.ts similarity index 92% rename from tests/pageinfo-records.test.js rename to tests/pageinfo-records.test.ts index fc6b61685..0334550fa 100644 --- a/tests/pageinfo-records.test.js +++ b/tests/pageinfo-records.test.ts @@ -24,9 +24,14 @@ test("run warc and ensure pageinfo records contain the correct resources", async let foundInvalid = false; for await (const record of parser) { - if (record.warcType === "response" && - (record.warcTargetURI === "https://old.webrecorder.net/" || record.warcTargetURI === "https://old.webrecorder.net/about")) { - expect(record.warcHeaders.headers.get("WARC-Protocol")).toBe("h2, tls/1.3"); + if ( + record.warcType === "response" && + (record.warcTargetURI === "https://old.webrecorder.net/" || + record.warcTargetURI === "https://old.webrecorder.net/about") + ) { + expect(record.warcHeaders.headers.get("WARC-Protocol")).toBe( + "h2, tls/1.3", + ); } if ( @@ -62,7 +67,7 @@ test("run warc and ensure pageinfo records contain the correct resources", async expect(foundInvalid).toBe(true); }); -function validateResourcesIndex(json) { +function validateResourcesIndex(json: Record) { expect(json).toHaveProperty("pageid"); expect(json).toHaveProperty("url"); expect(json).toHaveProperty("ts"); @@ -89,11 +94,12 @@ function validateResourcesIndex(json) { status: 200, type: "image", }, - "https://old.webrecorder.net/assets/brand/browsertrixcrawler-icon-color.svg": { - mime: "image/svg+xml", - status: 200, - type: "image", - }, + "https://old.webrecorder.net/assets/brand/browsertrixcrawler-icon-color.svg": + { + mime: "image/svg+xml", + status: 200, + type: "image", + }, "https://old.webrecorder.net/assets/brand/replaywebpage-icon-color.svg": { mime: "image/svg+xml", status: 200, @@ -130,7 +136,7 @@ function validateResourcesIndex(json) { }); } -function validateResourcesAbout(json) { +function validateResourcesAbout(json: Record) { expect(json).toHaveProperty("pageid"); expect(json).toHaveProperty("url"); expect(json).toHaveProperty("ts"); @@ -168,7 +174,7 @@ function validateResourcesAbout(json) { }); } -function validateResourcesInvalid(json) { +function validateResourcesInvalid(json: Record) { expect(json).toHaveProperty("pageid"); expect(json).toHaveProperty("url"); expect(json).toHaveProperty("urls"); diff --git a/tests/profiles.test.js b/tests/profiles.test.ts similarity index 78% rename from tests/profiles.test.js rename to tests/profiles.test.ts index 1d456c5b8..cc4d7c2e2 100644 --- a/tests/profiles.test.js +++ b/tests/profiles.test.ts @@ -1,60 +1,61 @@ -import { execSync } from "child_process"; +import { exec as execCallback, type ExecException } from "child_process"; import fs from "node:fs"; +import { promisify } from "node:util"; +const exec = promisify(execCallback); test("run with invalid profile, fail", async () => { - let status = 0; + let status: number | undefined = 0; try { - await execSync( + await exec( "docker run -v $PWD/test-crawls:/crawls -v $PWD/tests/fixtures:/tests/fixtures webrecorder/browsertrix-crawler crawl --collection profile-0 --url https://example-com.webrecorder.net/ --url https://old.webrecorder.net/ --url https://old.webrecorder.net/about --limit 1 --profile /tests/fixtures/invalid.tar.gz", ); } catch (error) { - status = error.status; + status = (error as ExecException).code; } expect(status).toBe(17); }); test("start with no profile", async () => { - let status = 0; + let status: number | undefined = 0; try { - await execSync( + await exec( "docker run -v $PWD/test-crawls:/crawls -v $PWD/tests/fixtures:/tests/fixtures webrecorder/browsertrix-crawler crawl --collection profile-1 --url https://example-com.webrecorder.net/ --url https://old.webrecorder.net/ --url https://old.webrecorder.net/about --limit 1", ); } catch (error) { - status = error.status; + status = (error as ExecException).code; } expect(status).toBe(0); }); test("resume same crawl, but with invalid profile, not valid as no previous valid profile", async () => { - let status = 0; + let status: number | undefined = 0; try { - await execSync( + await exec( "docker run -v $PWD/test-crawls:/crawls -v $PWD/tests/fixtures:/tests/fixtures webrecorder/browsertrix-crawler crawl --collection profile-1 --url https://example-com.webrecorder.net/ --url https://old.webrecorder.net/ --url https://old.webrecorder.net/about --limit 1 --profile /tests/fixtures/invalid.tar.gz", ); } catch (error) { - status = error.status; + status = (error as ExecException).code; } expect(status).toBe(17); }); - test("start with valid profile", async () => { - let status = 0; + let status: number | undefined = 0; try { - await execSync( + await exec( "docker run -v $PWD/test-crawls:/crawls -v $PWD/tests/fixtures:/tests/fixtures webrecorder/browsertrix-crawler crawl --collection profile-2 --url https://example-com.webrecorder.net/ --url https://old.webrecorder.net/ --url https://old.webrecorder.net/about --limit 1 --scopeType page --profile /tests/fixtures/sample-profile.tar.gz", ); } catch (error) { - status = error.status; + status = (error as ExecException).code; } expect(status).toBe(0); - let crawled_pages = fs.readFileSync( + const crawled_pages = fs.readFileSync( "test-crawls/collections/profile-2/pages/pages.jsonl", "utf8", ); @@ -63,20 +64,19 @@ test("start with valid profile", async () => { expect(crawled_pages.split("\n").length === 2); }); - test("resume same crawl, ignore invalid profile, use existing, finish crawl", async () => { - let status = 0; + let status: number | undefined = 0; try { - await execSync( + await exec( "docker run -v $PWD/test-crawls:/crawls -v $PWD/tests/fixtures:/tests/fixtures webrecorder/browsertrix-crawler crawl --collection profile-2 --url https://example-com.webrecorder.net/ --url https://old.webrecorder.net/ --url https://old.webrecorder.net/about --scopeType page --profile /tests/fixtures/invalid.tar.gz", ); } catch (error) { - status = error.status; + status = (error as ExecException).code; } expect(status).toBe(0); - let crawled_pages = fs.readFileSync( + const crawled_pages = fs.readFileSync( "test-crawls/collections/profile-1/pages/pages.jsonl", "utf8", ); @@ -84,4 +84,3 @@ test("resume same crawl, ignore invalid profile, use existing, finish crawl", as // crawled 3 pages expect(crawled_pages.split("\n").length === 4); }); - diff --git a/tests/proxy.test.js b/tests/proxy.test.js deleted file mode 100644 index 811ed325b..000000000 --- a/tests/proxy.test.js +++ /dev/null @@ -1,196 +0,0 @@ -import { execSync, exec } from "child_process"; - -import { getSafeProxyString } from "../dist/util/proxy.js"; - -const sleep = (ms) => new Promise((res) => setTimeout(res, ms)); - -const PROXY_IMAGE = "tarampampam/3proxy:1.9.1"; -const SOCKS_PORT = "1080"; -const HTTP_PORT = "3128"; -const WRONG_PORT = "33130"; - -const PROXY_EXIT_CODE = 21; - -const SSH_PROXY_IMAGE = "linuxserver/openssh-server" - -const PDF = "https://specs.webrecorder.net/wacz/1.1.1/wacz-2021.pdf"; -const HTML = "https://old.webrecorder.net/"; - -const extraArgs = "--limit 1 --failOnFailedSeed --timeout 10 --logging debug"; - -let proxyAuthId; -let proxyNoAuthId; -let proxySSHId; - -beforeAll(() => { - execSync("docker network create proxy-test-net"); - - proxyAuthId = execSync(`docker run -e PROXY_LOGIN=user -e PROXY_PASSWORD=passw0rd -d --rm --network=proxy-test-net --name proxy-with-auth ${PROXY_IMAGE}`, {encoding: "utf-8"}); - - proxyNoAuthId = execSync(`docker run -d --rm --network=proxy-test-net --name proxy-no-auth ${PROXY_IMAGE}`, {encoding: "utf-8"}); - - proxySSHId = execSync(`docker run -d --rm -e DOCKER_MODS=linuxserver/mods:openssh-server-ssh-tunnel -e USER_NAME=user -e PUBLIC_KEY_FILE=/keys/proxy-key.pub -v $PWD/tests/fixtures/proxies/proxy-key.pub:/keys/proxy-key.pub --network=proxy-test-net --name ssh-proxy ${SSH_PROXY_IMAGE}`); -}); - -afterAll(async () => { - execSync(`docker kill -s SIGINT ${proxyAuthId}`); - execSync(`docker kill -s SIGINT ${proxyNoAuthId}`); - execSync(`docker kill -s SIGINT ${proxySSHId}`); - await sleep(5000); - execSync("docker network rm proxy-test-net"); -}); - -describe("socks5 + https proxy tests", () => { - for (const scheme of ["socks5", "http"]) { - const port = scheme === "socks5" ? SOCKS_PORT : HTTP_PORT; - - for (const type of ["HTML page", "PDF"]) { - - const url = type === "PDF" ? PDF : HTML; - - test(`${scheme} proxy, ${type}, no auth`, () => { - let status = 0; - - try { - execSync(`docker run -e PROXY_SERVER=${scheme}://proxy-no-auth:${port} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${url} ${extraArgs}`, {encoding: "utf-8"}); - } catch (e) { - status = e.status; - } - expect(status).toBe(0); - }); - - test(`${scheme} proxy, ${type}, with auth`, () => { - let status = 0; - - try { - execSync(`docker run -e PROXY_SERVER=${scheme}://user:passw0rd@proxy-with-auth:${port} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${url} ${extraArgs}`, {encoding: "utf-8"}); - } catch (e) { - status = e.status; - } - // auth supported only for SOCKS5 - expect(status).toBe(scheme === "socks5" ? 0 : PROXY_EXIT_CODE); - }); - - test(`${scheme} proxy, ${type}, wrong auth`, () => { - let status = 0; - - try { - execSync(`docker run -e PROXY_SERVER=${scheme}://user:passw1rd@proxy-with-auth:${port} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${url} ${extraArgs}`, {encoding: "utf-8"}); - } catch (e) { - status = e.status; - } - expect(status).toBe(PROXY_EXIT_CODE); - }); - - test(`${scheme} proxy, ${type}, wrong protocol`, () => { - let status = 0; - - try { - execSync(`docker run -e PROXY_SERVER=${scheme}://user:passw1rd@proxy-with-auth:${scheme === "socks5" ? HTTP_PORT : SOCKS_PORT} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${url} ${extraArgs}`, {encoding: "utf-8"}); - } catch (e) { - status = e.status; - } - // wrong protocol (socks5 for http) causes connection to hang, causes a timeout, so just errors with 1 - expect(status === PROXY_EXIT_CODE || status === 1).toBe(true); - }); - } - - test(`${scheme} proxy, proxy missing error`, () => { - let status = 0; - - try { - execSync(`docker run -e PROXY_SERVER=${scheme}://proxy-no-auth:${WRONG_PORT} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${HTML} ${extraArgs}`, {encoding: "utf-8"}); - } catch (e) { - status = e.status; - } - expect(status).toBe(PROXY_EXIT_CODE); - }); - } -}); - - -test("http proxy, PDF, separate env vars", () => { - execSync(`docker run -e PROXY_HOST=proxy-no-auth -e PROXY_PORT=${HTTP_PORT} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${PDF} ${extraArgs}`, {encoding: "utf-8"}); -}); - -test("http proxy set, but not running, separate env vars", () => { - let status = 0; - - try { - execSync(`docker run -e PROXY_HOST=proxy-no-auth -e PROXY_PORT=${WRONG_PORT} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${PDF} ${extraArgs}`, {encoding: "utf-8"}); - } catch (e) { - status = e.status; - } - expect(status).toBe(PROXY_EXIT_CODE); -}); - -test("http proxy set, but not running, cli arg", () => { - let status = 0; - - try { - execSync(`docker run --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --proxyServer http://proxy-no-auth:${WRONG_PORT} --url ${PDF} ${extraArgs}`, {encoding: "utf-8"}); - } catch (e) { - status = e.status; - } - expect(status).toBe(PROXY_EXIT_CODE); -}); - - -test("ssh socks proxy with custom user", () => { - execSync(`docker run --rm --network=proxy-test-net -v $PWD/tests/fixtures/proxies/proxy-key:/keys/proxy-key webrecorder/browsertrix-crawler crawl --proxyServer ssh://user@ssh-proxy:2222 --sshProxyPrivateKeyFile /keys/proxy-key --url ${HTML} ${extraArgs}`, {encoding: "utf-8"}); -}); - - -test("ssh socks proxy, wrong user", () => { - let status = 0; - - try { - execSync(`docker run --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --proxyServer ssh://ssh-proxy:2222 --url ${HTML} ${extraArgs}`, {encoding: "utf-8"}); - } catch (e) { - status = e.status; - } - expect(status).toBe(PROXY_EXIT_CODE); -}); - - -test("ensure logged proxy string does not include any credentials", () => { - const testParams = [ - // [input, expectedOutput] - ["socks5://username:password@proxy-host.example.com:9001", "socks5://proxy-host.example.com:9001"], - ["socks5://username@proxy-host.example.com:9001", "socks5://proxy-host.example.com:9001"], - ["socks5://path-to-proxy-host.example.com:9001", "socks5://path-to-proxy-host.example.com:9001"], - ["ssh://localhost:9700", "ssh://localhost:9700"] - ]; - for (const testParamSet of testParams) { - expect(getSafeProxyString(testParamSet[0])).toEqual(testParamSet[1]); - } -}); - - -test("proxy with config file, wrong auth or no match", () => { - let status = 0; - try { - execSync(`docker run --rm --network=proxy-test-net -v $PWD/tests/fixtures/proxies/:/proxies/ webrecorder/browsertrix-crawler crawl --proxyServerConfig /proxies/proxy-test-bad-auth.pac --url ${HTML} ${extraArgs}`, {encoding: "utf-8"}); - } catch (e) { - status = e.status; - } - expect(status).toBe(PROXY_EXIT_CODE); - - // success, no match for PDF - execSync(`docker run --rm --network=proxy-test-net -v $PWD/tests/fixtures/proxies/:/proxies/ webrecorder/browsertrix-crawler crawl --proxyServerConfig /proxies/proxy-test-bad-auth.pac --url ${PDF} ${extraArgs}`, {encoding: "utf-8"}); -}); - - -test("proxy with config file, correct auth or no match", () => { - let status = 0; - try { - execSync(`docker run --rm --network=proxy-test-net -v $PWD/tests/fixtures/proxies/:/proxies/ webrecorder/browsertrix-crawler crawl --proxyServerConfig /proxies/proxy-test-good-auth.pac --url ${HTML} ${extraArgs}`, {encoding: "utf-8"}); - } catch (e) { - status = e.status; - } - expect(status).toBe(0); - - // success, no match for PDF - execSync(`docker run --rm --network=proxy-test-net -v $PWD/tests/fixtures/proxies/:/proxies/ webrecorder/browsertrix-crawler crawl --proxyServerConfig /proxies/proxy-test-good-auth.pac --url ${PDF} ${extraArgs}`, {encoding: "utf-8"}); - -}); diff --git a/tests/proxy.test.ts b/tests/proxy.test.ts new file mode 100644 index 000000000..518692f04 --- /dev/null +++ b/tests/proxy.test.ts @@ -0,0 +1,249 @@ +import { execSync } from "child_process"; + +import { getSafeProxyString } from "../src/util/proxy.js"; +import { ErrorWithStatus, sleep } from "./utils.js"; + +const PROXY_IMAGE = "tarampampam/3proxy:1.9.1"; +const SOCKS_PORT = "1080"; +const HTTP_PORT = "3128"; +const WRONG_PORT = "33130"; + +const PROXY_EXIT_CODE = 21; + +const SSH_PROXY_IMAGE = "linuxserver/openssh-server"; + +const PDF = "https://specs.webrecorder.net/wacz/1.1.1/wacz-2021.pdf"; +const HTML = "https://old.webrecorder.net/"; + +const extraArgs = "--limit 1 --failOnFailedSeed --timeout 10 --logging debug"; + +let proxyAuthId: string; +let proxyNoAuthId: string; +let proxySSHId: string; + +beforeAll(() => { + execSync("docker network create proxy-test-net"); + + proxyAuthId = execSync( + `docker run -e PROXY_LOGIN=user -e PROXY_PASSWORD=passw0rd -d --rm --network=proxy-test-net --name proxy-with-auth ${PROXY_IMAGE}`, + { encoding: "utf-8" }, + ); + + proxyNoAuthId = execSync( + `docker run -d --rm --network=proxy-test-net --name proxy-no-auth ${PROXY_IMAGE}`, + { encoding: "utf-8" }, + ); + + proxySSHId = execSync( + `docker run -d --rm -e DOCKER_MODS=linuxserver/mods:openssh-server-ssh-tunnel -e USER_NAME=user -e PUBLIC_KEY_FILE=/keys/proxy-key.pub -v $PWD/tests/fixtures/proxies/proxy-key.pub:/keys/proxy-key.pub --network=proxy-test-net --name ssh-proxy ${SSH_PROXY_IMAGE}`, + { encoding: "utf-8" }, + ); +}); + +afterAll(async () => { + execSync(`docker kill -s SIGINT ${proxyAuthId}`); + execSync(`docker kill -s SIGINT ${proxyNoAuthId}`); + execSync(`docker kill -s SIGINT ${proxySSHId}`); + await sleep(5000); + execSync("docker network rm proxy-test-net"); +}); + +describe("socks5 + https proxy tests", () => { + for (const scheme of ["socks5", "http"]) { + const port = scheme === "socks5" ? SOCKS_PORT : HTTP_PORT; + + for (const type of ["HTML page", "PDF"]) { + const url = type === "PDF" ? PDF : HTML; + + test(`${scheme} proxy, ${type}, no auth`, () => { + let status = 0; + + try { + execSync( + `docker run -e PROXY_SERVER=${scheme}://proxy-no-auth:${port} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${url} ${extraArgs}`, + { encoding: "utf-8" }, + ); + } catch (e) { + status = (e as ErrorWithStatus).status; + } + expect(status).toBe(0); + }); + + test(`${scheme} proxy, ${type}, with auth`, () => { + let status = 0; + + try { + execSync( + `docker run -e PROXY_SERVER=${scheme}://user:passw0rd@proxy-with-auth:${port} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${url} ${extraArgs}`, + { encoding: "utf-8" }, + ); + } catch (e) { + status = (e as ErrorWithStatus).status; + } + // auth supported only for SOCKS5 + expect(status).toBe(scheme === "socks5" ? 0 : PROXY_EXIT_CODE); + }); + + test(`${scheme} proxy, ${type}, wrong auth`, () => { + let status = 0; + + try { + execSync( + `docker run -e PROXY_SERVER=${scheme}://user:passw1rd@proxy-with-auth:${port} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${url} ${extraArgs}`, + { encoding: "utf-8" }, + ); + } catch (e) { + status = (e as ErrorWithStatus).status; + } + expect(status).toBe(PROXY_EXIT_CODE); + }); + + test(`${scheme} proxy, ${type}, wrong protocol`, () => { + let status = 0; + + try { + execSync( + `docker run -e PROXY_SERVER=${scheme}://user:passw1rd@proxy-with-auth:${ + scheme === "socks5" ? HTTP_PORT : SOCKS_PORT + } --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${url} ${extraArgs}`, + { encoding: "utf-8" }, + ); + } catch (e) { + status = (e as ErrorWithStatus).status; + } + // wrong protocol (socks5 for http) causes connection to hang, causes a timeout, so just errors with 1 + expect(status === PROXY_EXIT_CODE || status === 1).toBe(true); + }); + } + + test(`${scheme} proxy, proxy missing error`, () => { + let status = 0; + + try { + execSync( + `docker run -e PROXY_SERVER=${scheme}://proxy-no-auth:${WRONG_PORT} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${HTML} ${extraArgs}`, + { encoding: "utf-8" }, + ); + } catch (e) { + status = (e as ErrorWithStatus).status; + } + expect(status).toBe(PROXY_EXIT_CODE); + }); + } +}); + +test("http proxy, PDF, separate env vars", () => { + execSync( + `docker run -e PROXY_HOST=proxy-no-auth -e PROXY_PORT=${HTTP_PORT} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${PDF} ${extraArgs}`, + { encoding: "utf-8" }, + ); +}); + +test("http proxy set, but not running, separate env vars", () => { + let status = 0; + + try { + execSync( + `docker run -e PROXY_HOST=proxy-no-auth -e PROXY_PORT=${WRONG_PORT} --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --url ${PDF} ${extraArgs}`, + { encoding: "utf-8" }, + ); + } catch (e) { + status = (e as ErrorWithStatus).status; + } + expect(status).toBe(PROXY_EXIT_CODE); +}); + +test("http proxy set, but not running, cli arg", () => { + let status = 0; + + try { + execSync( + `docker run --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --proxyServer http://proxy-no-auth:${WRONG_PORT} --url ${PDF} ${extraArgs}`, + { encoding: "utf-8" }, + ); + } catch (e) { + status = (e as ErrorWithStatus).status; + } + expect(status).toBe(PROXY_EXIT_CODE); +}); + +test("ssh socks proxy with custom user", () => { + execSync( + `docker run --rm --network=proxy-test-net -v $PWD/tests/fixtures/proxies/proxy-key:/keys/proxy-key webrecorder/browsertrix-crawler crawl --proxyServer ssh://user@ssh-proxy:2222 --sshProxyPrivateKeyFile /keys/proxy-key --url ${HTML} ${extraArgs}`, + { encoding: "utf-8" }, + ); +}); + +test("ssh socks proxy, wrong user", () => { + let status = 0; + + try { + execSync( + `docker run --rm --network=proxy-test-net webrecorder/browsertrix-crawler crawl --proxyServer ssh://ssh-proxy:2222 --url ${HTML} ${extraArgs}`, + { encoding: "utf-8" }, + ); + } catch (e) { + status = (e as ErrorWithStatus).status; + } + expect(status).toBe(PROXY_EXIT_CODE); +}); + +test("ensure logged proxy string does not include any credentials", () => { + const testParams = [ + // [input, expectedOutput] + [ + "socks5://username:password@proxy-host.example.com:9001", + "socks5://proxy-host.example.com:9001", + ], + [ + "socks5://username@proxy-host.example.com:9001", + "socks5://proxy-host.example.com:9001", + ], + [ + "socks5://path-to-proxy-host.example.com:9001", + "socks5://path-to-proxy-host.example.com:9001", + ], + ["ssh://localhost:9700", "ssh://localhost:9700"], + ]; + for (const testParamSet of testParams) { + expect(getSafeProxyString(testParamSet[0])).toEqual(testParamSet[1]); + } +}); + +test("proxy with config file, wrong auth or no match", () => { + let status = 0; + try { + execSync( + `docker run --rm --network=proxy-test-net -v $PWD/tests/fixtures/proxies/:/proxies/ webrecorder/browsertrix-crawler crawl --proxyServerConfig /proxies/proxy-test-bad-auth.pac --url ${HTML} ${extraArgs}`, + { encoding: "utf-8" }, + ); + } catch (e) { + status = (e as ErrorWithStatus).status; + } + expect(status).toBe(PROXY_EXIT_CODE); + + // success, no match for PDF + execSync( + `docker run --rm --network=proxy-test-net -v $PWD/tests/fixtures/proxies/:/proxies/ webrecorder/browsertrix-crawler crawl --proxyServerConfig /proxies/proxy-test-bad-auth.pac --url ${PDF} ${extraArgs}`, + { encoding: "utf-8" }, + ); +}); + +test("proxy with config file, correct auth or no match", () => { + let status = 0; + try { + execSync( + `docker run --rm --network=proxy-test-net -v $PWD/tests/fixtures/proxies/:/proxies/ webrecorder/browsertrix-crawler crawl --proxyServerConfig /proxies/proxy-test-good-auth.pac --url ${HTML} ${extraArgs}`, + { encoding: "utf-8" }, + ); + } catch (e) { + status = (e as ErrorWithStatus).status; + } + expect(status).toBe(0); + + // success, no match for PDF + execSync( + `docker run --rm --network=proxy-test-net -v $PWD/tests/fixtures/proxies/:/proxies/ webrecorder/browsertrix-crawler crawl --proxyServerConfig /proxies/proxy-test-good-auth.pac --url ${PDF} ${extraArgs}`, + { encoding: "utf-8" }, + ); +}); diff --git a/tests/qa_compare.test.js b/tests/qa_compare.test.ts similarity index 91% rename from tests/qa_compare.test.js rename to tests/qa_compare.test.ts index f2b1aa870..945fc8dd4 100644 --- a/tests/qa_compare.test.js +++ b/tests/qa_compare.test.ts @@ -1,8 +1,7 @@ import child_process from "child_process"; import fs from "fs"; import { Redis } from "ioredis"; - -const sleep = (ms) => new Promise((res) => setTimeout(res, ms)); +import { sleep } from "./utils"; test("run initial crawl with text and screenshots to prepare for QA", async () => { fs.rmSync("./test-crawls/qa-wr-net", { recursive: true, force: true }); @@ -29,11 +28,16 @@ test("run QA comparison, with write pages to redis", async () => { crawler_exited = true; }); - const redis = new Redis("redis://127.0.0.1:36380/0", { lazyConnect: true, retryStrategy: () => null }); + const redis = new Redis("redis://127.0.0.1:36380/0", { + lazyConnect: true, + retryStrategy: () => null, + }); await sleep(3000); - await redis.connect({ maxRetriesPerRequest: 50 }); + redis.options.maxRetriesPerRequest = 50; + + await redis.connect(); let count = 0; diff --git a/tests/retry-failed.test.js b/tests/retry-failed.test.ts similarity index 51% rename from tests/retry-failed.test.js rename to tests/retry-failed.test.ts index b914ad346..48985f980 100644 --- a/tests/retry-failed.test.js +++ b/tests/retry-failed.test.ts @@ -2,27 +2,24 @@ import { exec, execSync } from "child_process"; import fs from "fs"; import http from "http"; import Redis from "ioredis"; +import { sleep } from "./utils"; const DOCKER_HOST_NAME = process.env.DOCKER_HOST_NAME || "host.docker.internal"; -async function sleep(time) { - await new Promise((resolve) => setTimeout(resolve, time)); -} - let requests = 0; let success = false; -let server = null; +let server: http.Server; beforeAll(() => { server = http.createServer((req, res) => { // 3 requests: 2 from browser, 1 direct fetch per attempt // succeed on 6th request == after 2 retries if (requests >= 6) { - res.writeHead(200, {"Content-Type": "text/html"}); + res.writeHead(200, { "Content-Type": "text/html" }); res.end("Test Data"); success = true; } else { - res.writeHead(503, {"Content-Type": "text/html"}); + res.writeHead(503, { "Content-Type": "text/html" }); res.end("Test Data"); } requests++; @@ -35,26 +32,28 @@ afterAll(() => { server.close(); }); - - test("run crawl with retries for no response", async () => { - execSync(`docker run -d -v $PWD/test-crawls:/crawls -e CRAWL_ID=test -p 36387:6379 --rm webrecorder/browsertrix-crawler crawl --url http://invalid-host-x:31501 --url https://example-com.webrecorder.net/ --limit 2 --pageExtraDelay 10 --debugAccessRedis --collection retry-fail --retries 5`); + execSync( + `docker run -d -v $PWD/test-crawls:/crawls -e CRAWL_ID=test -p 36387:6379 --rm webrecorder/browsertrix-crawler crawl --url http://invalid-host-x:31501 --url https://example-com.webrecorder.net/ --limit 2 --pageExtraDelay 10 --debugAccessRedis --collection retry-fail --retries 5`, + ); - const redis = new Redis("redis://127.0.0.1:36387/0", { lazyConnect: true, retryStrategy: () => null }); + const redis = new Redis("redis://127.0.0.1:36387/0", { + lazyConnect: true, + retryStrategy: () => null, + }); await sleep(3000); let numRetries = 0; try { - await redis.connect({ - maxRetriesPerRequest: 100, - }); + redis.options.maxRetriesPerRequest = 100; + await redis.connect(); while (true) { const res = await redis.lrange("test:f", 0, -1); if (res.length) { - const data = JSON.parse(res); + const data = JSON.parse(res[0]); if (data.retry) { numRetries = data.retry; break; @@ -62,7 +61,6 @@ test("run crawl with retries for no response", async () => { } await sleep(20); } - } catch (e) { console.error(e); } finally { @@ -70,7 +68,6 @@ test("run crawl with retries for no response", async () => { } }); - test("check only one failed page entry is made", () => { expect( fs.existsSync("test-crawls/collections/retry-fail/pages/pages.jsonl"), @@ -81,23 +78,29 @@ test("check only one failed page entry is made", () => { .readFileSync( "test-crawls/collections/retry-fail/pages/pages.jsonl", "utf8", - ).trim().split("\n").length + ) + .trim() + .split("\n").length, ).toBe(3); }); - test("run crawl with retries for 503, enough retries to succeed", async () => { requests = 0; success = false; - const child = exec(`docker run -v $PWD/test-crawls:/crawls --rm webrecorder/browsertrix-crawler crawl --url http://${DOCKER_HOST_NAME}:31501 --url https://example-com.webrecorder.net/ --limit 2 --collection retry-fail-2 --retries 2 --failOnInvalidStatus --failOnFailedSeed --logging stats,debug`); + const child = exec( + `docker run -v $PWD/test-crawls:/crawls --rm webrecorder/browsertrix-crawler crawl --url http://${DOCKER_HOST_NAME}:31501 --url https://example-com.webrecorder.net/ --limit 2 --collection retry-fail-2 --retries 2 --failOnInvalidStatus --failOnFailedSeed --logging stats,debug`, + ); - let status = 0; + let status: number | null = 0; - const crawlFinished = new Promise(r => resolve = r); + let resolve: () => void; + + const crawlFinished = new Promise((r) => (resolve = r)); // detect crawler exit - let crawler_exited = false; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const crawler_exited = false; child.on("exit", function (code) { status = code; resolve(); @@ -112,19 +115,23 @@ test("run crawl with retries for 503, enough retries to succeed", async () => { expect(success).toBe(true); }); - test("run crawl with retries for 503, not enough retries, fail", async () => { requests = 0; success = false; - const child = exec(`docker run -v $PWD/test-crawls:/crawls --rm webrecorder/browsertrix-crawler crawl --url http://${DOCKER_HOST_NAME}:31501 --url https://example-com.webrecorder.net/ --limit 2 --collection retry-fail-3 --retries 1 --failOnInvalidStatus --failOnFailedSeed --logging stats,debug`); + const child = exec( + `docker run -v $PWD/test-crawls:/crawls --rm webrecorder/browsertrix-crawler crawl --url http://${DOCKER_HOST_NAME}:31501 --url https://example-com.webrecorder.net/ --limit 2 --collection retry-fail-3 --retries 1 --failOnInvalidStatus --failOnFailedSeed --logging stats,debug`, + ); - let status = 0; + let status: number | null = 0; - const crawlFinished = new Promise(r => resolve = r); + let resolve: () => void; + + const crawlFinished = new Promise((r) => (resolve = r)); // detect crawler exit - let crawler_exited = false; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const crawler_exited = false; child.on("exit", function (code) { status = code; resolve(); @@ -138,19 +145,23 @@ test("run crawl with retries for 503, not enough retries, fail", async () => { expect(success).toBe(false); }); - test("run crawl with retries for 503, no retries, fail", async () => { requests = 0; success = false; - const child = exec(`docker run -v $PWD/test-crawls:/crawls --rm webrecorder/browsertrix-crawler crawl --url http://${DOCKER_HOST_NAME}:31501 --url https://example-com.webrecorder.net/ --limit 2 --collection retry-fail-4 --retries 0 --failOnInvalidStatus --failOnFailedSeed --logging stats,debug`); + const child = exec( + `docker run -v $PWD/test-crawls:/crawls --rm webrecorder/browsertrix-crawler crawl --url http://${DOCKER_HOST_NAME}:31501 --url https://example-com.webrecorder.net/ --limit 2 --collection retry-fail-4 --retries 0 --failOnInvalidStatus --failOnFailedSeed --logging stats,debug`, + ); - let status = 0; + let status: number | null = 0; - const crawlFinished = new Promise(r => resolve = r); + let resolve: () => void; + + const crawlFinished = new Promise((r) => (resolve = r)); // detect crawler exit - let crawler_exited = false; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const crawler_exited = false; child.on("exit", function (code) { status = code; resolve(); @@ -163,5 +174,3 @@ test("run crawl with retries for 503, no retries, fail", async () => { expect(requests).toBe(3); expect(success).toBe(false); }); - - diff --git a/tests/robots_txt.test.js b/tests/robots_txt.test.ts similarity index 100% rename from tests/robots_txt.test.js rename to tests/robots_txt.test.ts diff --git a/tests/rollover-writer.test.js b/tests/rollover-writer.test.ts similarity index 100% rename from tests/rollover-writer.test.js rename to tests/rollover-writer.test.ts diff --git a/tests/saved-state.test.js b/tests/saved-state.test.ts similarity index 81% rename from tests/saved-state.test.js rename to tests/saved-state.test.ts index de67e7135..bbfc8fb18 100644 --- a/tests/saved-state.test.js +++ b/tests/saved-state.test.ts @@ -3,17 +3,14 @@ import fs from "fs"; import path from "path"; import yaml from "js-yaml"; import Redis from "ioredis"; - +import { sleep } from "./utils"; +import { SaveState } from "../src/util/state"; const pagesFile = "test-crawls/collections/int-state-test/pages/pages.jsonl"; -const extraPagesFile = "test-crawls/collections/int-state-test/pages/extraPages.jsonl"; - - -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} +const extraPagesFile = + "test-crawls/collections/int-state-test/pages/extraPages.jsonl"; -async function waitContainerDone(containerId) { +async function waitContainerDone(containerId: string) { // containerId is initially the full id, but docker ps // only prints the short id (first 12 characters) containerId = containerId.slice(0, 12); @@ -31,7 +28,7 @@ async function waitContainerDone(containerId) { } } -async function killContainer(containerId) { +async function killContainer(containerId: string) { try { execSync(`docker kill -s SIGINT ${containerId}`); } catch (e) { @@ -41,19 +38,18 @@ async function killContainer(containerId) { await waitContainerDone(containerId); } - -let savedStateFile; -let state; -let numDone; -let numQueued; -let finished; +let savedStateFile: string; +let state: SaveState; +let numDone: number; +let numQueued: number; +let finished: string[]; test("check crawl interrupted + saved state written", async () => { let containerId = null; try { containerId = execSync( - "docker run -d -v $PWD/test-crawls:/crawls -v $PWD/tests/fixtures:/tests/fixtures webrecorder/browsertrix-crawler crawl --collection int-state-test --url http://old.webrecorder.net/ --limit 10 --behaviors \"\" --exclude community", + 'docker run -d -v $PWD/test-crawls:/crawls -v $PWD/tests/fixtures:/tests/fixtures webrecorder/browsertrix-crawler crawl --collection int-state-test --url http://old.webrecorder.net/ --limit 10 --behaviors "" --exclude community', { encoding: "utf-8" }, //wait.callback, ); @@ -85,7 +81,7 @@ test("check crawl interrupted + saved state written", async () => { await sleep(500); } - await killContainer(containerId); + await killContainer(containerId!); const savedStates = fs.readdirSync( "test-crawls/collections/int-state-test/crawls", @@ -103,7 +99,7 @@ test("check parsing saved state + page done + queue present", () => { "utf-8", ); - const saved = yaml.load(savedState); + const saved = yaml.load(savedState) as { state: SaveState }; state = saved.state; finished = state.finished; @@ -138,12 +134,15 @@ test("check crawl restarted with saved state", async () => { await sleep(2000); - const redis = new Redis(`redis://127.0.0.1:${port}/0`, { lazyConnect: true, retryStrategy: () => null }); + const redis = new Redis(`redis://127.0.0.1:${port}/0`, { + lazyConnect: true, + retryStrategy: () => null, + }); try { - await redis.connect({ - maxRetriesPerRequest: 100, - }); + redis.options.maxRetriesPerRequest = 100; + + await redis.connect(); await sleep(2000); @@ -156,7 +155,7 @@ test("check crawl restarted with saved state", async () => { } catch (e) { console.log(e); } finally { - await waitContainerDone(containerId); + await waitContainerDone(containerId!); } }); diff --git a/tests/scopes.test.js b/tests/scopes.test.ts similarity index 93% rename from tests/scopes.test.js rename to tests/scopes.test.ts index dce5b481b..e1ca7e146 100644 --- a/tests/scopes.test.js +++ b/tests/scopes.test.ts @@ -1,19 +1,25 @@ -import { parseArgs } from "../dist/util/argParser.js"; -import { parseSeeds } from "../dist/util/seeds.js"; +import { CrawlerArgs, parseArgs } from "../src/util/argParser.js"; +import { parseSeeds } from "../src/util/seeds.js"; import fs from "fs"; -async function getSeeds(config) { +async function getSeeds(config: string) { const orig = fs.readFileSync; - fs.readFileSync = (name, ...args) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (fs as any).readFileSync = (name: string, ...args: any[]) => { if (name.endsWith("/stdinconfig")) { return config; } return orig(name, ...args); }; - const params = parseArgs(["node", "crawler", "--config", "stdinconfig"]); + const params = parseArgs([ + "node", + "crawler", + "--config", + "stdinconfig", + ]) as CrawlerArgs; return await parseSeeds("", params); } @@ -344,19 +350,19 @@ seeds: const result1 = seeds[0].isIncluded( "https://example.com/page?foo=bar&baz=qux", 0, - 0 + 0, ); expect(result1).not.toBe(false); - expect(result1.isOOS).toBe(false); + expect((result1 as Exclude).isOOS).toBe(false); // Test with reordered query parameters (should still match) const result2 = seeds[0].isIncluded( "https://example.com/page?baz=qux&foo=bar", 0, - 0 + 0, ); expect(result2).not.toBe(false); - expect(result2.isOOS).toBe(false); + expect((result2 as Exclude).isOOS).toBe(false); }); test("scopeType page should match URLs with double-encoded query parameters", async () => { @@ -376,12 +382,12 @@ seeds: // Test with the same URL (should match) const result1 = seeds[0].isIncluded(doubleEncodedUrl, 0, 0); expect(result1).not.toBe(false); - expect(result1.isOOS).toBe(false); + expect((result1 as Exclude).isOOS).toBe(false); // Test with self (should match) const result2 = seeds[0].isIncluded(seeds[0].url, 0, 0); expect(result2).not.toBe(false); - expect(result2.isOOS).toBe(false); + expect((result2 as Exclude).isOOS).toBe(false); }); test("scopeType page includes single pages with hashtag", async () => { @@ -397,5 +403,5 @@ scopeType: page // Test with self (should match) const result = seeds[0].isIncluded("https://example.com/#hashtag", 0, 0); expect(result).not.toBe(false); - expect(result.isOOS).toBe(false); + expect((result as Exclude).isOOS).toBe(false); }); diff --git a/tests/screenshot.test.js b/tests/screenshot.test.ts similarity index 98% rename from tests/screenshot.test.js rename to tests/screenshot.test.ts index 2a3485d63..f2850b728 100644 --- a/tests/screenshot.test.js +++ b/tests/screenshot.test.ts @@ -3,7 +3,7 @@ import fs from "fs"; // screenshot -function screenshotWarcExists(name) { +function screenshotWarcExists(name: string) { const warcList = fs.readdirSync(`test-crawls/collections/${name}/archive/`); for (const warc of warcList) { @@ -15,7 +15,6 @@ function screenshotWarcExists(name) { return false; } - test("ensure basic crawl run with --screenshot passes", async () => { child_process.execSync( "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --collection test-with-screenshots --url http://www.example.com/ --screenshot view --workers 2", diff --git a/tests/seeds.test.js b/tests/seeds.test.ts similarity index 90% rename from tests/seeds.test.js rename to tests/seeds.test.ts index f53c48c6c..b708a3f57 100644 --- a/tests/seeds.test.js +++ b/tests/seeds.test.ts @@ -1,5 +1,5 @@ import util from "util"; -import { exec as execCallback } from "child_process"; +import { exec as execCallback, ExecException } from "child_process"; const exec = util.promisify(execCallback); @@ -23,7 +23,7 @@ test("ensure one invalid seed fails crawl if failOnFailedSeed is set", async () "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url example.invalid --generateWACZ --limit 2 --failOnFailedSeed --collection failseed", ); } catch (error) { - expect(error.code).toEqual(1); + expect((error as ExecException).code).toEqual(1); passed = false; } expect(passed).toBe(false); @@ -36,7 +36,7 @@ test("ensure seed with network error fails crawl if failOnFailedSeed and failOnI "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url https://example.invalid --generateWACZ --limit 2 --failOnFailedSeed --failOnInvalidStatus --collection failseedstatus", ); } catch (error) { - expect(error.code).toEqual(1); + expect((error as ExecException).code).toEqual(1); passed = false; } expect(passed).toBe(false); @@ -49,7 +49,7 @@ test("ensure seed with 4xx/5xx response fails crawl if failOnFailedSeed and fail "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://www.iana.org/ --url https://old.webrecorder.net/doesntexist --generateWACZ --limit 2 --failOnFailedSeed --failOnInvalidStatus --collection failseed404status", ); } catch (error) { - expect(error.code).toEqual(1); + expect((error as ExecException).code).toEqual(1); passed = false; } expect(passed).toBe(false); @@ -75,7 +75,7 @@ test("ensure crawl fails if no valid seeds are passed", async () => { "docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url iana.org/ --url example.invalid --generateWACZ --limit 2 --collection allinvalidseeds", ); } catch (error) { - expect(error.code).toEqual(17); + expect((error as ExecException).code).toEqual(17); passed = false; } expect(passed).toBe(false); diff --git a/tests/sitemap-parse.test.js b/tests/sitemap-parse.test.ts similarity index 65% rename from tests/sitemap-parse.test.js rename to tests/sitemap-parse.test.ts index c55f32326..5dff4520a 100644 --- a/tests/sitemap-parse.test.js +++ b/tests/sitemap-parse.test.ts @@ -1,11 +1,8 @@ import child_process from "child_process"; import Redis from "ioredis"; +import { sleep } from "./utils"; -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function waitContainer(containerId) { +async function waitContainer(containerId: string | null) { try { child_process.execSync(`docker kill -s SIGINT ${containerId}`); } catch (e) { @@ -29,29 +26,35 @@ async function waitContainer(containerId) { let startPort = 36701; -async function runCrawl(numExpected, url, sitemap="", limit=0, numExpectedLessThan=0, extra="") { +async function runCrawl( + numExpected: number, + url: string, + sitemap = "", + limit = 0, + numExpectedLessThan = 0, + extra = "", +) { const port = startPort++; const command = `docker run --rm -d -p ${port}:6379 -e CRAWL_ID=test webrecorder/browsertrix-crawler crawl --url ${url} --sitemap ${sitemap} --limit ${limit} --context sitemap --logging debug --debugAccessRedis ${extra}`; - const containerId = child_process.execSync(command, {encoding: "utf-8"}); + const containerId = child_process.execSync(command, { encoding: "utf-8" }); await sleep(3000); - const redis = new Redis(`redis://127.0.0.1:${port}/0`, { lazyConnect: true, retryStrategy: () => null }); + const redis = new Redis(`redis://127.0.0.1:${port}/0`, { + lazyConnect: true, + retryStrategy: () => null, + }); let finished = 0; try { - await redis.connect({ - maxRetriesPerRequest: 100, - }); + redis.options.maxRetriesPerRequest = 100; + await redis.connect(); - while (true) { + for (let i = 0; i < 30; i++) { finished = await redis.zcard("test:q"); - if (await redis.get("test:sitemapDone")) { - break; - } if (finished >= numExpected) { break; } @@ -79,13 +82,25 @@ test("test sitemap with limit", async () => { }); test("test sitemap with limit, specific URL", async () => { - await runCrawl(1900, "https://developer.mozilla.org/", "https://developer.mozilla.org/sitemap.xml", 2000); + await runCrawl( + 1900, + "https://developer.mozilla.org/", + "https://developer.mozilla.org/sitemap.xml", + 2000, + ); }); test("test sitemap with application/xml content-type", async () => { - await runCrawl(10, "https://bitarchivist.net/", "", 0); + await runCrawl(10, "https://bitarchivist.net/", "https://www.bitarchivist.net/sitemap.xml", 0); }, 180000); test("test sitemap with narrow scope, extraHops, to ensure out-of-scope sitemap URLs do not count as extraHops", async () => { - await runCrawl(0, "https://www.mozilla.org/", "", 2000, 100, "--extraHops 1 --scopeType page"); + await runCrawl( + 0, + "https://www.mozilla.org/", + "", + 2000, + 100, + "--extraHops 1 --scopeType page", + ); }, 180000); diff --git a/tests/storage.test.js b/tests/storage.test.ts similarity index 89% rename from tests/storage.test.js rename to tests/storage.test.ts index 310cb6ce5..c2c011d15 100644 --- a/tests/storage.test.js +++ b/tests/storage.test.ts @@ -1,7 +1,8 @@ import { calculatePercentageUsed, checkDiskUtilization, -} from "../dist/util/storage.js"; +} from "../src/util/storage.js"; +import type { CrawlerArgs } from "../src/util/argParser.js"; test("ensure calculatePercentageUsed returns expected values", () => { expect(calculatePercentageUsed(30, 100)).toEqual(30); @@ -29,11 +30,11 @@ grpcfuse 1000000 285000 715000 28% /crawls`; // with combineWARC + generateWACZ, projected is 285k + 4 * 5k = 310k = 31% // does not exceed 90% threshold const returnValue = await checkDiskUtilization( - '/crawls', - params, + "/crawls", + params as CrawlerArgs, 5000 * 1024, mockDfOutput, - false + false, ); expect(returnValue).toEqual({ stop: false, @@ -57,11 +58,11 @@ grpcfuse 100000 85000 15000 85% /crawls`; // with generateWACZ, projected is 85k + 3k x 2 = 91k = 91% // exceeds 90% threshold const returnValue = await checkDiskUtilization( - '/crawls', - params, + "/crawls", + params as CrawlerArgs, 3000 * 1024, mockDfOutput, - false + false, ); expect(returnValue).toEqual({ stop: true, diff --git a/tests/text-extract.test.js b/tests/text-extract.test.ts similarity index 86% rename from tests/text-extract.test.js rename to tests/text-extract.test.ts index 9f12edaa3..be442ecb4 100644 --- a/tests/text-extract.test.js +++ b/tests/text-extract.test.ts @@ -1,5 +1,5 @@ import fs from "fs"; -import child_process from "child_process"; +import child_process, { ExecException } from "child_process"; test("check that urn:text and urn:textfinal records are written to WARC", async () => { try { @@ -8,7 +8,7 @@ test("check that urn:text and urn:textfinal records are written to WARC", async ); } catch (error) { //console.log(new TextDecoder().decode(error)); - console.log(error.stderr); + console.log((error as ExecException).stderr); } const data = fs.readFileSync( diff --git a/tests/upload-wacz.test.js b/tests/upload-wacz.test.ts similarity index 67% rename from tests/upload-wacz.test.js rename to tests/upload-wacz.test.ts index 65e9a8ee9..01386cbab 100644 --- a/tests/upload-wacz.test.js +++ b/tests/upload-wacz.test.ts @@ -1,24 +1,23 @@ import { execSync, exec } from "child_process"; -import fs from "fs"; import { Redis } from "ioredis"; import { S3Client, PutBucketPolicyCommand } from "@aws-sdk/client-s3"; - - -const sleep = (ms) => new Promise((res) => setTimeout(res, ms)); +import { sleep } from "./utils"; const ACCESS = "ROOT"; const SECRET = "TESTSECRET"; -const BUCKET_NAME = "test-bucket" +const BUCKET_NAME = "test-bucket"; -let storageId; +let storageId: string; beforeAll(() => { execSync("docker network create upload-test-net"); - storageId = execSync(`docker run --rm -d -p 39900:9000 --name s3storage --network=upload-test-net versity/versitygw --port :9000 --access ${ACCESS} --secret ${SECRET} posix /tmp/`, {encoding: "utf-8"}); + storageId = execSync( + `docker run --rm -d -p 39900:9000 --name s3storage --network=upload-test-net versity/versitygw --port :9000 --access ${ACCESS} --secret ${SECRET} posix /tmp/`, + { encoding: "utf-8" }, + ); }); - afterAll(async () => { execSync(`docker kill -s SIGINT ${storageId}`); await sleep(5000); @@ -26,21 +25,20 @@ afterAll(async () => { }); test("run crawl with upload", async () => { - execSync(`docker exec ${storageId.trim()} mkdir -p /tmp/${BUCKET_NAME}`); const child = exec( - "docker run --rm " + - `-e STORE_ENDPOINT_URL=http://s3storage:9000/${BUCKET_NAME}/ ` + - `-e STORE_ACCESS_KEY=${ACCESS} ` + - `-e STORE_SECRET_KEY=${SECRET} ` + - "-e STORE_PATH=prefix/ " + - "--network=upload-test-net " + - "-p 36390:6379 -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://old.webrecorder.net/ --limit 2 --collection upload-test --crawlId upload-test --writePagesToRedis --debugAccessRedis --generateWACZ", + "docker run --rm " + + `-e STORE_ENDPOINT_URL=http://s3storage:9000/${BUCKET_NAME}/ ` + + `-e STORE_ACCESS_KEY=${ACCESS} ` + + `-e STORE_SECRET_KEY=${SECRET} ` + + "-e STORE_PATH=prefix/ " + + "--network=upload-test-net " + + "-p 36390:6379 -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://old.webrecorder.net/ --limit 2 --collection upload-test --crawlId upload-test --writePagesToRedis --debugAccessRedis --generateWACZ", ); - let resolve = null; - const crawlFinished = new Promise(r => resolve = r); + let resolve: () => void; + const crawlFinished = new Promise((r) => (resolve = r)); // detect crawler exit let crawler_exited = false; @@ -49,11 +47,15 @@ test("run crawl with upload", async () => { resolve(); }); - const redis = new Redis("redis://127.0.0.1:36390/0", { lazyConnect: true, retryStrategy: () => null }); + const redis = new Redis("redis://127.0.0.1:36390/0", { + lazyConnect: true, + retryStrategy: () => null, + }); await sleep(3000); - await redis.connect({ maxRetriesPerRequest: 50 }); + redis.options.maxRetriesPerRequest = 50; + await redis.connect(); let filename; @@ -83,12 +85,13 @@ test("run crawl with upload", async () => { // equivalent command with 'mc', leaving for reference for now // execSync(`docker run --entrypoint /bin/sh --network=upload-test-net minio/mc -c "mc alias set local http://s3storage:9000 ${ACCESS} ${SECRET} && mc anonymous set download local/${BUCKET_NAME}"`); - // wait for crawler to finish await crawlFinished; // ensure WACZ exists at the specified filename - const resp = await fetch(`http://127.0.0.1:39900/${BUCKET_NAME}/prefix/${filename}`); + const resp = await fetch( + `http://127.0.0.1:39900/${BUCKET_NAME}/prefix/${filename}`, + ); expect(resp.status).toBe(200); }); @@ -102,7 +105,7 @@ async function makeBucketPublic() { Effect: "Allow", Principal: "*", Action: "s3:GetObject", - Resource: `arn:aws:s3:::${BUCKET_NAME}/*` + Resource: `arn:aws:s3:::${BUCKET_NAME}/*`, }, ], }; @@ -110,7 +113,7 @@ async function makeBucketPublic() { const s3client = new S3Client({ credentials: { accessKeyId: ACCESS, - secretAccessKey: SECRET + secretAccessKey: SECRET, }, endpoint: "http://127.0.0.1:39900", region: "us-east-1", @@ -126,4 +129,4 @@ async function makeBucketPublic() { } catch (e) { console.error("Error applying policy", e); } -}; +} diff --git a/tests/url-normalize.test.js b/tests/url-normalize.test.ts similarity index 100% rename from tests/url-normalize.test.js rename to tests/url-normalize.test.ts diff --git a/tests/url_file_list.test.js b/tests/url_file_list.test.ts similarity index 84% rename from tests/url_file_list.test.js rename to tests/url_file_list.test.ts index d206e242f..2de1d33f0 100644 --- a/tests/url_file_list.test.js +++ b/tests/url_file_list.test.ts @@ -1,12 +1,11 @@ import util from "util"; -import { spawn, exec as execCallback } from "child_process"; +import { spawn, exec as execCallback, ChildProcess } from "child_process"; import fs from "fs"; -import os from "os"; import path from "path"; const exec = util.promisify(execCallback); -let proc = null; +let proc: ChildProcess | null = null; const DOCKER_HOST_NAME = process.env.DOCKER_HOST_NAME || "host.docker.internal"; const TEST_HOST = `http://${DOCKER_HOST_NAME}:31502`; @@ -17,7 +16,9 @@ const seedFileCopy = path.join(fixtures, "seedFileCopy.txt"); beforeAll(() => { fs.copyFileSync(path.join(fixtures, "urlSeedFile.txt"), seedFileCopy); - proc = spawn("../../node_modules/.bin/http-server", ["-p", "31502"], {cwd: fixtures}); + proc = spawn("../../node_modules/.bin/http-server", ["-p", "31502"], { + cwd: fixtures, + }); }); afterAll(() => { @@ -28,21 +29,19 @@ afterAll(() => { fs.unlinkSync(seedFileCopy); }); - -function verifyAllSeedsCrawled(collName, hasDownload) { - let crawled_pages = fs.readFileSync( +function verifyAllSeedsCrawled(collName: string, hasDownload: boolean) { + const crawled_pages = fs.readFileSync( `test-crawls/collections/${collName}/pages/pages.jsonl`, "utf8", ); - const seedFile = hasDownload ? `test-crawls/collections/${collName}/downloads/seeds-seedFileCopy.txt` : "tests/fixtures/urlSeedFile.txt"; - let seed_file = fs - .readFileSync(seedFile, "utf8") - .split("\n") - .sort(); + const seedFile = hasDownload + ? `test-crawls/collections/${collName}/downloads/seeds-seedFileCopy.txt` + : "tests/fixtures/urlSeedFile.txt"; + const seed_file = fs.readFileSync(seedFile, "utf8").split("\n").sort(); - let seed_file_list = []; - for (var j = 0; j < seed_file.length; j++) { + const seed_file_list = []; + for (let j = 0; j < seed_file.length; j++) { if (seed_file[j] != undefined) { seed_file_list.push(seed_file[j]); } @@ -50,7 +49,7 @@ function verifyAllSeedsCrawled(collName, hasDownload) { let foundSeedUrl = true; - for (var i = 1; i < seed_file_list.length; i++) { + for (let i = 1; i < seed_file_list.length; i++) { if (crawled_pages.indexOf(seed_file_list[i]) == -1) { foundSeedUrl = false; } @@ -58,8 +57,6 @@ function verifyAllSeedsCrawled(collName, hasDownload) { expect(foundSeedUrl).toBe(true); } - - test("check that URLs in seed-list are crawled", async () => { try { await exec( @@ -72,7 +69,6 @@ test("check that URLs in seed-list are crawled", async () => { verifyAllSeedsCrawled("filelisttest", false); }); - test("check that URLs in seed-list hosted at URL are crawled", async () => { try { await exec( @@ -83,10 +79,8 @@ test("check that URLs in seed-list hosted at URL are crawled", async () => { } verifyAllSeedsCrawled("onlinefilelisttest", true); - }); - test("start crawl, interrupt, remove seed file, and ensure all seed URLs are crawled", async () => { try { await exec( @@ -96,7 +90,7 @@ test("start crawl, interrupt, remove seed file, and ensure all seed URLs are cra console.log(error); } - let crawled_pages = fs.readFileSync( + const crawled_pages = fs.readFileSync( "test-crawls/collections/seed-file-removed/pages/pages.jsonl", "utf8", ); @@ -126,11 +120,9 @@ test("start crawl, interrupt, remove seed file, and ensure all seed URLs are cra fs.renameSync(seedFileCopy + ".bak", seedFileCopy); } - verifyAllSeedsCrawled("seed-file-removed", true); }); - test("start crawl, interrupt, stop seed file server, and ensure all seed URLs are crawled", async () => { try { await exec( @@ -140,7 +132,7 @@ test("start crawl, interrupt, stop seed file server, and ensure all seed URLs ar console.log(error); } - let crawled_pages = fs.readFileSync( + const crawled_pages = fs.readFileSync( "test-crawls/collections/seed-file-server-gone/pages/pages.jsonl", "utf8", ); @@ -148,10 +140,12 @@ test("start crawl, interrupt, stop seed file server, and ensure all seed URLs ar expect(crawled_pages.split("\n").length === 2); // kill server that serves the seed list - proc.kill(); + proc!.kill(); // server no longer up - await expect(() => fetch("http://localhost:31502/")).rejects.toThrow("fetch failed"); + await expect(() => fetch("http://localhost:31502/")).rejects.toThrow( + "fetch failed", + ); // restart crawl, but with invalid seed list now try { diff --git a/tests/utils.ts b/tests/utils.ts new file mode 100644 index 000000000..44b7b5bce --- /dev/null +++ b/tests/utils.ts @@ -0,0 +1,20 @@ +import type { CrawlerArgs } from "../src/util/argParser"; +import type { PartialDeep } from "type-fest"; +import type { BlockRuleDecl } from "../src/util/blockrules"; + +export type TestConfig = Omit, "blockRules"> & { + blockRules: BlockRuleDecl[]; +}; + +export const isCI = !!process.env.CI; +export const testIf = (condition: unknown, ...args: Parameters) => + condition ? test(...args) : test.skip(...args); +export const doValidate = process.argv.filter((x) => + x.startsWith("-validate"), +)[0]; + +export async function sleep(time: number) { + await new Promise((resolve) => setTimeout(resolve, time)); +} + +export type ErrorWithStatus = Error & { status: number }; diff --git a/tests/warcinfo.test.js b/tests/warcinfo.test.ts similarity index 92% rename from tests/warcinfo.test.js rename to tests/warcinfo.test.ts index 3d98b8997..b9e4a198f 100644 --- a/tests/warcinfo.test.js +++ b/tests/warcinfo.test.ts @@ -3,14 +3,15 @@ import zlib from "zlib"; import path from "path"; import child_process from "child_process"; -test("run crawl", async() => { +test("run crawl", async () => { let success = false; try { const configYaml = fs.readFileSync("tests/fixtures/crawl-2.yaml", "utf8"); - const proc = child_process.execSync( + + child_process.execSync( "docker run -i -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --config stdin --limit 1 --collection warcinfo --combineWARC", - { input: configYaml, stdin: "inherit", encoding: "utf8" }, + { input: configYaml, stdio: ["pipe", "ignore", "ignore"], encoding: "utf8" }, ); //console.log(proc); @@ -23,7 +24,6 @@ test("run crawl", async() => { }); test("check that the warcinfo for individual WARC is as expected", async () => { - const warcs = fs.readdirSync("test-crawls/collections/warcinfo/archive/"); let filename = ""; diff --git a/tsconfig.json b/tsconfig.json index 61251e386..d10b752d1 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -75,7 +75,7 @@ /* Interop Constraints */ // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ - //"esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */, diff --git a/tsconfig.test.json b/tsconfig.test.json new file mode 100644 index 000000000..ca4d78d2c --- /dev/null +++ b/tsconfig.test.json @@ -0,0 +1,13 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": ".", + "noEmit": true + }, + "include": ["src/**/*", "tests/**/*"], + "exclude": [ + "tests/custom-behaviors/**/*", + "tests/fixtures/**/*", + "tests/invalid-behaviors/**/*" + ] +} diff --git a/yarn.lock b/yarn.lock index 867f722bf..8b16840eb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -629,6 +629,15 @@ js-tokens "^4.0.0" picocolors "^1.0.0" +"@babel/code-frame@^7.27.1": + version "7.29.0" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.29.0.tgz#7cd7a59f15b3cc0dcd803038f7792712a7d0b15c" + integrity sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw== + dependencies: + "@babel/helper-validator-identifier" "^7.28.5" + js-tokens "^4.0.0" + picocolors "^1.1.1" + "@babel/compat-data@^7.25.9": version "7.26.2" resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.26.2.tgz#278b6b13664557de95b8f35b90d96785850bb56e" @@ -709,6 +718,11 @@ resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz#24b64e2c3ec7cd3b3c547729b8d16871f22cbdc7" integrity sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ== +"@babel/helper-validator-identifier@^7.28.5": + version "7.28.5" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz#010b6938fab7cb7df74aa2bbc06aa503b8fe5fb4" + integrity sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q== + "@babel/helper-validator-option@^7.25.9": version "7.25.9" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.25.9.tgz#86e45bd8a49ab7e03f276577f96179653d41da72" @@ -1057,6 +1071,11 @@ slash "^3.0.0" strip-ansi "^6.0.0" +"@jest/diff-sequences@30.3.0": + version "30.3.0" + resolved "https://registry.yarnpkg.com/@jest/diff-sequences/-/diff-sequences-30.3.0.tgz#25b0818d3d83f00b9c7b04e069b8810f9014b143" + integrity sha512-cG51MVnLq1ecVUaQ3fr6YuuAOitHK1S4WUJHnsPFE/quQr33ADUx1FfrTCpMCRxvy0Yr9BThKpDjSlcTi91tMA== + "@jest/environment@^29.7.0": version "29.7.0" resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-29.7.0.tgz#24d61f54ff1f786f3cd4073b4b94416383baf2a7" @@ -1067,6 +1086,13 @@ "@types/node" "*" jest-mock "^29.7.0" +"@jest/expect-utils@30.3.0": + version "30.3.0" + resolved "https://registry.yarnpkg.com/@jest/expect-utils/-/expect-utils-30.3.0.tgz#c45b2da9802ffed33bf43b3e019ddb95e5ad95e8" + integrity sha512-j0+W5iQQ8hBh7tHZkTQv3q2Fh/M7Je72cIsYqC4OaktgtO7v1So9UTjp6uPBHIaB6beoF/RRsCgMJKvti0wADA== + dependencies: + "@jest/get-type" "30.1.0" + "@jest/expect-utils@^29.7.0": version "29.7.0" resolved "https://registry.yarnpkg.com/@jest/expect-utils/-/expect-utils-29.7.0.tgz#023efe5d26a8a70f21677d0a1afc0f0a44e3a1c6" @@ -1094,6 +1120,11 @@ jest-mock "^29.7.0" jest-util "^29.7.0" +"@jest/get-type@30.1.0": + version "30.1.0" + resolved "https://registry.yarnpkg.com/@jest/get-type/-/get-type-30.1.0.tgz#4fcb4dc2ebcf0811be1c04fd1cb79c2dba431cbc" + integrity sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA== + "@jest/globals@^29.7.0": version "29.7.0" resolved "https://registry.yarnpkg.com/@jest/globals/-/globals-29.7.0.tgz#8d9290f9ec47ff772607fa864ca1d5a2efae1d4d" @@ -1104,6 +1135,14 @@ "@jest/types" "^29.6.3" jest-mock "^29.7.0" +"@jest/pattern@30.0.1": + version "30.0.1" + resolved "https://registry.yarnpkg.com/@jest/pattern/-/pattern-30.0.1.tgz#d5304147f49a052900b4b853dedb111d080e199f" + integrity sha512-gWp7NfQW27LaBQz3TITS8L7ZCQ0TLvtmI//4OwlQRx4rnWxcPNIYjxZpDcN4+UlGxgm3jS5QPz8IPTCkb59wZA== + dependencies: + "@types/node" "*" + jest-regex-util "30.0.1" + "@jest/reporters@^29.7.0": version "29.7.0" resolved "https://registry.yarnpkg.com/@jest/reporters/-/reporters-29.7.0.tgz#04b262ecb3b8faa83b0b3d321623972393e8f4c7" @@ -1134,6 +1173,13 @@ strip-ansi "^6.0.0" v8-to-istanbul "^9.0.1" +"@jest/schemas@30.0.5": + version "30.0.5" + resolved "https://registry.yarnpkg.com/@jest/schemas/-/schemas-30.0.5.tgz#7bdf69fc5a368a5abdb49fd91036c55225846473" + integrity sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA== + dependencies: + "@sinclair/typebox" "^0.34.0" + "@jest/schemas@^29.6.3": version "29.6.3" resolved "https://registry.yarnpkg.com/@jest/schemas/-/schemas-29.6.3.tgz#430b5ce8a4e0044a7e3819663305a7b3091c8e03" @@ -1191,6 +1237,19 @@ slash "^3.0.0" write-file-atomic "^4.0.2" +"@jest/types@30.3.0": + version "30.3.0" + resolved "https://registry.yarnpkg.com/@jest/types/-/types-30.3.0.tgz#cada800d323cb74945c24ac74615fdb312a6c85f" + integrity sha512-JHm87k7bA33hpBngtU8h6UBub/fqqA9uXfw+21j5Hmk7ooPHlboRNxHq0JcMtC+n8VJGP1mcfnD3Mk+XKe1oSw== + dependencies: + "@jest/pattern" "30.0.1" + "@jest/schemas" "30.0.5" + "@types/istanbul-lib-coverage" "^2.0.6" + "@types/istanbul-reports" "^3.0.4" + "@types/node" "*" + "@types/yargs" "^17.0.33" + chalk "^4.1.2" + "@jest/types@^29.6.3": version "29.6.3" resolved "https://registry.yarnpkg.com/@jest/types/-/types-29.6.3.tgz#1131f8cf634e7e84c5e77bab12f052af585fba59" @@ -1482,6 +1541,11 @@ resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.27.8.tgz#6667fac16c436b5434a387a34dedb013198f6e6e" integrity sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA== +"@sinclair/typebox@^0.34.0": + version "0.34.48" + resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.34.48.tgz#75b0ead87e59e1adbd6dccdc42bad4fddee73b59" + integrity sha512-kKJTNuK3AQOrgjjotVxMrCn1sUJwM76wMszfq1kdU4uYVJjvEWuFQ6HgvLt4Xz3fSmZlTOxJ/Ie13KnIcWQXFA== + "@sinonjs/commons@^3.0.0": version "3.0.1" resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-3.0.1.tgz#1029357e44ca901a615585f6d27738dbc89084cd" @@ -2048,7 +2112,7 @@ dependencies: "@types/node" "*" -"@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1": +"@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1", "@types/istanbul-lib-coverage@^2.0.6": version "2.0.6" resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz#7739c232a1fee9b4d3ce8985f314c0c6d33549d7" integrity sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w== @@ -2060,13 +2124,21 @@ dependencies: "@types/istanbul-lib-coverage" "*" -"@types/istanbul-reports@^3.0.0": +"@types/istanbul-reports@^3.0.0", "@types/istanbul-reports@^3.0.4": version "3.0.4" resolved "https://registry.yarnpkg.com/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz#0f03e3d2f670fbdac586e34b433783070cc16f54" integrity sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ== dependencies: "@types/istanbul-lib-report" "*" +"@types/jest@^30.0.0": + version "30.0.0" + resolved "https://registry.yarnpkg.com/@types/jest/-/jest-30.0.0.tgz#5e85ae568006712e4ad66f25433e9bdac8801f1d" + integrity sha512-XTYugzhuwqWjws0CVz8QpM36+T+Dz5mTEBKhNs/esGLnCIlGdRy+Dq78NRjd7ls7r8BC8ZRMOrKlkO1hU0JOwA== + dependencies: + expect "^30.0.0" + pretty-format "^30.0.0" + "@types/js-levenshtein@^1.1.3": version "1.1.3" resolved "https://registry.yarnpkg.com/@types/js-levenshtein/-/js-levenshtein-1.1.3.tgz#a6fd0bdc8255b274e5438e0bfb25f154492d1106" @@ -2082,6 +2154,11 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841" integrity sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA== +"@types/md5@^2.3.6": + version "2.3.6" + resolved "https://registry.yarnpkg.com/@types/md5/-/md5-2.3.6.tgz#db6901a9fc1d95eeed851a62c5ce5dedfac8ff9a" + integrity sha512-WD69gNXtRBnpknfZcb4TRQ0XJQbUPZcai/Qdhmka3sxUR3Et8NrXoeAoknG/LghYHTf4ve795rInVYHBTQdNVA== + "@types/node@*": version "22.9.3" resolved "https://registry.yarnpkg.com/@types/node/-/node-22.9.3.tgz#08f3d64b3bc6d74b162d36f60213e8a6704ef2b4" @@ -2089,12 +2166,12 @@ dependencies: undici-types "~6.19.8" -"@types/node@^20.8.7": - version "20.17.7" - resolved "https://registry.yarnpkg.com/@types/node/-/node-20.17.7.tgz#790151a28c5a172773d95d53a0c23d3c59a883c4" - integrity sha512-sZXXnpBFMKbao30dUAvzKbdwA2JM1fwUtVEq/kxKuPI5mMwZiRElCpTXb0Biq/LMEVpXDZL5G5V0RPnxKeyaYg== +"@types/node@^25.5.0": + version "25.5.0" + resolved "https://registry.yarnpkg.com/@types/node/-/node-25.5.0.tgz#5c99f37c443d9ccc4985866913f1ed364217da31" + integrity sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw== dependencies: - undici-types "~6.19.2" + undici-types "~7.18.0" "@types/pako@^1.0.7": version "1.0.7" @@ -2127,7 +2204,7 @@ resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.8.tgz#8268a8c57a3e4abd25c165ecd36237db7948a55e" integrity sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ== -"@types/stack-utils@^2.0.0": +"@types/stack-utils@^2.0.0", "@types/stack-utils@^2.0.3": version "2.0.3" resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.3.tgz#6209321eb2c1712a7e7466422b8cb1fc0d9dd5d8" integrity sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw== @@ -2156,6 +2233,13 @@ resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.3.tgz#815e30b786d2e8f0dcd85fd5bcf5e1a04d008f15" integrity sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ== +"@types/yargs@^17.0.33": + version "17.0.35" + resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-17.0.35.tgz#07013e46aa4d7d7d50a49e15604c1c5340d4eb24" + integrity sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg== + dependencies: + "@types/yargs-parser" "*" + "@types/yargs@^17.0.8": version "17.0.33" resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-17.0.33.tgz#8c32303da83eec050a84b3c7ae7b9f922d13e32d" @@ -2366,7 +2450,7 @@ ansi-styles@^4.0.0, ansi-styles@^4.1.0: dependencies: color-convert "^2.0.1" -ansi-styles@^5.0.0: +ansi-styles@^5.0.0, ansi-styles@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-5.2.0.tgz#07449690ad45777d1924ac2abb2fc8895dba836b" integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA== @@ -2745,6 +2829,13 @@ browsertrix-behaviors@^0.9.8: dependencies: query-selector-shadow-dom "^1.0.1" +bs-logger@^0.2.6: + version "0.2.6" + resolved "https://registry.yarnpkg.com/bs-logger/-/bs-logger-0.2.6.tgz#eb7d365307a72cf974cc6cda76b68354ad336bd8" + integrity sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog== + dependencies: + fast-json-stable-stringify "2.x" + bser@2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/bser/-/bser-2.1.1.tgz#e6787da20ece9d07998533cfd9de6f5c38f4bc05" @@ -2906,6 +2997,11 @@ ci-info@^3.2.0: resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.9.0.tgz#4279a62028a7b1f262f3473fc9605f5e218c59b4" integrity sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ== +ci-info@^4.2.0: + version "4.4.0" + resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-4.4.0.tgz#7d54eff9f54b45b62401c26032696eb59c8bd18c" + integrity sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg== + cjs-module-lexer@^1.0.0: version "1.4.1" resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-1.4.1.tgz#707413784dbb3a72aa11c2f2b042a0bef4004170" @@ -3771,6 +3867,18 @@ expect@^29.7.0: jest-message-util "^29.7.0" jest-util "^29.7.0" +expect@^30.0.0: + version "30.3.0" + resolved "https://registry.yarnpkg.com/expect/-/expect-30.3.0.tgz#1b82111517d1ab030f3db0cf1b4061c8aa644f61" + integrity sha512-1zQrciTiQfRdo7qJM1uG4navm8DayFa2TgCSRlzUyNkhcJ6XUZF3hjnpkyr3VhAqPH7i/9GkG7Tv5abz6fqz0Q== + dependencies: + "@jest/expect-utils" "30.3.0" + "@jest/get-type" "30.1.0" + jest-matcher-utils "30.3.0" + jest-message-util "30.3.0" + jest-mock "30.3.0" + jest-util "30.3.0" + extract-zip@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" @@ -3803,7 +3911,7 @@ fast-glob@^3.2.9: merge2 "^1.3.0" micromatch "^4.0.4" -fast-json-stable-stringify@^2.0.0, fast-json-stable-stringify@^2.1.0: +fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0, fast-json-stable-stringify@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== @@ -4136,7 +4244,7 @@ gopd@^1.2.0: resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.2.0.tgz#89f56b8217bdbc8802bd299df6d7f1081d7e51a1" integrity sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg== -graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.9: +graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.11, graceful-fs@^4.2.9: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== @@ -4146,6 +4254,18 @@ graphemer@^1.4.0: resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6" integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag== +handlebars@^4.7.9: + version "4.7.9" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.9.tgz#6f139082ab58dc4e5a0e51efe7db5ae890d56a0f" + integrity sha512-4E71E0rpOaQuJR2A3xDZ+GM1HyWYv1clR58tC8emQNeQe3RH7MAzSbat+V0wG78LQBo6m6bzSG/L4pBuCsgnUQ== + dependencies: + minimist "^1.2.5" + neo-async "^2.6.2" + source-map "^0.6.1" + wordwrap "^1.0.0" + optionalDependencies: + uglify-js "^3.1.4" + has-bigints@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.2.tgz#0871bd3e3d51626f6ca0966668ba35d5602d6eaa" @@ -4928,6 +5048,16 @@ jest-config@^29.7.0: slash "^3.0.0" strip-json-comments "^3.1.1" +jest-diff@30.3.0: + version "30.3.0" + resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-30.3.0.tgz#e0a4c84ef350ffd790ffd5b0016acabeecf5f759" + integrity sha512-n3q4PDQjS4LrKxfWB3Z5KNk1XjXtZTBwQp71OP0Jo03Z6V60x++K5L8k6ZrW8MY8pOFylZvHM0zsjS1RqlHJZQ== + dependencies: + "@jest/diff-sequences" "30.3.0" + "@jest/get-type" "30.1.0" + chalk "^4.1.2" + pretty-format "30.3.0" + jest-diff@^29.7.0: version "29.7.0" resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.7.0.tgz#017934a66ebb7ecf6f205e84699be10afd70458a" @@ -5000,6 +5130,16 @@ jest-leak-detector@^29.7.0: jest-get-type "^29.6.3" pretty-format "^29.7.0" +jest-matcher-utils@30.3.0: + version "30.3.0" + resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-30.3.0.tgz#d6c739fec1ecd33809f2d2b1348f6ab01d2f2493" + integrity sha512-HEtc9uFQgaUHkC7nLSlQL3Tph4Pjxt/yiPvkIrrDCt9jhoLIgxaubo1G+CFOnmHYMxHwwdaSN7mkIFs6ZK8OhA== + dependencies: + "@jest/get-type" "30.1.0" + chalk "^4.1.2" + jest-diff "30.3.0" + pretty-format "30.3.0" + jest-matcher-utils@^29.7.0: version "29.7.0" resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz#ae8fec79ff249fd592ce80e3ee474e83a6c44f12" @@ -5010,6 +5150,21 @@ jest-matcher-utils@^29.7.0: jest-get-type "^29.6.3" pretty-format "^29.7.0" +jest-message-util@30.3.0: + version "30.3.0" + resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-30.3.0.tgz#4d723544d36890ba862ac3961db52db5b0d1ba39" + integrity sha512-Z/j4Bo+4ySJ+JPJN3b2Qbl9hDq3VrXmnjjGEWD/x0BCXeOXPTV1iZYYzl2X8c1MaCOL+ewMyNBcm88sboE6YWw== + dependencies: + "@babel/code-frame" "^7.27.1" + "@jest/types" "30.3.0" + "@types/stack-utils" "^2.0.3" + chalk "^4.1.2" + graceful-fs "^4.2.11" + picomatch "^4.0.3" + pretty-format "30.3.0" + slash "^3.0.0" + stack-utils "^2.0.6" + jest-message-util@^29.7.0: version "29.7.0" resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-29.7.0.tgz#8bc392e204e95dfe7564abbe72a404e28e51f7f3" @@ -5025,6 +5180,15 @@ jest-message-util@^29.7.0: slash "^3.0.0" stack-utils "^2.0.3" +jest-mock@30.3.0: + version "30.3.0" + resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-30.3.0.tgz#e0fa4184a596a6c4fdec53d4f412158418923747" + integrity sha512-OTzICK8CpE+t4ndhKrwlIdbM6Pn8j00lvmSmq5ejiO+KxukbLjgOflKWMn3KE34EZdQm5RqTuKj+5RIEniYhog== + dependencies: + "@jest/types" "30.3.0" + "@types/node" "*" + jest-util "30.3.0" + jest-mock@^29.7.0: version "29.7.0" resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-29.7.0.tgz#4e836cf60e99c6fcfabe9f99d017f3fdd50a6347" @@ -5039,6 +5203,11 @@ jest-pnp-resolver@^1.2.2: resolved "https://registry.yarnpkg.com/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz#930b1546164d4ad5937d5540e711d4d38d4cad2e" integrity sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w== +jest-regex-util@30.0.1: + version "30.0.1" + resolved "https://registry.yarnpkg.com/jest-regex-util/-/jest-regex-util-30.0.1.tgz#f17c1de3958b67dfe485354f5a10093298f2a49b" + integrity sha512-jHEQgBXAgc+Gh4g0p3bCevgRCVRkB4VB70zhoAE48gxeSr1hfUOsM/C2WoJgVL7Eyg//hudYENbm3Ne+/dRVVA== + jest-regex-util@^29.6.3: version "29.6.3" resolved "https://registry.yarnpkg.com/jest-regex-util/-/jest-regex-util-29.6.3.tgz#4a556d9c776af68e1c5f48194f4d0327d24e8a52" @@ -5148,6 +5317,18 @@ jest-snapshot@^29.7.0: pretty-format "^29.7.0" semver "^7.5.3" +jest-util@30.3.0: + version "30.3.0" + resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-30.3.0.tgz#95a4fbacf2dac20e768e2f1744b70519f2ba7980" + integrity sha512-/jZDa00a3Sz7rdyu55NLrQCIrbyIkbBxareejQI315f/i8HjYN+ZWsDLLpoQSiUIEIyZF/R8fDg3BmB8AtHttg== + dependencies: + "@jest/types" "30.3.0" + "@types/node" "*" + chalk "^4.1.2" + ci-info "^4.2.0" + graceful-fs "^4.2.11" + picomatch "^4.0.3" + jest-util@^29.7.0: version "29.7.0" resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.7.0.tgz#23c2b62bfb22be82b44de98055802ff3710fc0bc" @@ -5422,6 +5603,11 @@ lodash.isarguments@^3.1.0: resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg== +lodash.memoize@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz#bcc6c49a42a2840ed997f323eada5ecd182e0bfe" + integrity sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag== + lodash.merge@^4.6.2: version "4.6.2" resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" @@ -5470,6 +5656,11 @@ make-dir@^4.0.0: dependencies: semver "^7.5.3" +make-error@^1.3.6: + version "1.3.6" + resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" + integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== + makeerror@1.0.12: version "1.0.12" resolved "https://registry.yarnpkg.com/makeerror/-/makeerror-1.0.12.tgz#3e5dd2079a82e812e983cc6610c4a2cb0eaa801a" @@ -5553,7 +5744,7 @@ minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2: dependencies: brace-expansion "^1.1.7" -minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.6: +minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.5, minimist@^1.2.6: version "1.2.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== @@ -5595,6 +5786,11 @@ natural-compare@^1.4.0: resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw== +neo-async@^2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" + integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== + netmask@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/netmask/-/netmask-2.0.2.tgz#8b01a07644065d536383835823bc52004ebac5e7" @@ -5914,7 +6110,7 @@ pend@~1.2.0: resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" integrity sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg== -picocolors@^1.0.0, picocolors@^1.1.0: +picocolors@^1.0.0, picocolors@^1.1.0, picocolors@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== @@ -5924,6 +6120,11 @@ picomatch@^2.0.4, picomatch@^2.2.3, picomatch@^2.3.1: resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== +picomatch@^4.0.3: + version "4.0.4" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-4.0.4.tgz#fd6f5e00a143086e074dffe4c924b8fb293b0589" + integrity sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A== + pirates@^4.0.4: version "4.0.6" resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.6.tgz#3018ae32ecfcff6c29ba2267cbf21166ac1f36b9" @@ -5995,6 +6196,15 @@ prettier@3.0.3: resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.0.3.tgz#432a51f7ba422d1469096c0fdc28e235db8f9643" integrity sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg== +pretty-format@30.3.0, pretty-format@^30.0.0: + version "30.3.0" + resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-30.3.0.tgz#e977eed4bcd1b6195faed418af8eac68b9ea1f29" + integrity sha512-oG4T3wCbfeuvljnyAzhBvpN45E8iOTXCU/TD3zXW80HA3dQ4ahdqMkWGiPWZvjpQwlbyHrPTWUAqUzGzv4l1JQ== + dependencies: + "@jest/schemas" "30.0.5" + ansi-styles "^5.2.0" + react-is "^18.3.1" + pretty-format@^29.7.0: version "29.7.0" resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-29.7.0.tgz#ca42c758310f365bfa71a0bda0a807160b776812" @@ -6154,7 +6364,7 @@ react-is@^16.13.1: resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.13.1.tgz#789729a4dc36de2999dc156dd6c1d9c18cea56a4" integrity sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ== -react-is@^18.0.0: +react-is@^18.0.0, react-is@^18.3.1: version "18.3.1" resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.3.1.tgz#e83557dc12eae63a99e003a46388b1dcbb44db7e" integrity sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg== @@ -6606,7 +6816,7 @@ sprintf-js@~1.0.2: resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== -stack-utils@^2.0.3: +stack-utils@^2.0.3, stack-utils@^2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-2.0.6.tgz#aaf0748169c02fc33c8232abccf933f54a1cc34f" integrity sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ== @@ -6798,6 +7008,11 @@ supports-preserve-symlinks-flag@^1.0.0: resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== +tagged-tag@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/tagged-tag/-/tagged-tag-1.0.0.tgz#a0b5917c2864cba54841495abfa3f6b13edcf4d6" + integrity sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng== + tar-fs@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" @@ -6934,6 +7149,21 @@ ts-api-utils@^1.0.1: resolved "https://registry.yarnpkg.com/ts-api-utils/-/ts-api-utils-1.4.0.tgz#709c6f2076e511a81557f3d07a0cbd566ae8195c" integrity sha512-032cPxaEKwM+GT3vA5JXNzIaizx388rhsSW79vGRNGXfRRAdEAn2mvk36PvK5HnOchyWZ7afLEXqYCvPCrzuzQ== +ts-jest@^29.4.9: + version "29.4.9" + resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-29.4.9.tgz#47dc33d0f5c36bddcedd16afefae285e0b049d2d" + integrity sha512-LTb9496gYPMCqjeDLdPrKuXtncudeV1yRZnF4Wo5l3SFi0RYEnYRNgMrFIdg+FHvfzjCyQk1cLncWVqiSX+EvQ== + dependencies: + bs-logger "^0.2.6" + fast-json-stable-stringify "^2.1.0" + handlebars "^4.7.9" + json5 "^2.2.3" + lodash.memoize "^4.1.2" + make-error "^1.3.6" + semver "^7.7.4" + type-fest "^4.41.0" + yargs-parser "^21.1.1" + tsc@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/tsc/-/tsc-2.0.4.tgz#5f6499146abea5dca4420b451fa4f2f9345238f5" @@ -7000,6 +7230,18 @@ type-fest@^2.12.2: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-2.19.0.tgz#88068015bb33036a598b952e55e9311a60fd3a9b" integrity sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA== +type-fest@^4.41.0: + version "4.41.0" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-4.41.0.tgz#6ae1c8e5731273c2bf1f58ad39cbae2c91a46c58" + integrity sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA== + +type-fest@^5.5.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-5.5.0.tgz#78fca72f3a1f9ec964e6ae260db492b070c56f3b" + integrity sha512-PlBfpQwiUvGViBNX84Yxwjsdhd1TUlXr6zjX7eoirtCPIr08NAmxwa+fcYBTeRQxHo9YC9wwF3m9i700sHma8g== + dependencies: + tagged-tag "^1.0.0" + typed-array-buffer@^1.0.2, typed-array-buffer@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz#a72395450a4869ec033fd549371b47af3a2ee536" @@ -7067,6 +7309,11 @@ typescript@^5.5.4: resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.7.2.tgz#3169cf8c4c8a828cde53ba9ecb3d2b1d5dd67be6" integrity sha512-i5t66RHxDvVN40HfDd1PsEThGNnlMCMT3jMUuoh9/0TaqWevNontacunWyN02LA9/fIbEWlcHZcgTKb9QoaLfg== +uglify-js@^3.1.4: + version "3.19.3" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.19.3.tgz#82315e9bbc6f2b25888858acd1fff8441035b77f" + integrity sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ== + unbox-primitive@^1.0.2, unbox-primitive@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.1.0.tgz#8d9d2c9edeea8460c7f35033a88867944934d1e2" @@ -7077,11 +7324,16 @@ unbox-primitive@^1.0.2, unbox-primitive@^1.1.0: has-symbols "^1.1.0" which-boxed-primitive "^1.1.1" -undici-types@~6.19.2, undici-types@~6.19.8: +undici-types@~6.19.8: version "6.19.8" resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02" integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw== +undici-types@~7.18.0: + version "7.18.2" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-7.18.2.tgz#29357a89e7b7ca4aef3bf0fd3fd0cd73884229e9" + integrity sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w== + undici@>=6, undici@^7.16.0: version "7.16.0" resolved "https://registry.yarnpkg.com/undici/-/undici-7.16.0.tgz#cb2a1e957726d458b536e3f076bf51f066901c1a" @@ -7296,6 +7548,11 @@ word-wrap@^1.2.5: resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.5.tgz#d2c45c6dd4fbce621a66f136cbe328afd0410b34" integrity sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA== +wordwrap@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" + integrity sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q== + wrap-ansi@7.0.0, wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"