[pull] main from webrecorder:main

package.json

@@ -1,6 +1,6 @@
 {
   "name": "browsertrix-crawler",
-  "version": "1.12.1",
+  "version": "1.12.2",
   "main": "browsertrix-crawler",
   "type": "module",
   "repository": "https://github.com/webrecorder/browsertrix-crawler",

src/crawler.ts

@@ -48,6 +48,7 @@ import {
   InterruptReason,
   BxFunctionBindings,
   MAX_JS_DIALOG_PER_PAGE,
+  CrawlStatus,
 } from "./util/constants.js";
 
 import { AdBlockRules, BlockRuleDecl, BlockRules } from "./util/blockrules.js";
@@ -343,7 +344,7 @@ export class Crawler {
     this.isExternalDedupeStore = dedupeRedisUrl !== redisUrl;
 
     if (!redisUrl.startsWith("redis://")) {
-      logger.fatal(
+      await logger.fatal(
         "stateStoreUrl must start with redis:// -- Only redis-based store currently supported",
       );
     }
@@ -381,9 +382,7 @@ export class Crawler {
       logger.setLogBehaviorsToRedis(true);
     }
 
-    if (this.params.logErrorsToRedis || this.params.logBehaviorsToRedis) {
-      logger.setCrawlState(this.crawlState);
-    }
+    logger.setCrawlState(this.crawlState);
 
     // if automatically restarts on error exit code,
     // exit with 0 from fatal always, to avoid unnecessary restart
@@ -476,7 +475,7 @@ export class Crawler {
 
   async bootstrap() {
     if (await isDiskFull(this.params.cwd)) {
-      logger.interrupt(
+      await logger.interrupt(
         "Out of disk space, exiting",
         {},
         "general",
@@ -626,7 +625,7 @@ export class Crawler {
   async run() {
     await this.bootstrap();
 
-    let status = "done";
+    let status: CrawlStatus = "done";
     let exitCode = ExitCodes.Success;
 
     try {
@@ -992,7 +991,7 @@ self.__bx_behaviors.selectMainBehavior();
             return;
           }
           await this.crawlState.setFailReason(reason);
-          logger.fatal(
+          await logger.fatal(
             "Content check failed, failing crawl",
             { reason },
             "behavior",
@@ -1377,7 +1376,7 @@ self.__bx_behaviors.selectMainBehavior();
                 }
                 break;
             }
-            logger.fatal(
+            await logger.fatal(
               "Seed Page Load Failed, failing crawl",
               {},
               "general",
@@ -1634,7 +1633,7 @@ self.__bx_behaviors.selectMainBehavior();
       const numFailed = await this.crawlState.numFailed();
       const failedLimit = this.params.failOnFailedLimit;
       if (numFailed >= failedLimit) {
-        logger.fatal(
+        await logger.fatal(
           `Failed threshold reached ${numFailed} >= ${failedLimit}, failing crawl`,
           {},
           "general",
@@ -1695,7 +1694,7 @@ self.__bx_behaviors.selectMainBehavior();
       await this.closeFiles();
 
       if (!this.done) {
-        logger.interrupt(
+        await logger.interrupt(
           "Forced interrupt by signal",
           {},
           "general",
@@ -2066,7 +2065,8 @@ self.__bx_behaviors.selectMainBehavior();
         return null;
       }
       // interrupt crawl otherwise
-      logger.fatal("No WARC Files, assuming crawl failed");
+      await logger.fatal("No WARC Files, assuming crawl failed");
+      return null;
     }
 
     const waczPath = path.join(this.collDir, this.params.collection + ".wacz");
@@ -2126,7 +2126,7 @@ self.__bx_behaviors.selectMainBehavior();
       }
       return wacz;
     } catch (e) {
-      logger.interrupt(
+      await logger.interrupt(
         "Error creating / uploading WACZ",
         formatErr(e),
         "wacz",

src/indexer.ts

@@ -104,7 +104,7 @@ export class CrawlIndexer {
       await dedupeIndex.clearUncommitted(params.cancelCrawlId);
       process.exit(ExitCodes.Success);
     } else if (!params.sourceUrl) {
-      logger.fatal(
+      await logger.fatal(
         "One of --commitCrawlId, --cancelCrawlId or --sourceUrl for import is required",
       );
       return;

src/util/argParser.ts

@@ -794,7 +794,7 @@ class ArgParser {
 
     // Check that the collection name is valid.
     if (argv.collection.search(/^[\w][\w-]*$/) === -1) {
-      logger.fatal(
+      void logger.fatal(
         `\n${argv.collection} is an invalid collection name. Please supply a collection name only using alphanumeric characters and the following characters [_ - ]\n`,
       );
     }
@@ -806,7 +806,7 @@ class ArgParser {
         try {
           parser(argv.clickSelector);
         } catch (e) {
-          logger.fatal("Invalid Autoclick CSS Selector", {
+          void logger.fatal("Invalid Autoclick CSS Selector", {
             selector: argv.clickSelector,
           });
         }
@@ -839,7 +839,7 @@ class ArgParser {
         argv.mobileDevice.replace("-", " ")
       ];
       if (!argv.emulateDevice) {
-        logger.fatal("Unknown device: " + argv.mobileDevice);
+        void logger.fatal("Unknown device: " + argv.mobileDevice);
       }
     } else {
       argv.emulateDevice = { viewport: null };
@@ -849,7 +849,9 @@ class ArgParser {
 
     if (argv.lang) {
       if (!ISO6391.validate(argv.lang)) {
-        logger.fatal("Invalid ISO-639-1 country code for --lang: " + argv.lang);
+        void logger.fatal(
+          "Invalid ISO-639-1 country code for --lang: " + argv.lang,
+        );
       }
     }
 
@@ -865,7 +867,9 @@ class ArgParser {
         try {
           parser(selector);
         } catch (e) {
-          logger.fatal("Invalid Link Extraction CSS Selector", { selector });
+          void logger.fatal("Invalid Link Extraction CSS Selector", {
+            selector,
+          });
         }
         return { selector, extract, isAttribute };
       });
@@ -876,7 +880,7 @@ class ArgParser {
     argv.selectLinks = selectLinks;
 
     if (isQA && !argv.qaSource) {
-      logger.fatal("--qaSource required for QA mode");
+      void logger.fatal("--qaSource required for QA mode");
     }
 
     // Resolve statsFilename

src/util/blockrules.ts

@@ -47,7 +47,7 @@ class BlockRule {
     }
 
     if (!RULE_TYPES.includes(this.type)) {
-      logger.fatal('Rule "type" must be: ' + RULE_TYPES.join(", "));
+      void logger.fatal('Rule "type" must be: ' + RULE_TYPES.join(", "));
     }
   }
 

src/util/browser.ts

@@ -231,7 +231,7 @@ export class Browser {
       } else {
         // remove the temp profile dir, likely empty
         await fsp.rm(tmpProfileDir, { recursive: true });
-        logger.fatal("Profile setup failed", formatErr(e), "browser");
+        await logger.fatal("Profile setup failed", formatErr(e), "browser");
       }
     }
     this.customProfile = true;
@@ -508,7 +508,7 @@ export class Browser {
       expression: script,
     });
     if (exceptionDetails) {
-      logger.fatal(
+      await logger.fatal(
         "Custom behavior load error, aborting",
         { filename, ...exceptionDetails },
         "behavior",

src/util/constants.ts

@@ -103,3 +103,15 @@ export enum InterruptReason {
   SignalInterrupted = 6,
   CrawlPaused = 7,
 }
+
+export type CrawlStatus =
+  | "running"
+  | "generate-wacz"
+  | "uploading-wacz"
+  | "generate-cdx"
+  | "generate-warc"
+  | "pending-wait"
+  | "done"
+  | "interrupted"
+  | "failed"
+  | "canceled";

src/util/file_reader.ts

@@ -46,7 +46,7 @@ export async function replaceDir(sourceDir: string, destDir: string) {
     //await exec(`mv ${sourceDir} ${destDir}`);
     await fsp.rename(sourceDir, destDir);
   } catch (e) {
-    logger.fatal("Error moving/renaming directories, should not happen", {
+    await logger.fatal("Error moving/renaming directories, should not happen", {
       ...formatErr(e),
     });
   }
@@ -124,7 +124,7 @@ export async function collectOnlineSeedFile(
     logger.info("Seed file downloaded", { url, path: filepath });
     return filepath;
   } catch (e) {
-    logger.fatal("Error downloading seed file from URL", {
+    await logger.fatal("Error downloading seed file from URL", {
       url,
       ...formatErr(e),
     });
@@ -203,7 +203,7 @@ async function collectGitBehaviors(
     );
   } catch (e) {
     if (!exists) {
-      logger.fatal(
+      await logger.fatal(
         "Error downloading custom behaviors from Git repo",
         { url: urlStripped, ...formatErr(e) },
         "behavior",
@@ -247,7 +247,7 @@ async function collectOnlineBehavior(
     );
     return await collectLocalPathBehaviors(behaviorFilepath, 0, url);
   } catch (e) {
-    logger.fatal(
+    await logger.fatal(
       "Error downloading custom behavior from URL",
       { url, ...formatErr(e) },
       "behavior",
@@ -286,7 +286,7 @@ async function collectLocalPathBehaviors(
         try {
           contents = parseRecorderFlowJson(contents, source);
         } catch (e) {
-          logger.fatal(
+          await logger.fatal(
             "Unable to parse recorder flow JSON, ignored",
             formatErr(e),
             "behavior",
@@ -329,15 +329,15 @@ async function collectLocalPathBehaviors(
       }
     }
   } catch (e) {
-    logger.fatal(
+    await logger.fatal(
       "Error fetching local custom behaviors",
       { path: resolvedPath, ...formatErr(e) },
       "behavior",
     );
   }
 
   if (!behaviors && depth === 0) {
-    logger.fatal(
+    await logger.fatal(
       "No custom behaviors found at specified path",
       { path: resolvedPath },
       "behavior",

src/util/logger.ts

@@ -3,7 +3,7 @@
 
 import { Writable } from "node:stream";
 import { RedisCrawlState } from "./state.js";
-import { ExitCodes } from "./constants.js";
+import { CrawlStatus, ExitCodes } from "./constants.js";
 import { streamFinish } from "./warcwriter.js";
 import fs from "node:fs";
 
@@ -236,7 +236,7 @@ class Logger {
     }
   }
 
-  interrupt(
+  async interrupt(
     message: string,
     data = {},
     context: LogContext,
@@ -249,18 +249,18 @@ class Logger {
       "interrupt",
     );
 
-    void this.setStatusAndExit(exitCode, "interrupted");
+    await this.setStatusAndExit(exitCode, "interrupted");
   }
 
-  fatal(
+  async fatal(
     message: string,
     data = {},
     context: LogContext = this.defaultLogContext,
     exitCode = ExitCodes.Fatal,
   ) {
     this.logAsJSON(`${message}. Quitting`, data, context, "fatal");
 
-    void this.setStatusAndExit(
+    await this.setStatusAndExit(
       this.overrideFatalExitCode ?? exitCode,
       "failed",
     );
@@ -274,7 +274,10 @@ class Logger {
     }
   }
 
-  async setStatusAndExit(exitCode: ExitCodes, status: string): Promise<void> {
+  async setStatusAndExit(
+    exitCode: ExitCodes,
+    status: CrawlStatus,
+  ): Promise<void> {
     try {
       await this.closeLog();
 

src/util/proxy.ts

@@ -193,7 +193,9 @@ export async function initProxy(
     const entry = nameToProxy.get(name);
 
     if (!entry) {
-      logger.fatal("Proxy specified but not found in proxies list: " + name);
+      await logger.fatal(
+        "Proxy specified but not found in proxies list: " + name,
+      );
       return {};
     }
 
@@ -433,7 +435,7 @@ export async function runSSHD(
   try {
     await waitForSocksPort;
   } catch (e) {
-    logger.interrupt(
+    await logger.interrupt(
       "Unable to establish SSH connection for proxy",
       {
         error: e,

src/util/redis.ts

@@ -19,7 +19,11 @@ console.error = function (...args) {
 
     if (now - lastLogTime > REDIS_ERROR_LOG_INTERVAL_SECS) {
       if (lastLogTime && exitOnError) {
-        logger.fatal("Crawl interrupted, redis gone, exiting", {}, "redis");
+        void logger.fatal(
+          "Crawl interrupted, redis gone, exiting",
+          {},
+          "redis",
+        );
       }
       logger.warn("ioredis error", { error: args[0] }, "redis");
       lastLogTime = now;

src/util/seeds.ts

@@ -226,7 +226,7 @@ export class ScopedSeed {
         break;
 
       default:
-        logger.fatal(
+        void logger.fatal(
           `Invalid scope type "${scopeType}" specified, valid types are: page, page-spa, prefix, host, domain, any`,
         );
     }
@@ -365,7 +365,7 @@ export async function parseSeeds(
         ...newSeed,
       });
       if (params.failOnFailedSeed) {
-        logger.fatal(
+        await logger.fatal(
           "Invalid seed specified, aborting crawl",
           { url: newSeed.url },
           "general",
@@ -376,7 +376,7 @@ export async function parseSeeds(
   }
 
   if (!params.qaSource && !scopedSeeds.length) {
-    logger.fatal("No valid seeds specified, aborting crawl");
+    await logger.fatal("No valid seeds specified, aborting crawl");
   }
 
   return scopedSeeds;