bugfix(normalize): normalize urls for seeds, add normalizeUrl wrapper (webrecorder#959)

- applies normalizeUrl() to seed URL and seed isIncluded() check
- add normalizeUrl() wrapper which applies standard opts and also catches and logs any errors from normalization
- test: add scope tests to ensure URL with differently sorted query args still in scope
---------

Co-authored-by: Ilya Kreymer <ikreymer@gmail.com>

Author: badgersec

src/util/normalize.ts

@@ -0,0 +1,26 @@
+import { Options, default as normalize } from "normalize-url";
+import { logger } from "./logger";
+
+// URL normalization options for consistent URL handling across the crawler
+// Query parameters are sorted alphabetically by the normalize-url library
+export const normalizeUrlOpts: Options = {
+  defaultProtocol: "https",
+  stripAuthentication: false,
+  stripTextFragment: false,
+  stripWWW: false,
+  stripHash: false,
+  removeTrailingSlash: false,
+  removeSingleSlash: false,
+  removeExplicitPort: false,
+  sortQueryParameters: true,
+  removePath: false,
+};
+
+export function normalizeUrl(url: string) {
+  try {
+    return normalize(url, normalizeUrlOpts);
+  } catch (e) {
+    logger.warn("normalizeUrl failed for url, using unmodified url", { url });
+    return url;
+  }
+}

src/util/seeds.ts

@@ -4,6 +4,7 @@ import { MAX_DEPTH } from "./constants.js";
 import { collectOnlineSeedFile } from "./file_reader.js";
 import { logger } from "./logger.js";
 import { type CrawlerArgs } from "./argParser.js";
+import { normalizeUrl } from "./normalize.js";
 
 type ScopeType =
   | "prefix"
@@ -63,7 +64,8 @@ export class ScopedSeed {
     parsedUrl.username = "";
     parsedUrl.password = "";
 
-    this.url = parsedUrl.href;
+    // Normalize URL with sorted query parameters for consistent matching
+    this.url = normalizeUrl(parsedUrl.href);
     this.include = parseRx(include);
     this.exclude = parseRx(exclude);
 
@@ -250,7 +252,8 @@ export class ScopedSeed {
       urlParsed.hash = "";
     }
 
-    url = urlParsed.href;
+    // Normalize URL with sorted query parameters for consistent matching
+    url = normalizeUrl(urlParsed.href);
 
     if (url === this.url) {
       return { url, isOOS: false };

src/util/state.ts

@@ -11,7 +11,7 @@ import {
 import { ScopedSeed } from "./seeds.js";
 import { Frame } from "puppeteer-core";
 import { interpolateFilename, UploadResult } from "./storage.js";
-import normalizeUrl, { Options as NormamlizeUrlOptions } from "normalize-url";
+import { normalizeUrl } from "./normalize.js";
 
 // ============================================================================
 export enum LoadState {
@@ -29,20 +29,6 @@ export enum QueueState {
   DUPE_URL = 2,
 }
 
-// ============================================================================
-const normalizeUrlOpts: NormamlizeUrlOptions = {
-  defaultProtocol: "https",
-  stripAuthentication: false,
-  stripTextFragment: false,
-  stripWWW: false,
-  stripHash: false,
-  removeTrailingSlash: false,
-  removeSingleSlash: false,
-  removeExplicitPort: false,
-  sortQueryParameters: true,
-  removePath: false,
-};
-
 // ============================================================================
 // treat 0 or 206 as 200 for purposes of dedup
 function normalizeDedupStatus(status: number): number {
@@ -726,7 +712,7 @@ return inx;
     }: QueueEntry,
     limit = 0,
   ) {
-    url = normalizeUrl(url, normalizeUrlOpts);
+    url = normalizeUrl(url);
     const added = this._timestamp();
     const data: QueueEntry = { added, url, seedId, depth, extraHops };
 
@@ -1062,7 +1048,7 @@ return inx;
   }
 
   async addIfNoDupe(key: string, url: string, status: number) {
-    url = normalizeUrl(url, normalizeUrlOpts);
+    url = normalizeUrl(url);
     return (
       (await this.redis.sadd(key, normalizeDedupStatus(status) + "|" + url)) ===
       1

tests/scopes.test.js

@@ -326,3 +326,35 @@ seeds:
   expect(seeds[8].exclude).toEqual([/false/]);
   expect(seeds[9].exclude).toEqual([/true/]);
 });
+
+test("scopeType page should match URLs with reordered query parameters", async () => {
+  const seeds = await getSeeds(`
+seeds:
+  - url: https://example.com/page?foo=bar&baz=qux
+    scopeType: page
+`);
+
+  expect(seeds.length).toEqual(1);
+  expect(seeds[0].scopeType).toEqual("page");
+  expect(seeds[0].include).toEqual([]);
+  expect(seeds[0].maxDepth).toEqual(0);
+  expect(seeds[0].maxExtraHops).toEqual(0);
+
+  // Test with the same URL (should match)
+  const result1 = seeds[0].isIncluded(
+    "https://example.com/page?foo=bar&baz=qux",
+    0,
+    0
+  );
+  expect(result1).not.toBe(false);
+  expect(result1.isOOS).toBe(false);
+
+  // Test with reordered query parameters (should still match)
+  const result2 = seeds[0].isIncluded(
+    "https://example.com/page?baz=qux&foo=bar",
+    0,
+    0
+  );
+  expect(result2).not.toBe(false);
+  expect(result2.isOOS).toBe(false);
+});