chore: Refactor crawl controller and crawl utility function for improved performance and readability

Author: n4ze3m

server/src/queue/controllers/crawl.controller.ts

@@ -7,8 +7,9 @@ const prisma = new PrismaClient();
 export const crawlQueueController = async (source: QSource) => {
   let maxDepth = source.maxDepth || 1;
   let maxLinks = source.maxLinks || 1;
-  const links = Array.from(await crawl(source.content!, maxDepth, 0, maxLinks));
-
+  const data = await crawl(source.content!, maxDepth, maxLinks);
+  const links = Array.from(data?.links || []);
+  
   for (const link of links) {
     const newSource = await prisma.botSource.create({
       data: {

server/src/utils/crawl.ts

@@ -1,85 +1,82 @@
 import axios from "axios";
 import { load } from "cheerio";
 
+type CrawlResult = {
+  links: Set<string>;
+  errors: Set<string>;
+};
+
 const visitedLinks: Set<string> = new Set();
+const errorLinks: Set<string> = new Set();
+const queuedLinks: Set<string> = new Set();
 
 export const crawl = async (
-  link: string,
+  startUrl: string,
   maxDepth = 2,
-  currentDepth = 0,
-  maxLinks = 20,
-): Promise<Set<string>> => {
-  const parentUrl = new URL(link);
-
-  if (currentDepth > maxDepth || visitedLinks.size >= maxLinks) {
-    return new Set();
-  }
-
-  if (visitedLinks.has(link)) {
-    return new Set();
-  }
+  maxLinks = 20
+): Promise<CrawlResult> => {
+  const queue: { url: string; depth: number }[] = [{ url: startUrl, depth: 0 }];
+  const fetchedLinks: Set<string> = new Set();
 
-  visitedLinks.add(link);
+  while (queue.length > 0 && visitedLinks.size < maxLinks) {
+    const batch = queue.splice(0, Math.min(queue.length, maxLinks - visitedLinks.size));
+    
+    await Promise.all(
+      batch.map(async ({ url, depth }) => {
+        if (visitedLinks.has(url) || depth > maxDepth) {
+          return;
+        }
 
-  try {
-    const response = await axios.get(link, {
-      headers: {
-        Accept: "text/html",
-      },
-    });
-
-    const contentType = response.headers["content-type"];
+        try {
+          const response = await axios.get(url, {
+            headers: { Accept: "text/html" },
+          });
 
-    if (!contentType.includes("text/html")) {
-      console.log(`Skipping ${link} (content type: ${contentType})`);
-      return new Set();
-    }
+          const contentType = response.headers['content-type'];
+          if (!contentType || !contentType.includes("text/html")) {
+            return;
+          }
 
-    const $ = load(response.data);
-    const links = $("a");
-    const fetchedLinks: Set<string> = new Set();
+          const $ = load(response.data);
 
-    for (let i = 0; i < links.length; i++) {
-      const href = $(links[i]).attr("href");
+          visitedLinks.add(url);
+          fetchedLinks.add(url);
 
-      if (!href) {
-        continue;
-      }
+          $("a").each((_, element) => {
+            const href = $(element).attr("href");
+            if (!href) {
+              return;
+            }
 
-      let absolute: string;
-      if (href.startsWith("/")) {
-        absolute = new URL(href, parentUrl.origin).href;
-      } else if (!isWebUrl(href)) {
-        absolute = new URL(href, parentUrl.origin).href;
-      } else {
-        absolute = href;
-      }
+            const absoluteUrl = normalizeUrl(new URL(href, url).href);
+            if (isSameDomain(absoluteUrl, startUrl) && !visitedLinks.has(absoluteUrl) && !queuedLinks.has(absoluteUrl)) {
+              queue.push({ url: absoluteUrl, depth: depth + 1 });
+              queuedLinks.add(absoluteUrl);
+            }
+          });
+        } catch (error: any) {
+          console.error(`Failed to fetch ${url}:`, error?.message || error);
+          errorLinks.add(url);
+        }
+      })
+    );
+  }
 
-      if (new URL(absolute).host !== parentUrl.host) {
-        continue;
-      }
+  return { links: fetchedLinks, errors: errorLinks };
+};
 
-      const childLinks = await crawl(
-        absolute,
-        maxDepth,
-        currentDepth + 1,
-        maxLinks,
-      );
-      childLinks.forEach((childLink) => fetchedLinks.add(childLink));
-    }
-    fetchedLinks.add(link);
-    return fetchedLinks;
-  } catch (error: any) {
-    console.log(`Error crawling ${link}: ${error?.message}`);
-    return new Set();
-  }
+const isSameDomain = (url1: string, url2: string): boolean => {
+  const { hostname: host1 } = new URL(url1);
+  const { hostname: host2 } = new URL(url2);
+  return host1 === host2;
 };
 
-function isWebUrl(url: string): boolean {
+const normalizeUrl = (url: string): string => {
   try {
-    new URL(url);
-    return true;
+    const urlObj = new URL(url);
+    urlObj.hash = '';
+    return urlObj.href;
   } catch (error) {
-    return false;
+    return url;
   }
-}
+};