fix(WebScraper/sitemap): await urlsHandler to fix race condition

mendableai · Dec 30, 2024 · 71a8f74 · 71a8f74
1 parent 8ae34a0
commit 71a8f74
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -506,7 +506,7 @@ export class WebCrawler {
           // Get all links from the main domain's sitemap
           sitemapCount += await getLinksFromSitemap(
             { sitemapUrl: mainDomainSitemapUrl, urlsHandler(urls) {
-              urlsHandler(urls.filter(link => {
+              return urlsHandler(urls.filter(link => {
                 try {
                   const linkUrl = new URL(link);
                   return linkUrl.hostname.endsWith(hostname);

diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -103,7 +103,11 @@ export async function getLinksFromSitemap(
         )
         .map((url) => url.loc[0]);
       count += validUrls.length;
-      urlsHandler(validUrls);
+
+      const h = urlsHandler(validUrls);
+      if (h instanceof Promise) {
+        await h;
+      }
     }
 
     return count;