Add progress bar via polling

src/api/crawlee.ts

@@ -6,14 +6,19 @@ import axios from 'axios';
 
 import { Config, configSchema } from "./configValidation.js";
 import { ingestPdf, uploadPdfToS3 } from "./uploadToS3.js";
+import { setProgressForScrape } from "./progressStore.js";
 
-export async function crawl(config: Config) {
+export async function crawl(config: Config & { scrapeId: string }) {
   configSchema.parse(config);
   console.log("Crawling with config:", config)
   let pageCounter = 0;
   const ingestionPromises: Promise<any>[] = [];
   // const results: Array<{ title: string; url: string; html: string }> = [];
 
+  const updateProgress = (scrapeId: string, progress: number) => {
+    setProgressForScrape(scrapeId, progress);
+  };
+
   if (config.url) {
     console.log(`Crawling URL: ${config.url}`);
     if (process.env.NO_CRAWL !== "true") {
@@ -28,6 +33,7 @@ export async function crawl(config: Config) {
           log.info(
             `Crawling: Page ${pageCounter} / ${config.maxPagesToCrawl} - URL: ${request.loadedUrl}...`,
           );
+          updateProgress(config.scrapeId, (pageCounter / config.maxPagesToCrawl));
 
           // Use custom handling for XPath selector
           if (config.selector) {

src/api/progressStore.ts

@@ -0,0 +1,13 @@
+const progressStore: Record<string, number> = {};
+
+export function setProgressForScrape(scrapeId: string, progress: number) {
+    progressStore[scrapeId] = progress;
+}
+
+export function getProgressForScrape(scrapeId: string): number {
+    return progressStore[scrapeId] || 0;
+}
+
+export function clearProgressForScrape(scrapeId: string) {
+    delete progressStore[scrapeId];
+}

src/main.ts

@@ -2,6 +2,9 @@
 import express, { Request, Response } from 'express';
 import { crawl } from './api/crawlee.js';
 import cors from 'cors';
+// import { v4 as uuidv4 } from 'uuid';
+import { clearProgressForScrape, getProgressForScrape } from './api/progressStore.js';
+
 
 const app = express();
 
@@ -15,8 +18,10 @@ app.use(express.json());
 
 app.post('/crawl', async (req: Request, res: Response) => {
     console.log('in /crawl. req.body:', req.body)
+    // TODO: Frontend should send the UUID to make fetching results easier.
+    // const scrapeId = uuidv4(); // Generate a unique ID for this scrape session
     try {
-        const { url, scrapeStrategy, match, maxPagesToCrawl, courseName, maxTokens } = req.body.params;
+        const { url, scrapeStrategy, match, maxPagesToCrawl, courseName, maxTokens, scrapeId } = req.body.params;
         console.log('in /crawl -- got variables :) url:', url, 'scrapeStrategy:', scrapeStrategy, 'match', match, 'maxPagesToCrawl:', maxPagesToCrawl, 'maxTokens:', maxTokens, 'courseName:', courseName)
 
         const config = {
@@ -29,9 +34,15 @@ app.post('/crawl', async (req: Request, res: Response) => {
             exclude: ["https://www.facebook.com/**", "https://youtube.com/**", "https://linkedin.com/**", "https://instagram.com/**"],
         };
 
-        const results = await crawl(config);
+        // const results = await crawl(config);
+        const results = await crawl({ ...config, scrapeId });
+
         console.log(`Crawl completed successfully. Number of results: ${results}`);
-        res.status(200).json(results);
+        // res.status(200).json(results);
+        res.status(200).json({ results, scrapeId });
+        if (scrapeId) {
+            clearProgressForScrape(scrapeId);
+        }
     } catch (error) {
         const e = error as Error;
         res.status(500).json({ error: 'An error occurred during the upload', errorTitle: e.name, errorMessage: e.message });
@@ -42,8 +53,30 @@ app.post('/crawl', async (req: Request, res: Response) => {
     }
 });
 
+app.get('/progress/:scrapeId', (req: Request, res: Response) => {
+    const scrapeId = req.params.scrapeId;
+    const progress = getProgressForScrape(scrapeId);
+    res.status(200).json({ progress });
+});
 
 const PORT = process.env.PORT || 3000;
 app.listen(PORT, () => {
     console.log(`Server running on port ${PORT}`);
-});
+});
+
+
+// TODO finish frontend polling
+
+// Example using JavaScript setInterval
+// const scrapeId = 'receivedScrapeIdFromServer'; // Replace with actual scrapeId from server response
+// const progressCheckInterval = setInterval(() => {
+//     fetch(`/progress/${scrapeId}`)
+//         .then(response => response.json())
+//         .then(data => {
+//             updateProgressBar(data.progress); // Implement this function to update the UI
+//             if (data.progress >= 100) {
+//                 clearInterval(progressCheckInterval); // Stop polling when complete
+//             }
+//         })
+//         .catch(error => console.error('Error fetching progress:', error));
+// }, 2000); // Poll every 2 seconds