mendableai · mogery · Nov 14, 2024 · Nov 14, 2024 · Nov 14, 2024 · Nov 14, 2024
diff --git a/apps/api/.gitignore b/apps/api/.gitignore
@@ -9,3 +9,5 @@ dump.rdb
 
 .rdb
 .sentryclirc
+
+doctor-*.html
diff --git a/apps/api/requests.http b/apps/api/requests.http
@@ -46,4 +46,13 @@ content-type: application/json
 @batchScrapeId = {{batchScrape.response.body.$.id}}
 # @name batchScrapeStatus
 GET {{baseUrl}}/v1/crawl/{{batchScrapeId}} HTTP/1.1
-Authorization: Bearer {{$dotenv TEST_API_KEY}}
+Authorization: Bearer {{$dotenv TEST_API_KEY}}
+
+### URL Doctor
+# @name urlDoctor
+POST {{baseUrl}}/admin/{{$dotenv BULL_AUTH_KEY}}/doctor HTTP/1.1
+Content-Type: application/json
+
+{
+  "url": "https://firecrawl.dev"
+}
diff --git a/apps/api/src/controllers/v1/admin/doctor-status.ts b/apps/api/src/controllers/v1/admin/doctor-status.ts
@@ -0,0 +1,104 @@
+import { Request, Response } from "express";
+import { logger as _logger } from "../../../lib/logger";
+import { ScrapeUrlResponse } from "../../../scraper/scrapeURL";
+import { getScrapeQueue, redisConnection } from "../../../services/queue-service";
+import type { Permutation } from "./doctor";
+import { Job } from "bullmq";
+
+const logger = _logger.child({ module: "doctorStatusController" });
+
+const errorReplacer = (_, value) => {
+    if (value instanceof Error) {
+        return {
+            ...value,
+            name: value.name,
+            message: value.message,
+            stack: value.stack,
+            cause: value.cause,
+        }
+    } else {
+        return value;
+    }
+};
+
+type PermutationResult = ({
+    state: "done",
+    result: ScrapeUrlResponse & {
+        success: true
+    },
+} | {
+    state: "thrownError",
+    error: string | Error | null | undefined,
+} | {
+    state: "error",
+    result: ScrapeUrlResponse & {
+        success: false
+    },
+} | {
+    state: "pending",
+}) & {
+    permutation: Permutation,
+};
+
+export async function doctorStatusController(req: Request, res: Response) {
+  try {
+    const doctorId = req.params.id;
+
+    const meta: { url: string } | null = JSON.parse(await redisConnection.get("doctor:" + doctorId) ?? "null");
+    const permutations: Permutation[] | null = JSON.parse(await redisConnection.get("doctor:" + doctorId + ":permutations") ?? "null");
+    if (permutations === null || meta === null) {
+        return res.status(404).json({ error: "Doctor entry not found" });
+    }
+
+    const jobs = (await Promise.all(permutations.map(x => getScrapeQueue().getJob(x.jobId)))).filter(x => x) as Job<unknown, ScrapeUrlResponse>[];
+
+    const results: PermutationResult[] = await Promise.all(jobs.map(async job => {
+        const permutation = permutations.find(x => x.jobId === job.id)!;
+        const state = await job.getState();
+        if (state === "completed" && job.data) {
+            if (job.returnvalue.success) {
+                return {
+                    state: "done",
+                    result: job.returnvalue,
+                    permutation,
+                }
+            } else {
+                return {
+                    state: "error",
+                    result: job.returnvalue,
+                    permutation,
+                }
+            }
+        } else if (state === "failed") {
+            return {
+                state: "thrownError",
+                error: job.failedReason,
+                permutation,
+            }
+        } else {
+            return {
+                state: "pending",
+                permutation,
+            }
+        }
+    }));
+
+    const html = "<head><meta charset=\"utf8\"></head><body style=\"font-family: sans-serif; padding: 1rem;\"><h1>Doctor</h1><p>URL: <code>" + meta.url + "</code></p>"
+        + results.map(x => "<h2>" + (x.state === "pending" ? "⏳" : x.state === "done" ? "✅" : "❌") + " " + x.permutation.name + "</h2><p>Scrape options: <code>" + JSON.stringify(x.permutation.options) + "</code></p>"
+            + "<p>Internal options: <code>" + JSON.stringify(x.permutation.internal) + "</code></p>"
+            + (x.state !== "pending" ? ("<code><pre>" + ((x.state === "done"
+                ? JSON.stringify(x.result, errorReplacer, 4)
+                : x.state === "thrownError"
+                    ? (x.error instanceof Error
+                        ? (x.error.message + "\n" + (x.error.stack ?? ""))
+                        : (x.error ?? "<unknown error>")) 
+                    : (JSON.stringify(x.result, errorReplacer, 4))))
+                .replaceAll("<", "&lt;").replaceAll(">", "&gt;") + "</pre></code>"): "")).join("") 
+        + "</body>"
+
+    res.header("Content-Type", "text/html").send(html);
+  } catch (error) {
+    logger.error("Doctor status error", { error });
+    res.status(500).json({ error: "Internal server error" });
+  }
+}
diff --git a/apps/api/src/controllers/v1/admin/doctor.ts b/apps/api/src/controllers/v1/admin/doctor.ts
@@ -0,0 +1,84 @@
+import { Request, Response } from "express";
+import { logger as _logger } from "../../../lib/logger";
+import { ScrapeUrlResponse, InternalOptions } from "../../../scraper/scrapeURL";
+import { z } from "zod";
+import { scrapeOptions } from "../types";
+import { Engine, engineOptions, engines } from "../../../scraper/scrapeURL/engines";
+import { addScrapeJob, addScrapeJobs } from "../../../services/queue-jobs";
+import { redisConnection } from "../../../services/queue-service";
+
+const logger = _logger.child({ module: "doctorController" });
+
+export type Permutation = {
+    options: z.input<typeof scrapeOptions>,
+    internal: InternalOptions,
+    name: string,
+    jobId: string,
+};
+
+export async function doctorController(req: Request, res: Response) {
+  try {
+    const doctorId = crypto.randomUUID();
+
+    const permutations: Permutation[] = [
+        { options: {}, internal: { verbose: true }, name: "bare", jobId: crypto.randomUUID() },
+        ...Object.entries(engineOptions).filter(([name, options]) => options.quality > 0 && engines.includes(name as Engine)).map(([name, _options]) => ({
+            options: {}, internal: { forceEngine: name as Engine, verbose: true }, name, jobId: crypto.randomUUID(),
+        })),
+    ];
+
+    await addScrapeJobs(permutations.map(perm => ({
+        data: {
+            url: req.body.url,
+            mode: "single_urls",
+            team_id: null,
+            scrapeOptions: scrapeOptions.parse(perm.options),
+            internalOptions: perm.internal,
+            plan: null,
+            origin: "doctor",
+            is_scrape: true,
+            doctor: true,
+        },
+        opts: {
+            jobId: perm.jobId,
+            priority: 10,
+        },
+    })));
+
+    await redisConnection.set("doctor:" + doctorId, JSON.stringify({ url: req.body.url }), "EX", 86400);
+    await redisConnection.set("doctor:" + doctorId + ":permutations", JSON.stringify(permutations), "EX", 86400);
+
+    const protocol = process.env.ENV === "local" ? req.protocol : "https";
+
+    res.json({ ok: true, id: doctorId, url: `${protocol}://${req.get("host")}/admin/${process.env.BULL_AUTH_KEY}/doctor/${doctorId}` });
+
+    // await Promise.all(permutations.map(async perm => {
+        // try {
+        //     const result = await scrapeURL(doctorId + ":bare", url, scrapeOptions.parse(perm.options), perm.internal);
+        //     if (result.success) { 
+        //         results.push({
+        //             state: "done",
+        //             result,
+        //             permutation: perm,
+        //         });
+        //     } else {
+        //         results.push({
+        //             state: "error",
+        //             result,
+        //             permutation: perm,
+        //         });
+        //     }
+        // } catch (error) {
+        //     console.error("Permutation " + perm.name + " failed with error", { error });
+        //     results.push({
+        //         state: "thrownError",
+        //         error,
+        //         permutation: perm,
+        //     });
+        // }
+    // }));
+  } catch (error) {
+    logger.error("Doctor error", { error });
+    res.status(500).json({ error: "Internal server error" });
+  }
+}
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -16,6 +16,7 @@ import { logCrawl } from "../../services/logging/crawl_log";
 import { getScrapeQueue } from "../../services/queue-service";
 import { getJobPriority } from "../../lib/job-priority";
 import { addScrapeJobs } from "../../services/queue-jobs";
+import { callWebhook } from "../../services/webhook";
 
 export async function batchScrapeController(
   req: RequestWithAuth<{}, CrawlResponse, BatchScrapeRequest>,
@@ -66,6 +67,7 @@ export async function batchScrapeController(
         crawl_id: id,
         sitemapped: true,
         v1: true,
+        webhook: req.body.webhook,
       },
       opts: {
         jobId: uuidv4(),
@@ -85,6 +87,10 @@ export async function batchScrapeController(
   );
   await addScrapeJobs(jobs);
 
+  if(req.body.webhook) {
+    await callWebhook(req.auth.team_id, id, null, req.body.webhook, true, "batch_scrape.started");
+  }
+
   const protocol = process.env.ENV === "local" ? req.protocol : "https";
 
   return res.status(200).json({
Original file line number	Diff line number	Diff line change
Expand Up		@@ -9,3 +9,5 @@ dump.rdb

		.rdb
		.sentryclirc

		doctor-*.html