diff --git a/components/scout-settings-modal.tsx b/components/scout-settings-modal.tsx index f6a5a39..a5df774 100644 --- a/components/scout-settings-modal.tsx +++ b/components/scout-settings-modal.tsx @@ -23,6 +23,13 @@ import { supabase } from "@/lib/supabase/client"; import { X, Plus, Pencil } from "lucide-react"; import posthog from "posthog-js"; +type ScrapeOptions = { + cookies?: string; + headers?: Record; + waitFor?: number | string; + timeout?: number; +}; + type Scout = { id: string; title: string; @@ -38,6 +45,7 @@ type Scout = { } | null; frequency: "daily" | "every_3_days" | "weekly" | null; is_active: boolean; + scrape_options?: ScrapeOptions; }; type Location = { @@ -73,6 +81,8 @@ export function ScoutSettingsModal({ >(null); const [isActive, setIsActive] = useState(false); const [isSaving, setIsSaving] = useState(false); + // Scrape options state + const [cookies, setCookies] = useState(""); // Track which fields are in edit mode const [editMode, setEditMode] = useState({ @@ -80,6 +90,7 @@ export function ScoutSettingsModal({ goal: false, description: false, location: false, + cookies: false, searchQueries: false, frequency: false, }); @@ -98,6 +109,7 @@ export function ScoutSettingsModal({ setSearchQueries(scout.search_queries || []); setFrequency(scout.frequency); setIsActive(scout.is_active); + setCookies(scout.scrape_options?.cookies || ""); // Check if location is "any" (0, 0) or a real location const isAnyLocation = scout.location && @@ -115,6 +127,7 @@ export function ScoutSettingsModal({ goal: false, description: false, location: false, + cookies: false, searchQueries: false, frequency: false, }); @@ -172,6 +185,14 @@ export function ScoutSettingsModal({ locationToSave = scout.location; } + // Build scrape_options object + const scrapeOptions: ScrapeOptions = {}; + if (cookies.trim()) { + scrapeOptions.cookies = cookies.trim(); + } + // Only save if there are options set + const scrapeOptionsToSave = Object.keys(scrapeOptions).length > 0 ? scrapeOptions : null; + const { error } = await supabase .from("scouts") .update({ @@ -181,6 +202,7 @@ export function ScoutSettingsModal({ search_queries: searchQueries, frequency, location: locationToSave, + scrape_options: scrapeOptionsToSave, }) .eq("id", scout.id); @@ -552,6 +574,46 @@ export function ScoutSettingsModal({ )} + {/* Cookies (Advanced) */} +
+
+
+ +

+ Optional: Send cookies when scraping websites (for authenticated content) +

+
+ +
+ {editMode.cookies ? ( +