modified function to accept urls as a post request

UIUC-Chatbot · Feb 28, 2024 · 9bc772f · 9bc772f
1 parent fbcdd05
commit 9bc772f
Showing 1 changed file with 2 additions and 4 deletions.
diff --git a/ai_ta_backend/crawlee_ext_scrape.py b/ai_ta_backend/crawlee_ext_scrape.py
@@ -9,15 +9,13 @@ def crawlee_scrape(course_name: str, urls: list):
     This function takes in a pre-defined set of URLs and scrapes the content from each URL.
     """
     print("Scraping URLs:", urls)
-    print("Course Name:", course_name)
-    exit()
-
+
     payload = {
             "params": {
                 "url": "",
                 "scrapeStrategy": "equal-and-below",
                 "match": "",
-                "maxPagesToCrawl": 20000,
+                "maxPagesToCrawl": 10000,
                 "maxTokens": 2000000,
                 "courseName": course_name
             }