added concurrency and max requests parameters

UIUC-Chatbot · Feb 28, 2024 · 3ed598c · 3ed598c
1 parent 9bc772f
commit 3ed598c
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/ai_ta_backend/crawlee_ext_scrape.py b/ai_ta_backend/crawlee_ext_scrape.py
@@ -17,6 +17,8 @@ def crawlee_scrape(course_name: str, urls: list):
                 "match": "",
                 "maxPagesToCrawl": 10000,
                 "maxTokens": 2000000,
+                "maxConcurrency": 20,
+                "maxRequestsPerMinute": 120,
                 "courseName": course_name
             }
     }