MODIFIED: lower the chunk size and chunk overlap in groq_pdf func bec…

…ause of rate limit
AquibPy · May 6, 2024 · 44eee4b · 44eee4b
1 parent 6143a89
commit 44eee4b
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 __pycache__/
 venv/
 .env
-.pytest_cache/
+.pytest_cache/
+.vscode
diff --git a/helper_functions.py b/helper_functions.py
@@ -200,7 +200,7 @@ def groq_pdf(pdf,model):
             model_name=model
     )
     text = "".join(page.extract_text() for page in PdfReader(pdf).pages)
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=10000, chunk_overlap=1000)
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
     chunks = text_splitter.split_text(text)
     vectorstore = FAISS.from_texts(chunks, embedding=google_embedding)
     retriever = vectorstore.as_retriever()