Major refactor introducing Dependency injection

.env.template

@@ -5,7 +5,7 @@ SUPABASE_READ_ONLY=
 SUPABASE_JWT_SECRET=
 
 MATERIALS_SUPABASE_TABLE=uiuc_chatbot
-NEW_NEW_NEWNEW_MATERIALS_SUPABASE_TABLE=documents
+SUPABASE_DOCUMENTS_TABLE=documents
 
 # QDRANT
 QDRANT_COLLECTION_NAME=uiuc-chatbot

ai_ta_backend/beam/ingest.py

@@ -23,7 +23,7 @@
 import pytesseract
 import sentry_sdk
 import supabase
-from beam import App, QueueDepthAutoscaler, RequestLatencyAutoscaler, Runtime
+from beam import App, QueueDepthAutoscaler, Runtime  # RequestLatencyAutoscaler,
 from bs4 import BeautifulSoup
 from git.repo import Repo
 from langchain.document_loaders import (
@@ -962,7 +962,7 @@ def split_and_upload(self, texts: List[str], metadatas: List[Dict[str, Any]]):
       }
 
       response = self.supabase_client.table(
-          os.getenv('NEW_NEW_NEWNEW_MATERIALS_SUPABASE_TABLE')).insert(document).execute()  # type: ignore
+          os.getenv('SUPABASE_DOCUMENTS_TABLE')).insert(document).execute()  # type: ignore
 
       # add to Nomic document map
       if len(response.data) > 0:
@@ -991,7 +991,7 @@ def check_for_duplicates(self, texts: List[Dict], metadatas: List[Dict[str, Any]
     For given metadata, fetch docs from Supabase based on S3 path or URL.
     If docs exists, concatenate the texts and compare with current texts, if same, return True.
     """
-    doc_table = os.getenv('NEW_NEW_NEWNEW_MATERIALS_SUPABASE_TABLE', '')
+    doc_table = os.getenv('SUPABASE_DOCUMENTS_TABLE', '')
     course_name = metadatas[0]['course_name']
     incoming_s3_path = metadatas[0]['s3_path']
     url = metadatas[0]['url']
@@ -1090,8 +1090,8 @@ def delete_data(self, course_name: str, s3_path: str, source_url: str):
         try:
           # delete from Nomic
           response = self.supabase_client.from_(
-              os.environ['NEW_NEW_NEWNEW_MATERIALS_SUPABASE_TABLE']).select("id, s3_path, contexts").eq(
-                  's3_path', s3_path).eq('course_name', course_name).execute()
+              os.environ['SUPABASE_DOCUMENTS_TABLE']).select("id, s3_path, contexts").eq('s3_path', s3_path).eq(
+                  'course_name', course_name).execute()
           data = response.data[0]  #single record fetched
           nomic_ids_to_delete = []
           context_count = len(data['contexts'])
@@ -1105,8 +1105,8 @@ def delete_data(self, course_name: str, s3_path: str, source_url: str):
           sentry_sdk.capture_exception(e)
 
         try:
-          self.supabase_client.from_(os.environ['NEW_NEW_NEWNEW_MATERIALS_SUPABASE_TABLE']).delete().eq(
-              's3_path', s3_path).eq('course_name', course_name).execute()
+          self.supabase_client.from_(os.environ['SUPABASE_DOCUMENTS_TABLE']).delete().eq('s3_path', s3_path).eq(
+              'course_name', course_name).execute()
         except Exception as e:
           print("Error in deleting file from supabase:", e)
           sentry_sdk.capture_exception(e)
@@ -1134,9 +1134,8 @@ def delete_data(self, course_name: str, s3_path: str, source_url: str):
             sentry_sdk.capture_exception(e)
         try:
           # delete from Nomic
-          response = self.supabase_client.from_(
-              os.environ['NEW_NEW_NEWNEW_MATERIALS_SUPABASE_TABLE']).select("id, url, contexts").eq(
-                  'url', source_url).eq('course_name', course_name).execute()
+          response = self.supabase_client.from_(os.environ['SUPABASE_DOCUMENTS_TABLE']).select("id, url, contexts").eq(
+              'url', source_url).eq('course_name', course_name).execute()
           data = response.data[0]  #single record fetched
           nomic_ids_to_delete = []
           context_count = len(data['contexts'])
@@ -1151,8 +1150,8 @@ def delete_data(self, course_name: str, s3_path: str, source_url: str):
 
         try:
           # delete from Supabase
-          self.supabase_client.from_(os.environ['NEW_NEW_NEWNEW_MATERIALS_SUPABASE_TABLE']).delete().eq(
-              'url', source_url).eq('course_name', course_name).execute()
+          self.supabase_client.from_(os.environ['SUPABASE_DOCUMENTS_TABLE']).delete().eq('url', source_url).eq(
+              'course_name', course_name).execute()
         except Exception as e:
           print("Error in deleting file from supabase:", e)
           sentry_sdk.capture_exception(e)