From d39fe2de54b4f696854d42e4fa2bf1c2915218f1 Mon Sep 17 00:00:00 2001 From: star-nox Date: Tue, 12 Mar 2024 15:27:54 -0500 Subject: [PATCH 1/3] fixed convo export bug --- ai_ta_backend/service/export_service.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ai_ta_backend/service/export_service.py b/ai_ta_backend/service/export_service.py index 6eb889c2..726f95d1 100644 --- a/ai_ta_backend/service/export_service.py +++ b/ai_ta_backend/service/export_service.py @@ -35,7 +35,7 @@ def export_documents_json(self, course_name: str, from_date='', to_date=''): response = self.sql.getDocumentsBetweenDates(course_name, from_date, to_date, 'documents') # add a condition to route to direct download or s3 download - if response.count > 1000: + if response.count > 500: # call background task to upload to s3 filename = course_name + '_' + str(uuid.uuid4()) + '_documents.zip' @@ -43,7 +43,8 @@ def export_documents_json(self, course_name: str, from_date='', to_date=''): # background task of downloading data - map it with above ID executor = ProcessPoolExecutor() executor.submit(self.export_data_in_bg, response, "documents", course_name, s3_filepath) - return {"response": 'Download from S3', "s3_path": s3_filepath} + return {"response": 'There are 500+ documents in the export. You will receive an email with the download link shortly!', + "s3_path": s3_filepath} else: # Fetch data @@ -206,13 +207,13 @@ def export_convo_history_json(self, course_name: str, from_date='', to_date=''): from_date (str, optional): The start date for the data export. Defaults to ''. to_date (str, optional): The end date for the data export. Defaults to ''. """ - print("Exporting conversation history to csv file...") + print("Exporting conversation history to json file...") response = self.sql.getDocumentsBetweenDates(course_name, from_date, to_date, 'llm-convo-monitor') - if response.count > 1000: + if response.count > 500: # call background task to upload to s3 - filename = course_name + '_' + str(uuid.uuid4()) + '_convo_history.zip' + filename = course_name + '_' + str(uuid.uuid4()) + '_convo_history.json' s3_filepath = f"courses/{course_name}/{filename}" # background task of downloading data - map it with above ID executor = ProcessPoolExecutor() From 0c768541a61007c9cef93c5add7a841cf7d870b3 Mon Sep 17 00:00:00 2001 From: star-nox Date: Tue, 12 Mar 2024 15:30:33 -0500 Subject: [PATCH 2/3] file type minor fix --- ai_ta_backend/service/export_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ai_ta_backend/service/export_service.py b/ai_ta_backend/service/export_service.py index 726f95d1..253ab1fa 100644 --- a/ai_ta_backend/service/export_service.py +++ b/ai_ta_backend/service/export_service.py @@ -213,7 +213,7 @@ def export_convo_history_json(self, course_name: str, from_date='', to_date=''): if response.count > 500: # call background task to upload to s3 - filename = course_name + '_' + str(uuid.uuid4()) + '_convo_history.json' + filename = course_name + '_' + str(uuid.uuid4()) + '_convo_history.zip' s3_filepath = f"courses/{course_name}/{filename}" # background task of downloading data - map it with above ID executor = ProcessPoolExecutor() @@ -227,7 +227,7 @@ def export_convo_history_json(self, course_name: str, from_date='', to_date=''): last_id = response.data[-1]['id'] total_count = response.count - filename = course_name + '_' + str(uuid.uuid4()) + '_convo_history.csv' + filename = course_name + '_' + str(uuid.uuid4()) + '_convo_history.json' file_path = os.path.join(os.getcwd(), filename) curr_count = 0 # Fetch data in batches of 25 from first_id to last_id From 2107aae1b35a7475822ee487a7e81d704d956e29 Mon Sep 17 00:00:00 2001 From: star-nox Date: Wed, 13 Mar 2024 16:30:22 -0500 Subject: [PATCH 3/3] reverse response text for doc export --- ai_ta_backend/service/export_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ai_ta_backend/service/export_service.py b/ai_ta_backend/service/export_service.py index 253ab1fa..1c300bc4 100644 --- a/ai_ta_backend/service/export_service.py +++ b/ai_ta_backend/service/export_service.py @@ -43,7 +43,7 @@ def export_documents_json(self, course_name: str, from_date='', to_date=''): # background task of downloading data - map it with above ID executor = ProcessPoolExecutor() executor.submit(self.export_data_in_bg, response, "documents", course_name, s3_filepath) - return {"response": 'There are 500+ documents in the export. You will receive an email with the download link shortly!', + return {"response": 'Download from S3', "s3_path": s3_filepath} else: