Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Conversation Export Fix #229

Closed
wants to merge 1 commit into from
Closed

Conversation Export Fix #229

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions ai_ta_backend/export_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ def export_data_in_bg(response, download_type, course_name, s3_path):
print("admin_emails: ", admin_emails)
print("bcc_emails: ", bcc_emails)


# add a check for emails, don't send email if no admin emails
if len(admin_emails) == 0:
return "No admin emails found. Email not sent."
Expand Down Expand Up @@ -291,8 +292,8 @@ def export_convo_history_json(course_name: str, from_date='', to_date=''):
response = SUPABASE_CLIENT.table("llm-convo-monitor").select("id", count='exact').eq(
"course_name", course_name).gte('created_at', from_date).lte('created_at', to_date).order('id',
desc=False).execute()

if response.count > 1000:
print("response count: ", response.count)
if response.count > 500:
# call background task to upload to s3
filename = course_name + '_' + str(uuid.uuid4()) + '_convo_history.zip'
s3_filepath = s3_file = f"courses/{course_name}/{filename}"
Expand All @@ -308,7 +309,7 @@ def export_convo_history_json(course_name: str, from_date='', to_date=''):
last_id = response.data[-1]['id']
total_count = response.count

filename = course_name + '_' + str(uuid.uuid4()) + '_convo_history.csv'
filename = course_name + '_' + str(uuid.uuid4()) + '_convo_history.json'
file_path = os.path.join(os.getcwd(), filename)
curr_count = 0
# Fetch data in batches of 25 from first_id to last_id
Expand All @@ -330,17 +331,19 @@ def export_convo_history_json(course_name: str, from_date='', to_date=''):
if len(response.data) > 0:
first_id = response.data[-1]['id'] + 1
print("updated first_id: ", first_id)

print("DOWNLOAD DONE.")
# Download file
try:
# zip file
zip_filename = filename.split('.')[0] + '.zip'
zip_file_path = os.path.join(os.getcwd(), zip_filename)

print("ZIP FILE PATH: ", zip_file_path)

with zipfile.ZipFile(zip_file_path, 'w', compression=zipfile.ZIP_DEFLATED) as zipf:
zipf.write(file_path, filename)
os.remove(file_path)

print("IN TRY BLOCK.")
return {"response": (zip_file_path, zip_filename, os.getcwd())}
except Exception as e:
print(e)
Expand Down