diff --git a/ai_ta_backend/main.py b/ai_ta_backend/main.py index 1f2ee238..78fab519 100644 --- a/ai_ta_backend/main.py +++ b/ai_ta_backend/main.py @@ -427,10 +427,10 @@ def logToNomic(): # background execution of tasks!! response = executor.submit(log_convo_to_nomic, course_name, data) - #response = jsonify(response) - #response.headers.add('Access-Control-Allow-Origin', '*') - print(response) - return "response" + response = jsonify({'outcome': 'success'}) + response.headers.add('Access-Control-Allow-Origin', '*') + + return response if __name__ == '__main__': diff --git a/ai_ta_backend/nomic_logging.py b/ai_ta_backend/nomic_logging.py index 9e816302..b92be51d 100644 --- a/ai_ta_backend/nomic_logging.py +++ b/ai_ta_backend/nomic_logging.py @@ -58,7 +58,10 @@ def log_convo_to_nomic(course_name: str, conversation) -> str: embeddings = map_embeddings_df[prev_index-1].reshape(1, 1536) prev_convo = prev_data['conversation'].values[0] prev_id = prev_data['id'].values[0] + print("prev_id: ", prev_id) prev_created_at = prev_data['created_at'].values[0] + print("prev_created_at: ", prev_created_at) + print("before delete") # delete that convo data point from Nomic project.delete_data([prev_id]) diff --git a/ai_ta_backend/nomic_map_creation.ipynb b/ai_ta_backend/nomic_map_creation.ipynb new file mode 100644 index 00000000..c11b80db --- /dev/null +++ b/ai_ta_backend/nomic_map_creation.ipynb @@ -0,0 +1,1915 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "# import required libraries\n", + "\n", + "import os\n", + "import supabase\n", + "from nomic import atlas\n", + "from dotenv import load_dotenv\n", + "from langchain.embeddings import OpenAIEmbeddings\n", + "import numpy as np\n", + "import time\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# loading environment variables\n", + "\n", + "env_path = \"../.env\"\n", + "load_dotenv(dotenv_path=env_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "# initialize supabase client\n", + "\n", + "url = os.environ.get(\"SUPABASE_URL\")\n", + "key = os.environ.get(\"SUPABASE_API_KEY\")\n", + "\n", + "supabase_client = supabase.create_client(url, key)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idcreated_atconvoconvo_idcourse_nameuser_email
051932023-09-06T23:21:36.639848+00:00{'id': 'd6e83182-56d8-4151-b84e-a09dd84b8836',...d6e83182-56d8-4151-b84e-a09dd84b8836cropwizard-betadabholkar.asmita@gmail.com
151922023-09-06T23:04:50.982857+00:00{'id': '1390b226-2115-4078-a594-ed4bad0fb4e0',...1390b226-2115-4078-a594-ed4bad0fb4e0cropwizard-betakastanday@live.com
251742023-09-06T22:22:44.107536+00:00{'id': 'fca0cf0b-6bac-4cf6-bd4d-d3501827c947',...fca0cf0b-6bac-4cf6-bd4d-d3501827c947cropwizard-betadabholkar.asmita@gmail.com
351842023-09-06T23:01:06.796384+00:00{'id': '1572f8a1-9ceb-42f7-b39f-fe95ec2b3c0e',...1572f8a1-9ceb-42f7-b39f-fe95ec2b3c0ecropwizard-betadabholkar.asmita@gmail.com
451822023-09-06T22:58:21.66316+00:00{'id': '0c9d9873-2c52-4b12-90ec-d4a495cbf4e0',...0c9d9873-2c52-4b12-90ec-d4a495cbf4e0cropwizard-betadabholkar.asmita@gmail.com
551942023-09-06T23:22:06.786717+00:00{'id': 'd51bbdd8-c5c8-4e5b-a003-556a8ac74726',...d51bbdd8-c5c8-4e5b-a003-556a8ac74726cropwizard-betaavd6@illinois.edu
651132023-09-05T21:34:17.944623+00:00{'id': '2a39551b-7b6c-4ba0-aa77-ffc9b1316718',...2a39551b-7b6c-4ba0-aa77-ffc9b1316718cropwizard-betahetarth2@illinois.edu
753332023-09-07T22:25:52.858867+00:00{'id': 'bd3c9174-4512-4ac6-ae05-8aafa4099215',...bd3c9174-4512-4ac6-ae05-8aafa4099215cropwizard-betadabholkar.asmita@gmail.com
851082023-09-05T21:11:51.289178+00:00{'id': 'a49941c4-2049-478b-b43e-9c574cadbd05',...a49941c4-2049-478b-b43e-9c574cadbd05cropwizard-betavadve@illinois.edu
951042023-09-05T20:38:27.687893+00:00{'id': '1be7956c-d5c8-4b1f-a62f-145421a2e7f7',...1be7956c-d5c8-4b1f-a62f-145421a2e7f7cropwizard-betakastanday@live.com
1051032023-09-05T20:11:51.810222+00:00{'id': '20e6e160-6fd3-4e0a-82b7-98457f2ff1c3',...20e6e160-6fd3-4e0a-82b7-98457f2ff1c3cropwizard-betakastanday@live.com
1150982023-09-05T19:17:36.190976+00:00{'id': 'fbd19bd0-373b-4a9c-b58c-a47cbd4e9e1c',...fbd19bd0-373b-4a9c-b58c-a47cbd4e9e1ccropwizard-betadabholkar.asmita@gmail.com
1250422023-09-02T14:43:02.160608+00:00{'id': '8df8a436-adbf-441a-92ec-f6d926aee789',...8df8a436-adbf-441a-92ec-f6d926aee789cropwizard-betaNone
1350092023-09-01T18:26:57.628766+00:00{'id': 'b401031c-5540-43fc-baff-7c6db90ff3ae',...b401031c-5540-43fc-baff-7c6db90ff3aecropwizard-betarohan13@illinois.edu
1450062023-09-01T18:06:33.350981+00:00{'id': 'c42e70a8-03c0-4c9e-ac7f-ee718b4ac968',...c42e70a8-03c0-4c9e-ac7f-ee718b4ac968cropwizard-betahetarth2@illinois.edu
1550452023-09-03T20:07:34.895841+00:00{'id': 'cbecc7f4-a94c-49d6-ae1a-e42f20136676',...cbecc7f4-a94c-49d6-ae1a-e42f20136676cropwizard-betaNone
1650142023-09-01T18:36:04.202716+00:00{'id': '3303ec4f-84d5-4eec-a4b3-af020e62b79d',...3303ec4f-84d5-4eec-a4b3-af020e62b79dcropwizard-betavadve@illinois.edu
1750132023-09-01T18:35:32.12103+00:00{'id': 'fc91f681-54e3-4df5-8fd4-952d02b8feba',...fc91f681-54e3-4df5-8fd4-952d02b8febacropwizard-betarohan13@illinois.edu
1850112023-09-01T18:28:24.588312+00:00{'id': '5c2d89b3-3e41-4a8a-84be-f93c7b8127c3',...5c2d89b3-3e41-4a8a-84be-f93c7b8127c3cropwizard-betarohan13@illinois.edu
1950072023-09-01T18:15:08.636935+00:00{'id': 'd7191543-c517-4007-a2fb-ae28660ef37c',...d7191543-c517-4007-a2fb-ae28660ef37ccropwizard-betaNone
2050042023-09-01T16:28:23.202471+00:00{'id': 'edabed51-e581-48f7-865c-89a4b1d9d120',...edabed51-e581-48f7-865c-89a4b1d9d120cropwizard-betadabholkar.asmita@gmail.com
\n", + "
" + ], + "text/plain": [ + " id created_at \\\n", + "0 5193 2023-09-06T23:21:36.639848+00:00 \n", + "1 5192 2023-09-06T23:04:50.982857+00:00 \n", + "2 5174 2023-09-06T22:22:44.107536+00:00 \n", + "3 5184 2023-09-06T23:01:06.796384+00:00 \n", + "4 5182 2023-09-06T22:58:21.66316+00:00 \n", + "5 5194 2023-09-06T23:22:06.786717+00:00 \n", + "6 5113 2023-09-05T21:34:17.944623+00:00 \n", + "7 5333 2023-09-07T22:25:52.858867+00:00 \n", + "8 5108 2023-09-05T21:11:51.289178+00:00 \n", + "9 5104 2023-09-05T20:38:27.687893+00:00 \n", + "10 5103 2023-09-05T20:11:51.810222+00:00 \n", + "11 5098 2023-09-05T19:17:36.190976+00:00 \n", + "12 5042 2023-09-02T14:43:02.160608+00:00 \n", + "13 5009 2023-09-01T18:26:57.628766+00:00 \n", + "14 5006 2023-09-01T18:06:33.350981+00:00 \n", + "15 5045 2023-09-03T20:07:34.895841+00:00 \n", + "16 5014 2023-09-01T18:36:04.202716+00:00 \n", + "17 5013 2023-09-01T18:35:32.12103+00:00 \n", + "18 5011 2023-09-01T18:28:24.588312+00:00 \n", + "19 5007 2023-09-01T18:15:08.636935+00:00 \n", + "20 5004 2023-09-01T16:28:23.202471+00:00 \n", + "\n", + " convo \\\n", + "0 {'id': 'd6e83182-56d8-4151-b84e-a09dd84b8836',... \n", + "1 {'id': '1390b226-2115-4078-a594-ed4bad0fb4e0',... \n", + "2 {'id': 'fca0cf0b-6bac-4cf6-bd4d-d3501827c947',... \n", + "3 {'id': '1572f8a1-9ceb-42f7-b39f-fe95ec2b3c0e',... \n", + "4 {'id': '0c9d9873-2c52-4b12-90ec-d4a495cbf4e0',... \n", + "5 {'id': 'd51bbdd8-c5c8-4e5b-a003-556a8ac74726',... \n", + "6 {'id': '2a39551b-7b6c-4ba0-aa77-ffc9b1316718',... \n", + "7 {'id': 'bd3c9174-4512-4ac6-ae05-8aafa4099215',... \n", + "8 {'id': 'a49941c4-2049-478b-b43e-9c574cadbd05',... \n", + "9 {'id': '1be7956c-d5c8-4b1f-a62f-145421a2e7f7',... \n", + "10 {'id': '20e6e160-6fd3-4e0a-82b7-98457f2ff1c3',... \n", + "11 {'id': 'fbd19bd0-373b-4a9c-b58c-a47cbd4e9e1c',... \n", + "12 {'id': '8df8a436-adbf-441a-92ec-f6d926aee789',... \n", + "13 {'id': 'b401031c-5540-43fc-baff-7c6db90ff3ae',... \n", + "14 {'id': 'c42e70a8-03c0-4c9e-ac7f-ee718b4ac968',... \n", + "15 {'id': 'cbecc7f4-a94c-49d6-ae1a-e42f20136676',... \n", + "16 {'id': '3303ec4f-84d5-4eec-a4b3-af020e62b79d',... \n", + "17 {'id': 'fc91f681-54e3-4df5-8fd4-952d02b8feba',... \n", + "18 {'id': '5c2d89b3-3e41-4a8a-84be-f93c7b8127c3',... \n", + "19 {'id': 'd7191543-c517-4007-a2fb-ae28660ef37c',... \n", + "20 {'id': 'edabed51-e581-48f7-865c-89a4b1d9d120',... \n", + "\n", + " convo_id course_name \\\n", + "0 d6e83182-56d8-4151-b84e-a09dd84b8836 cropwizard-beta \n", + "1 1390b226-2115-4078-a594-ed4bad0fb4e0 cropwizard-beta \n", + "2 fca0cf0b-6bac-4cf6-bd4d-d3501827c947 cropwizard-beta \n", + "3 1572f8a1-9ceb-42f7-b39f-fe95ec2b3c0e cropwizard-beta \n", + "4 0c9d9873-2c52-4b12-90ec-d4a495cbf4e0 cropwizard-beta \n", + "5 d51bbdd8-c5c8-4e5b-a003-556a8ac74726 cropwizard-beta \n", + "6 2a39551b-7b6c-4ba0-aa77-ffc9b1316718 cropwizard-beta \n", + "7 bd3c9174-4512-4ac6-ae05-8aafa4099215 cropwizard-beta \n", + "8 a49941c4-2049-478b-b43e-9c574cadbd05 cropwizard-beta \n", + "9 1be7956c-d5c8-4b1f-a62f-145421a2e7f7 cropwizard-beta \n", + "10 20e6e160-6fd3-4e0a-82b7-98457f2ff1c3 cropwizard-beta \n", + "11 fbd19bd0-373b-4a9c-b58c-a47cbd4e9e1c cropwizard-beta \n", + "12 8df8a436-adbf-441a-92ec-f6d926aee789 cropwizard-beta \n", + "13 b401031c-5540-43fc-baff-7c6db90ff3ae cropwizard-beta \n", + "14 c42e70a8-03c0-4c9e-ac7f-ee718b4ac968 cropwizard-beta \n", + "15 cbecc7f4-a94c-49d6-ae1a-e42f20136676 cropwizard-beta \n", + "16 3303ec4f-84d5-4eec-a4b3-af020e62b79d cropwizard-beta \n", + "17 fc91f681-54e3-4df5-8fd4-952d02b8feba cropwizard-beta \n", + "18 5c2d89b3-3e41-4a8a-84be-f93c7b8127c3 cropwizard-beta \n", + "19 d7191543-c517-4007-a2fb-ae28660ef37c cropwizard-beta \n", + "20 edabed51-e581-48f7-865c-89a4b1d9d120 cropwizard-beta \n", + "\n", + " user_email \n", + "0 dabholkar.asmita@gmail.com \n", + "1 kastanday@live.com \n", + "2 dabholkar.asmita@gmail.com \n", + "3 dabholkar.asmita@gmail.com \n", + "4 dabholkar.asmita@gmail.com \n", + "5 avd6@illinois.edu \n", + "6 hetarth2@illinois.edu \n", + "7 dabholkar.asmita@gmail.com \n", + "8 vadve@illinois.edu \n", + "9 kastanday@live.com \n", + "10 kastanday@live.com \n", + "11 dabholkar.asmita@gmail.com \n", + "12 None \n", + "13 rohan13@illinois.edu \n", + "14 hetarth2@illinois.edu \n", + "15 None \n", + "16 vadve@illinois.edu \n", + "17 rohan13@illinois.edu \n", + "18 rohan13@illinois.edu \n", + "19 None \n", + "20 dabholkar.asmita@gmail.com " + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "course = 'cropwizard-beta'\n", + "response = supabase_client.table(\"llm-convo-monitor\").select(\"*\").eq('course_name', course).execute()\n", + "data = response.data\n", + "df = pd.DataFrame(data)\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "id int64\n", + "created_at object\n", + "convo object\n", + "convo_id object\n", + "course_name object\n", + "user_email object\n", + "dtype: object\n", + "id int64\n", + "created_at datetime64[ns, UTC]\n", + "convo object\n", + "convo_id object\n", + "course_name object\n", + "user_email object\n", + "dtype: object\n" + ] + } + ], + "source": [ + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idcreated_atconvoconvo_idcourse_nameuser_email
050042023-09-01 16:28:23.202471+00:00{'id': 'edabed51-e581-48f7-865c-89a4b1d9d120',...edabed51-e581-48f7-865c-89a4b1d9d120cropwizard-betadabholkar.asmita@gmail.com
150062023-09-01 18:06:33.350981+00:00{'id': 'c42e70a8-03c0-4c9e-ac7f-ee718b4ac968',...c42e70a8-03c0-4c9e-ac7f-ee718b4ac968cropwizard-betahetarth2@illinois.edu
250072023-09-01 18:15:08.636935+00:00{'id': 'd7191543-c517-4007-a2fb-ae28660ef37c',...d7191543-c517-4007-a2fb-ae28660ef37ccropwizard-betaNone
350112023-09-01 18:28:24.588312+00:00{'id': '5c2d89b3-3e41-4a8a-84be-f93c7b8127c3',...5c2d89b3-3e41-4a8a-84be-f93c7b8127c3cropwizard-betarohan13@illinois.edu
450132023-09-01 18:35:32.121030+00:00{'id': 'fc91f681-54e3-4df5-8fd4-952d02b8feba',...fc91f681-54e3-4df5-8fd4-952d02b8febacropwizard-betarohan13@illinois.edu
\n", + "
" + ], + "text/plain": [ + " id created_at \\\n", + "0 5004 2023-09-01 16:28:23.202471+00:00 \n", + "1 5006 2023-09-01 18:06:33.350981+00:00 \n", + "2 5007 2023-09-01 18:15:08.636935+00:00 \n", + "3 5011 2023-09-01 18:28:24.588312+00:00 \n", + "4 5013 2023-09-01 18:35:32.121030+00:00 \n", + "\n", + " convo \\\n", + "0 {'id': 'edabed51-e581-48f7-865c-89a4b1d9d120',... \n", + "1 {'id': 'c42e70a8-03c0-4c9e-ac7f-ee718b4ac968',... \n", + "2 {'id': 'd7191543-c517-4007-a2fb-ae28660ef37c',... \n", + "3 {'id': '5c2d89b3-3e41-4a8a-84be-f93c7b8127c3',... \n", + "4 {'id': 'fc91f681-54e3-4df5-8fd4-952d02b8feba',... \n", + "\n", + " convo_id course_name \\\n", + "0 edabed51-e581-48f7-865c-89a4b1d9d120 cropwizard-beta \n", + "1 c42e70a8-03c0-4c9e-ac7f-ee718b4ac968 cropwizard-beta \n", + "2 d7191543-c517-4007-a2fb-ae28660ef37c cropwizard-beta \n", + "3 5c2d89b3-3e41-4a8a-84be-f93c7b8127c3 cropwizard-beta \n", + "4 fc91f681-54e3-4df5-8fd4-952d02b8feba cropwizard-beta \n", + "\n", + " user_email \n", + "0 dabholkar.asmita@gmail.com \n", + "1 hetarth2@illinois.edu \n", + "2 None \n", + "3 rohan13@illinois.edu \n", + "4 rohan13@illinois.edu " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Timestamp('2023-09-01 16:28:23.202471+0000', tz='UTC')" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.iloc[0]['created_at']" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "course_list = df['course_name'].unique()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "21\n", + "21\n" + ] + } + ], + "source": [ + "import datetime\n", + "\n", + "metadata = []\n", + "user_queries = []\n", + "i = 1\n", + "\n", + "# log conversation instead of individual messages\n", + "for index, row in df.iterrows():\n", + " user_email = row['user_email']\n", + " created_at = row['created_at']\n", + "\n", + " current_time = time.time()\n", + " dt_object = datetime.datetime.fromtimestamp(current_time)\n", + " formatted_timestamp = dt_object.strftime(\"%Y-%m-%d %H:%M:%S.%f+00:00\")\n", + "\n", + "\n", + "\n", + " convo = row['convo']\n", + " messages = convo['messages']\n", + " first_message = messages[0]['content']\n", + " \n", + " user_queries.append(first_message)\n", + " # create metadata for multi-turn conversation\n", + " conversation = \"\"\n", + " for message in messages:\n", + " # string of role: content, role: content, ...\n", + " if message['role'] == 'user':\n", + " emoji = \"🙋\"\n", + " else:\n", + " emoji = \"🤖\"\n", + " conversation += \"\\n>>> \" + emoji + message['role'] + \": \" + message['content'] + \"\\n\"\n", + " # add to metadata\n", + " metadata_row = {\"course\": row['course_name'], \"conversation\": conversation, \"conversation_id\": convo['id'], \n", + " \"id\": i, \"user_email\": user_email, \"first_query\": first_message, \"created_at\": created_at,\n", + " \"modified_at\": formatted_timestamp}\n", + " metadata.append(metadata_row)\n", + " i += 1\n", + "\n", + "print(len(user_queries))\n", + "print(len(metadata))" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'course': 'cropwizard-beta', 'conversation': \"\\n>>> 🙋user: What is the best approach to prevent weeds in my soybean field?\\n\\n>>> 🤖assistant: Weed management in soybean fields involves a combination of cultural, mechanical, and chemical strategies. Here are some recommended practices:\\n\\n1. **Crop Rotation**: Rotating crops can help reduce the presence of certain weed species that are more prevalent in soybean fields. \\n\\n2. **Cover Crops**: Planting cover crops can suppress weed growth by competing for sunlight, water, and nutrients. \\n\\n3. **Tillage**: Tillage can be used to disrupt weed growth. However, it should be used judiciously to prevent soil erosion.\\n\\n4. **Planting Date and Seeding Rate**: Planting soybeans early and at a higher seeding rate can help the crop compete better against weeds.\\n\\n5. **Herbicide Application**: Use pre-emergent and post-emergent herbicides. Pre-emergent herbicides are applied before the weeds germinate, while post-emergent herbicides are applied after the weeds have sprouted. \\n\\n6. **Herbicide Rotation**: Rotate herbicides with different modes of action to prevent weeds from developing resistance.\\n\\n7. **Scouting and Monitoring**: Regularly scout and monitor your fields to identify weed problems early and respond quickly.\\n\\n8. **Clean Equipment**: Clean your farming equipment to prevent the spread of weed seeds from one field to another.\\n\\nRemember, the most effective weed management strategies often involve a combination of these practices. It's also essential to consider the specific types of weeds you're dealing with, as different species may require different control methods.\\n\\n>>> 🙋user: What is the best approach to prevent weeds in my soybean field?\\n\\n\\n\\n\\n>>> 🤖assistant: Managing weeds in soybean fields requires a comprehensive and integrated approach. Here are some best practices:\\n\\n1. **Crop Rotation**: This practice can help break the life cycle of certain weeds. By alternating the types of crops in each field, you can naturally reduce the presence of weeds that thrive in soybean fields.\\n\\n2. **Cover Crops**: Planting cover crops in the off-season can suppress weed growth by outcompeting them for resources such as sunlight, water, and nutrients.\\n\\n3. **Tillage**: Tillage can help control weeds by physically removing or burying them. However, excessive tillage can lead to soil erosion, so it should be used judiciously.\\n\\n4. **Planting Date and Seeding Rate**: Planting soybeans early and at a higher seeding rate can help the crop establish itself and compete more effectively against weeds.\\n\\n5. **Herbicide Application**: Use pre-emergence and post-emergence herbicides to control weeds. Pre-emergence herbicides are used before the weeds germinate, while post-emergence herbicides are used after the weeds have sprouted. \\n\\n6. **Herbicide Rotation**: To prevent herbicide resistance, rotate the types of herbicides used, ensuring they have different modes of action.\\n\\n7. **Regular Scouting**: Regularly inspect your fields to identify weed problems early. This allows for timely intervention and can prevent the spread of weeds.\\n\\n8. **Clean Equipment**: Ensure that your farming equipment is cleaned regularly to prevent the spread of weed seeds from one field to another.\\n\\nRemember, the most effective weed management strategies often involve a combination of these practices. Tailor your approach based on the specific types of weeds prevalent in your field, as different species may require different control strategies.\\n\", 'conversation_id': 'd6e83182-56d8-4151-b84e-a09dd84b8836', 'id': 1, 'user_email': 'dabholkar.asmita@gmail.com', 'first_query': 'What is the best approach to prevent weeds in my soybean field?', 'created_at': '2023-09-06T23:21:36.639848+00:00', 'modified_at': '2023-09-12 18:17:49.784979+00:00'}\n" + ] + } + ], + "source": [ + "print(metadata[0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Mapping Conversations to Nomic" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cropwizard-beta\n", + "21\n", + "21\n", + "21\n", + "(21, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-09-12 22:36:00.418 | INFO | nomic.project:_create_project:790 - Creating project `Conversation Map for cropwizard-beta` in organization `dabholkar.asmita`\n", + "2023-09-12 22:36:01.434 | INFO | nomic.atlas:map_embeddings:108 - Uploading embeddings to Atlas.\n", + "2023-09-12 22:36:01.454 | WARNING | nomic.project:_validate_and_correct_arrow_upload:238 - Replacing 3 null values for field user_email with string 'null'. This behavior will change in a future version.\n", + "2023-09-12 22:36:01.459 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.11it/s]\n", + "2023-09-12 22:36:02.373 | INFO | nomic.project:_add_data:1422 - Upload succeeded.\n", + "2023-09-12 22:36:02.374 | INFO | nomic.atlas:map_embeddings:127 - Embedding upload succeeded.\n", + "2023-09-12 22:36:03.570 | INFO | nomic.project:create_index:1132 - Created map `Conversation Map for cropwizard-beta` in project `Conversation Map for cropwizard-beta`: https://atlas.nomic.ai/map/8634eb7a-22b5-4ab8-adc4-fbbc4bc654bf/f5ad20c7-d1cb-4fe7-be16-04d0502af4fe\n", + "2023-09-12 22:36:03.571 | INFO | nomic.atlas:map_embeddings:140 - Conversation Map for cropwizard-beta: https://atlas.nomic.ai/map/8634eb7a-22b5-4ab8-adc4-fbbc4bc654bf/f5ad20c7-d1cb-4fe7-be16-04d0502af4fe\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Conversation Map for cropwizard-beta: https://atlas.nomic.ai/map/8634eb7a-22b5-4ab8-adc4-fbbc4bc654bf/f5ad20c7-d1cb-4fe7-be16-04d0502af4fe]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-09-12 22:36:05.071 | INFO | nomic.project:create_index:1132 - Created map `cropwizard-beta_convo_index` in project `Conversation Map for cropwizard-beta`: https://atlas.nomic.ai/map/8634eb7a-22b5-4ab8-adc4-fbbc4bc654bf/9507bda2-b846-4e48-90d8-684a40fb3220\n" + ] + } + ], + "source": [ + "embeddings_model = OpenAIEmbeddings()\n", + "course_list = ['cropwizard-beta']\n", + "\n", + "for course in course_list:\n", + " print(course)\n", + " try:\n", + " response = supabase_client.table(\"llm-convo-monitor\").select(\"*\").eq('course_name', course).execute()\n", + " data = response.data\n", + " course_df = pd.DataFrame(data)\n", + " print(len(course_df))\n", + "\n", + " if len(course_df) < 20:\n", + " continue\n", + " else:\n", + " \n", + " user_queries = []\n", + " metadata = []\n", + " i = 1\n", + "\n", + " # log conversation instead of individual messages\n", + " for index, row in course_df.iterrows():\n", + " user_email = row['user_email']\n", + " created_at = row['created_at']\n", + " convo = row['convo']\n", + " messages = convo['messages']\n", + " first_message = messages[0]['content']\n", + "\n", + " current_time = time.time()\n", + " dt_object = datetime.datetime.fromtimestamp(current_time)\n", + " formatted_timestamp = dt_object.strftime(\"%Y-%m-%d %H:%M:%S.%f+00:00\")\n", + " \n", + " user_queries.append(first_message)\n", + " # create metadata for multi-turn conversation\n", + " conversation = \"\"\n", + " for message in messages:\n", + " # string of role: content, role: content, ...\n", + " if message['role'] == 'user':\n", + " emoji = \"🙋\"\n", + " else:\n", + " emoji = \"🤖\"\n", + " conversation += \"\\n>>> \" + emoji + message['role'] + \": \" + message['content'] + \"\\n\"\n", + " # add to metadata\n", + " metadata_row = {\"course\": row['course_name'], \"conversation\": conversation, \"conversation_id\": convo['id'], \n", + " \"id\": i, \"user_email\": user_email, \"first_query\": first_message, \"created_at\": created_at,\n", + " \"modified_at\": formatted_timestamp}\n", + " metadata.append(metadata_row)\n", + " i += 1\n", + "\n", + " print(len(user_queries))\n", + " print(len(metadata))\n", + "\n", + " metadata = pd.DataFrame(metadata)\n", + " embeddings = embeddings_model.embed_documents(user_queries)\n", + " embeddings = np.array(embeddings)\n", + " print(embeddings.shape)\n", + "\n", + " # create an Atlas project\n", + " project_name = \"Conversation Map for \" + course\n", + " index_name = course + \"_convo_index\"\n", + " project = atlas.map_embeddings(embeddings=np.array(embeddings),\n", + " data=metadata,\n", + " id_field='id',\n", + " build_topic_model=True,\n", + " topic_label_field='first_query',\n", + " name=project_name,\n", + " colorable_fields=['conversation_id', 'first_query'])\n", + " print(project.maps)\n", + "\n", + " project.create_index(index_name, build_topic_model=True)\n", + " except Exception as e:\n", + " print(\"course_name:\", course)\n", + " print(\"error: \", e)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'course': 'cropwizard', 'conversation': '\\n>>> user: What type of pest is most harmful to corn crops in Illinois and how can I prevent it?\\n\\n>>> assistant: The most harmful pest to corn crops in Illinois is the corn rootworm. Corn rootworms, specifically the western corn rootworm (Diabrotica virgifera virgifera), are the most impactful species in Illinois and the North Central U.S. These pests can cause significant damage to corn plants by feeding on the roots, leading to reduced plant vigor and yield.\\n\\nTo prevent corn rootworm infestations, several measures can be taken:\\n\\n1. Crop rotation: Implementing a crop rotation strategy by alternating corn with non-host crops, such as soybeans, can help reduce corn rootworm populations. The larvae are not capable of surviving on soybean or other broadleaf plants, so rotating crops disrupts their life cycle.\\n\\n2. Utilize Bt hybrids: Planting corn hybrids that have been genetically modified to express Bacillus thuringiensis (Bt) proteins targeting corn rootworm can provide effective control. These Bt hybrids produce toxins that are toxic to the pests, reducing their feeding and damage to corn plants.\\n\\n3. Insecticide treatments: Seed-applied insecticides containing neonicotinoid or diamide insecticides can be used to protect corn seedlings from early-season pests, including corn rootworms. Soil-applied insecticides can also be applied at planting to control corn rootworms and other early-season pests.\\n\\n4. Monitoring and scouting: Regular monitoring of corn fields for signs of corn rootworm damage, such as root feeding and lodging, is essential. Scouting helps determine if pest populations have reached economic thresholds that require action, such as insecticide application.\\n\\n5. Integrated Pest Management (IPM): Implementing an IPM approach involves combining multiple pest management strategies, including cultural practices, biological control, and chemical control, to minimize the impact of corn rootworms and other pests. This holistic approach aims to reduce reliance on chemical treatments and promote sustainable pest management.\\n\\nIt is important to consult with local agricultural extension services, seed dealers, or agricultural experts for specific recommendations tailored to your region and field conditions to effectively manage corn rootworm infestations.\\n', 'conversation_id': '3e5d4861-b128-4c64-96ac-87c74f3217e5', 'id': 2, 'user_email': 'avd6@illinois.edu', 'first_query': 'What type of pest is most harmful to corn crops in Illinois and how can I prevent it?'}\n" + ] + } + ], + "source": [ + "print(metadata[1])" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(22, 1536)\n" + ] + } + ], + "source": [ + "# initialize langchain OpenAI embeddings model\n", + "embeddings_model = OpenAIEmbeddings()\n", + "metadata = pd.DataFrame(metadata)\n", + "embeddings = embeddings_model.embed_documents(user_queries)\n", + "embeddings = np.array(embeddings)\n", + "print(embeddings.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-09-07 14:53:44.900 | INFO | nomic.project:_create_project:779 - Creating project `Conversation Map for cropwizard` in organization `dabholkar.asmita`\n", + "2023-09-07 14:53:45.794 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-09-07 14:53:45.798 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.07it/s]\n", + "2023-09-07 14:53:46.743 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-09-07 14:53:46.744 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-09-07 14:53:47.568 | INFO | nomic.project:create_index:1111 - Created map `Conversation Map for cropwizard` in project `Conversation Map for cropwizard`: https://atlas.nomic.ai/map/27ba0013-71ba-4fd6-a710-187d114442ab/f74b45b3-7213-45df-a4cb-c16ec9feb8b7\n", + "2023-09-07 14:53:47.570 | INFO | nomic.atlas:map_embeddings:139 - Conversation Map for cropwizard: https://atlas.nomic.ai/map/27ba0013-71ba-4fd6-a710-187d114442ab/f74b45b3-7213-45df-a4cb-c16ec9feb8b7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Conversation Map for cropwizard: https://atlas.nomic.ai/map/27ba0013-71ba-4fd6-a710-187d114442ab/f74b45b3-7213-45df-a4cb-c16ec9feb8b7]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-09-07 14:53:48.872 | INFO | nomic.project:create_index:1111 - Created map `cropwizard_convo_index_2` in project `Conversation Map for cropwizard`: https://atlas.nomic.ai/map/27ba0013-71ba-4fd6-a710-187d114442ab/3336aa7f-5995-4f02-831b-7161fd0c0b71\n" + ] + }, + { + "data": { + "text/html": [ + "Atlas Projection cropwizard_convo_index_2. Status Topic Modeling. view online" + ], + "text/plain": [ + "cropwizard_convo_index_2: https://atlas.nomic.ai/map/27ba0013-71ba-4fd6-a710-187d114442ab/3336aa7f-5995-4f02-831b-7161fd0c0b71" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# create an Atlas project\n", + "project_name = \"Conversation Map for \" + course\n", + "index_name = course + \"_convo_index\"\n", + "project = atlas.map_embeddings(embeddings=np.array(embeddings),\n", + " data=metadata,\n", + " id_field='id',\n", + " build_topic_model=True,\n", + " topic_label_field='first_query',\n", + " name=project_name,\n", + " colorable_fields=['conversation_id', 'first_query'])\n", + "print(project.maps)\n", + "\n", + "project.create_index(index_name, build_topic_model=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "ename": "ReadTimeout", + "evalue": "The read operation timed out", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mTimeoutError\u001b[0m Traceback (most recent call last)", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_exceptions.py:10\u001b[0m, in \u001b[0;36mmap_exceptions\u001b[1;34m(map)\u001b[0m\n\u001b[0;32m 9\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m---> 10\u001b[0m \u001b[39myield\u001b[39;00m\n\u001b[0;32m 11\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m exc: \u001b[39m# noqa: PIE786\u001b[39;00m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\backends\\sync.py:28\u001b[0m, in \u001b[0;36mSyncStream.read\u001b[1;34m(self, max_bytes, timeout)\u001b[0m\n\u001b[0;32m 27\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_sock\u001b[39m.\u001b[39msettimeout(timeout)\n\u001b[1;32m---> 28\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_sock\u001b[39m.\u001b[39;49mrecv(max_bytes)\n", + "File \u001b[1;32mC:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.10_3.10.3056.0_x64__qbz5n2kfra8p0\\lib\\ssl.py:1259\u001b[0m, in \u001b[0;36mSSLSocket.recv\u001b[1;34m(self, buflen, flags)\u001b[0m\n\u001b[0;32m 1256\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[0;32m 1257\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mnon-zero flags not allowed in calls to recv() on \u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m\"\u001b[39m \u001b[39m%\u001b[39m\n\u001b[0;32m 1258\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m)\n\u001b[1;32m-> 1259\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mread(buflen)\n\u001b[0;32m 1260\u001b[0m \u001b[39melse\u001b[39;00m:\n", + "File \u001b[1;32mC:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.10_3.10.3056.0_x64__qbz5n2kfra8p0\\lib\\ssl.py:1132\u001b[0m, in \u001b[0;36mSSLSocket.read\u001b[1;34m(self, len, buffer)\u001b[0m\n\u001b[0;32m 1131\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m-> 1132\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_sslobj\u001b[39m.\u001b[39;49mread(\u001b[39mlen\u001b[39;49m)\n\u001b[0;32m 1133\u001b[0m \u001b[39mexcept\u001b[39;00m SSLError \u001b[39mas\u001b[39;00m x:\n", + "\u001b[1;31mTimeoutError\u001b[0m: The read operation timed out", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[1;31mReadTimeout\u001b[0m Traceback (most recent call last)", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_transports\\default.py:60\u001b[0m, in \u001b[0;36mmap_httpcore_exceptions\u001b[1;34m()\u001b[0m\n\u001b[0;32m 59\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m---> 60\u001b[0m \u001b[39myield\u001b[39;00m\n\u001b[0;32m 61\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m exc: \u001b[39m# noqa: PIE-786\u001b[39;00m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_transports\\default.py:218\u001b[0m, in \u001b[0;36mHTTPTransport.handle_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 217\u001b[0m \u001b[39mwith\u001b[39;00m map_httpcore_exceptions():\n\u001b[1;32m--> 218\u001b[0m resp \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_pool\u001b[39m.\u001b[39;49mhandle_request(req)\n\u001b[0;32m 220\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39misinstance\u001b[39m(resp\u001b[39m.\u001b[39mstream, typing\u001b[39m.\u001b[39mIterable)\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_sync\\connection_pool.py:253\u001b[0m, in \u001b[0;36mConnectionPool.handle_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 252\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mresponse_closed(status)\n\u001b[1;32m--> 253\u001b[0m \u001b[39mraise\u001b[39;00m exc\n\u001b[0;32m 254\u001b[0m \u001b[39melse\u001b[39;00m:\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_sync\\connection_pool.py:237\u001b[0m, in \u001b[0;36mConnectionPool.handle_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 236\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> 237\u001b[0m response \u001b[39m=\u001b[39m connection\u001b[39m.\u001b[39;49mhandle_request(request)\n\u001b[0;32m 238\u001b[0m \u001b[39mexcept\u001b[39;00m ConnectionNotAvailable:\n\u001b[0;32m 239\u001b[0m \u001b[39m# The ConnectionNotAvailable exception is a special case, that\u001b[39;00m\n\u001b[0;32m 240\u001b[0m \u001b[39m# indicates we need to retry the request on a new connection.\u001b[39;00m\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 244\u001b[0m \u001b[39m# might end up as an HTTP/2 connection, but which actually ends\u001b[39;00m\n\u001b[0;32m 245\u001b[0m \u001b[39m# up as HTTP/1.1.\u001b[39;00m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_sync\\connection.py:90\u001b[0m, in \u001b[0;36mHTTPConnection.handle_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 88\u001b[0m \u001b[39mraise\u001b[39;00m ConnectionNotAvailable()\n\u001b[1;32m---> 90\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_connection\u001b[39m.\u001b[39;49mhandle_request(request)\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_sync\\http11.py:112\u001b[0m, in \u001b[0;36mHTTP11Connection.handle_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 111\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_response_closed()\n\u001b[1;32m--> 112\u001b[0m \u001b[39mraise\u001b[39;00m exc\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_sync\\http11.py:91\u001b[0m, in \u001b[0;36mHTTP11Connection.handle_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 83\u001b[0m \u001b[39mwith\u001b[39;00m Trace(\n\u001b[0;32m 84\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mhttp11.receive_response_headers\u001b[39m\u001b[39m\"\u001b[39m, request, kwargs\n\u001b[0;32m 85\u001b[0m ) \u001b[39mas\u001b[39;00m trace:\n\u001b[0;32m 86\u001b[0m (\n\u001b[0;32m 87\u001b[0m http_version,\n\u001b[0;32m 88\u001b[0m status,\n\u001b[0;32m 89\u001b[0m reason_phrase,\n\u001b[0;32m 90\u001b[0m headers,\n\u001b[1;32m---> 91\u001b[0m ) \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_receive_response_headers(\u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m 92\u001b[0m trace\u001b[39m.\u001b[39mreturn_value \u001b[39m=\u001b[39m (\n\u001b[0;32m 93\u001b[0m http_version,\n\u001b[0;32m 94\u001b[0m status,\n\u001b[0;32m 95\u001b[0m reason_phrase,\n\u001b[0;32m 96\u001b[0m headers,\n\u001b[0;32m 97\u001b[0m )\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_sync\\http11.py:155\u001b[0m, in \u001b[0;36mHTTP11Connection._receive_response_headers\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 154\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[1;32m--> 155\u001b[0m event \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_receive_event(timeout\u001b[39m=\u001b[39;49mtimeout)\n\u001b[0;32m 156\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(event, h11\u001b[39m.\u001b[39mResponse):\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_sync\\http11.py:191\u001b[0m, in \u001b[0;36mHTTP11Connection._receive_event\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m 190\u001b[0m \u001b[39mif\u001b[39;00m event \u001b[39mis\u001b[39;00m h11\u001b[39m.\u001b[39mNEED_DATA:\n\u001b[1;32m--> 191\u001b[0m data \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_network_stream\u001b[39m.\u001b[39;49mread(\n\u001b[0;32m 192\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mREAD_NUM_BYTES, timeout\u001b[39m=\u001b[39;49mtimeout\n\u001b[0;32m 193\u001b[0m )\n\u001b[0;32m 195\u001b[0m \u001b[39m# If we feed this case through h11 we'll raise an exception like:\u001b[39;00m\n\u001b[0;32m 196\u001b[0m \u001b[39m#\u001b[39;00m\n\u001b[0;32m 197\u001b[0m \u001b[39m# httpcore.RemoteProtocolError: can't handle event type\u001b[39;00m\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 201\u001b[0m \u001b[39m# perspective. Instead we handle this case distinctly and treat\u001b[39;00m\n\u001b[0;32m 202\u001b[0m \u001b[39m# it as a ConnectError.\u001b[39;00m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\backends\\sync.py:26\u001b[0m, in \u001b[0;36mSyncStream.read\u001b[1;34m(self, max_bytes, timeout)\u001b[0m\n\u001b[0;32m 25\u001b[0m exc_map: ExceptionMapping \u001b[39m=\u001b[39m {socket\u001b[39m.\u001b[39mtimeout: ReadTimeout, \u001b[39mOSError\u001b[39;00m: ReadError}\n\u001b[1;32m---> 26\u001b[0m \u001b[39mwith\u001b[39;00m map_exceptions(exc_map):\n\u001b[0;32m 27\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_sock\u001b[39m.\u001b[39msettimeout(timeout)\n", + "File \u001b[1;32mC:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.10_3.10.3056.0_x64__qbz5n2kfra8p0\\lib\\contextlib.py:153\u001b[0m, in \u001b[0;36m_GeneratorContextManager.__exit__\u001b[1;34m(self, typ, value, traceback)\u001b[0m\n\u001b[0;32m 152\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> 153\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mgen\u001b[39m.\u001b[39;49mthrow(typ, value, traceback)\n\u001b[0;32m 154\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mStopIteration\u001b[39;00m \u001b[39mas\u001b[39;00m exc:\n\u001b[0;32m 155\u001b[0m \u001b[39m# Suppress StopIteration *unless* it's the same exception that\u001b[39;00m\n\u001b[0;32m 156\u001b[0m \u001b[39m# was passed to throw(). This prevents a StopIteration\u001b[39;00m\n\u001b[0;32m 157\u001b[0m \u001b[39m# raised inside the \"with\" statement from being suppressed.\u001b[39;00m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_exceptions.py:14\u001b[0m, in \u001b[0;36mmap_exceptions\u001b[1;34m(map)\u001b[0m\n\u001b[0;32m 13\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(exc, from_exc):\n\u001b[1;32m---> 14\u001b[0m \u001b[39mraise\u001b[39;00m to_exc(exc)\n\u001b[0;32m 15\u001b[0m \u001b[39mraise\u001b[39;00m\n", + "\u001b[1;31mReadTimeout\u001b[0m: The read operation timed out", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[1;31mReadTimeout\u001b[0m Traceback (most recent call last)", + "\u001b[1;32mf:\\MSIM\\ML_Projects\\ai-ta-backend\\ai_ta_backend\\nomic_map_creation.ipynb Cell 13\u001b[0m line \u001b[0;36m1\n\u001b[1;32m----> 1\u001b[0m response \u001b[39m=\u001b[39m supabase_client\u001b[39m.\u001b[39;49mtable(\u001b[39m\"\u001b[39;49m\u001b[39mllm-convo-monitor\u001b[39;49m\u001b[39m\"\u001b[39;49m)\u001b[39m.\u001b[39;49mselect(\u001b[39m\"\u001b[39;49m\u001b[39m*\u001b[39;49m\u001b[39m\"\u001b[39;49m)\u001b[39m.\u001b[39;49mexecute()\n\u001b[0;32m 2\u001b[0m data \u001b[39m=\u001b[39m response\u001b[39m.\u001b[39mdata\n\u001b[0;32m 3\u001b[0m df \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39mDataFrame(data)\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\postgrest\\_sync\\request_builder.py:55\u001b[0m, in \u001b[0;36mSyncQueryRequestBuilder.execute\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 43\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mexecute\u001b[39m(\u001b[39mself\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m APIResponse:\n\u001b[0;32m 44\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"Execute the query.\u001b[39;00m\n\u001b[0;32m 45\u001b[0m \n\u001b[0;32m 46\u001b[0m \u001b[39m .. tip::\u001b[39;00m\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 53\u001b[0m \u001b[39m :class:`APIError` If the API raised an error.\u001b[39;00m\n\u001b[0;32m 54\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[1;32m---> 55\u001b[0m r \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msession\u001b[39m.\u001b[39;49mrequest(\n\u001b[0;32m 56\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mhttp_method,\n\u001b[0;32m 57\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mpath,\n\u001b[0;32m 58\u001b[0m json\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mjson,\n\u001b[0;32m 59\u001b[0m params\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mparams,\n\u001b[0;32m 60\u001b[0m headers\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mheaders,\n\u001b[0;32m 61\u001b[0m )\n\u001b[0;32m 62\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 63\u001b[0m \u001b[39mif\u001b[39;00m (\n\u001b[0;32m 64\u001b[0m \u001b[39m200\u001b[39m \u001b[39m<\u001b[39m\u001b[39m=\u001b[39m r\u001b[39m.\u001b[39mstatus_code \u001b[39m<\u001b[39m\u001b[39m=\u001b[39m \u001b[39m299\u001b[39m\n\u001b[0;32m 65\u001b[0m ): \u001b[39m# Response.ok from JS (https://developer.mozilla.org/en-US/docs/Web/API/Response/ok)\u001b[39;00m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_client.py:821\u001b[0m, in \u001b[0;36mClient.request\u001b[1;34m(self, method, url, content, data, files, json, params, headers, cookies, auth, follow_redirects, timeout, extensions)\u001b[0m\n\u001b[0;32m 806\u001b[0m warnings\u001b[39m.\u001b[39mwarn(message, \u001b[39mDeprecationWarning\u001b[39;00m)\n\u001b[0;32m 808\u001b[0m request \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbuild_request(\n\u001b[0;32m 809\u001b[0m method\u001b[39m=\u001b[39mmethod,\n\u001b[0;32m 810\u001b[0m url\u001b[39m=\u001b[39murl,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 819\u001b[0m extensions\u001b[39m=\u001b[39mextensions,\n\u001b[0;32m 820\u001b[0m )\n\u001b[1;32m--> 821\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msend(request, auth\u001b[39m=\u001b[39;49mauth, follow_redirects\u001b[39m=\u001b[39;49mfollow_redirects)\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_client.py:908\u001b[0m, in \u001b[0;36mClient.send\u001b[1;34m(self, request, stream, auth, follow_redirects)\u001b[0m\n\u001b[0;32m 900\u001b[0m follow_redirects \u001b[39m=\u001b[39m (\n\u001b[0;32m 901\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfollow_redirects\n\u001b[0;32m 902\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(follow_redirects, UseClientDefault)\n\u001b[0;32m 903\u001b[0m \u001b[39melse\u001b[39;00m follow_redirects\n\u001b[0;32m 904\u001b[0m )\n\u001b[0;32m 906\u001b[0m auth \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_build_request_auth(request, auth)\n\u001b[1;32m--> 908\u001b[0m response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_send_handling_auth(\n\u001b[0;32m 909\u001b[0m request,\n\u001b[0;32m 910\u001b[0m auth\u001b[39m=\u001b[39;49mauth,\n\u001b[0;32m 911\u001b[0m follow_redirects\u001b[39m=\u001b[39;49mfollow_redirects,\n\u001b[0;32m 912\u001b[0m history\u001b[39m=\u001b[39;49m[],\n\u001b[0;32m 913\u001b[0m )\n\u001b[0;32m 914\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 915\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m stream:\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_client.py:936\u001b[0m, in \u001b[0;36mClient._send_handling_auth\u001b[1;34m(self, request, auth, follow_redirects, history)\u001b[0m\n\u001b[0;32m 933\u001b[0m request \u001b[39m=\u001b[39m \u001b[39mnext\u001b[39m(auth_flow)\n\u001b[0;32m 935\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[1;32m--> 936\u001b[0m response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_send_handling_redirects(\n\u001b[0;32m 937\u001b[0m request,\n\u001b[0;32m 938\u001b[0m follow_redirects\u001b[39m=\u001b[39;49mfollow_redirects,\n\u001b[0;32m 939\u001b[0m history\u001b[39m=\u001b[39;49mhistory,\n\u001b[0;32m 940\u001b[0m )\n\u001b[0;32m 941\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 942\u001b[0m \u001b[39mtry\u001b[39;00m:\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_client.py:973\u001b[0m, in \u001b[0;36mClient._send_handling_redirects\u001b[1;34m(self, request, follow_redirects, history)\u001b[0m\n\u001b[0;32m 970\u001b[0m \u001b[39mfor\u001b[39;00m hook \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_event_hooks[\u001b[39m\"\u001b[39m\u001b[39mrequest\u001b[39m\u001b[39m\"\u001b[39m]:\n\u001b[0;32m 971\u001b[0m hook(request)\n\u001b[1;32m--> 973\u001b[0m response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_send_single_request(request)\n\u001b[0;32m 974\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 975\u001b[0m \u001b[39mfor\u001b[39;00m hook \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_event_hooks[\u001b[39m\"\u001b[39m\u001b[39mresponse\u001b[39m\u001b[39m\"\u001b[39m]:\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_client.py:1009\u001b[0m, in \u001b[0;36mClient._send_single_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 1004\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\n\u001b[0;32m 1005\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mAttempted to send an async request with a sync Client instance.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m 1006\u001b[0m )\n\u001b[0;32m 1008\u001b[0m \u001b[39mwith\u001b[39;00m request_context(request\u001b[39m=\u001b[39mrequest):\n\u001b[1;32m-> 1009\u001b[0m response \u001b[39m=\u001b[39m transport\u001b[39m.\u001b[39;49mhandle_request(request)\n\u001b[0;32m 1011\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39misinstance\u001b[39m(response\u001b[39m.\u001b[39mstream, SyncByteStream)\n\u001b[0;32m 1013\u001b[0m response\u001b[39m.\u001b[39mrequest \u001b[39m=\u001b[39m request\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_transports\\default.py:217\u001b[0m, in \u001b[0;36mHTTPTransport.handle_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 203\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39misinstance\u001b[39m(request\u001b[39m.\u001b[39mstream, SyncByteStream)\n\u001b[0;32m 205\u001b[0m req \u001b[39m=\u001b[39m httpcore\u001b[39m.\u001b[39mRequest(\n\u001b[0;32m 206\u001b[0m method\u001b[39m=\u001b[39mrequest\u001b[39m.\u001b[39mmethod,\n\u001b[0;32m 207\u001b[0m url\u001b[39m=\u001b[39mhttpcore\u001b[39m.\u001b[39mURL(\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 215\u001b[0m extensions\u001b[39m=\u001b[39mrequest\u001b[39m.\u001b[39mextensions,\n\u001b[0;32m 216\u001b[0m )\n\u001b[1;32m--> 217\u001b[0m \u001b[39mwith\u001b[39;00m map_httpcore_exceptions():\n\u001b[0;32m 218\u001b[0m resp \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_pool\u001b[39m.\u001b[39mhandle_request(req)\n\u001b[0;32m 220\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39misinstance\u001b[39m(resp\u001b[39m.\u001b[39mstream, typing\u001b[39m.\u001b[39mIterable)\n", + "File \u001b[1;32mC:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.10_3.10.3056.0_x64__qbz5n2kfra8p0\\lib\\contextlib.py:153\u001b[0m, in \u001b[0;36m_GeneratorContextManager.__exit__\u001b[1;34m(self, typ, value, traceback)\u001b[0m\n\u001b[0;32m 151\u001b[0m value \u001b[39m=\u001b[39m typ()\n\u001b[0;32m 152\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> 153\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mgen\u001b[39m.\u001b[39;49mthrow(typ, value, traceback)\n\u001b[0;32m 154\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mStopIteration\u001b[39;00m \u001b[39mas\u001b[39;00m exc:\n\u001b[0;32m 155\u001b[0m \u001b[39m# Suppress StopIteration *unless* it's the same exception that\u001b[39;00m\n\u001b[0;32m 156\u001b[0m \u001b[39m# was passed to throw(). This prevents a StopIteration\u001b[39;00m\n\u001b[0;32m 157\u001b[0m \u001b[39m# raised inside the \"with\" statement from being suppressed.\u001b[39;00m\n\u001b[0;32m 158\u001b[0m \u001b[39mreturn\u001b[39;00m exc \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m value\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_transports\\default.py:77\u001b[0m, in \u001b[0;36mmap_httpcore_exceptions\u001b[1;34m()\u001b[0m\n\u001b[0;32m 74\u001b[0m \u001b[39mraise\u001b[39;00m\n\u001b[0;32m 76\u001b[0m message \u001b[39m=\u001b[39m \u001b[39mstr\u001b[39m(exc)\n\u001b[1;32m---> 77\u001b[0m \u001b[39mraise\u001b[39;00m mapped_exc(message) \u001b[39mfrom\u001b[39;00m \u001b[39mexc\u001b[39;00m\n", + "\u001b[1;31mReadTimeout\u001b[0m: The read operation timed out" + ] + } + ], + "source": [ + "response = supabase_client.table(\"llm-convo-monitor\").select(\"*\").execute()\n", + "data = response.data\n", + "df = pd.DataFrame(data)\n", + "len(df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Mapping Query-Response Pairs to Nomic" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2\n", + "user queries: 1\n", + "metadata 1\n", + "------------------------\n" + ] + } + ], + "source": [ + "user_queries = []\n", + "metadata = []\n", + "i = 1\n", + "for convo in course_df:\n", + " messages = convo['messages']\n", + " print(len(messages))\n", + " pair_check = 0\n", + " for message in messages:\n", + " if message['role'] == 'user' and pair_check == 0:\n", + " query = message['content']\n", + " metadata_row = {'course': course, 'query': message['content']}\n", + " #print(\"metadata row: \", metadata_row)\n", + " pair_check += 1\n", + " if message['role'] == 'assistant' and pair_check == 1:\n", + " metadata_row['response'] = message['content']\n", + " metadata_row['id'] = i \n", + " #print(\"response metadata row: \", metadata_row)\n", + " i += 1\n", + " pair_check += 1\n", + " if pair_check == 2:\n", + " # one conversation pair is complete\n", + " user_queries.append(query)\n", + " metadata.append(metadata_row)\n", + " pair_check = 0\n", + "\n", + " \n", + "print(\"user queries: \", len(user_queries))\n", + "print(\"metadata\", len(metadata))\n", + "print(\"------------------------\")\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(590, 1536)\n" + ] + } + ], + "source": [ + "# initialize langchain OpenAI embeddings model\n", + "embeddings_model = OpenAIEmbeddings()\n", + "metadata = pd.DataFrame(metadata)\n", + "embeddings = embeddings_model.embed_documents(user_queries)\n", + "embeddings = np.array(embeddings)\n", + "print(embeddings.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-31 19:55:40.276 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for ECE408FA23` in organization `dabholkar.asmita`\n", + "2023-08-31 19:55:41.466 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-31 19:55:41.491 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:02, 2.43s/it]\n", + "2023-08-31 19:55:43.932 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-31 19:55:43.932 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-31 19:55:45.475 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for ECE408FA23` in project `Query-Response Map for ECE408FA23`: https://atlas.nomic.ai/map/d8e02f68-501e-4a25-9060-502710c082ce/b55cdca3-3ff4-454e-8f1b-a73ebc22cdff\n", + "2023-08-31 19:55:45.480 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for ECE408FA23: https://atlas.nomic.ai/map/d8e02f68-501e-4a25-9060-502710c082ce/b55cdca3-3ff4-454e-8f1b-a73ebc22cdff\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for ECE408FA23: https://atlas.nomic.ai/map/d8e02f68-501e-4a25-9060-502710c082ce/b55cdca3-3ff4-454e-8f1b-a73ebc22cdff]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-31 19:55:47.813 | INFO | nomic.project:create_index:1111 - Created map `ECE408FA23_qr_index` in project `Query-Response Map for ECE408FA23`: https://atlas.nomic.ai/map/d8e02f68-501e-4a25-9060-502710c082ce/ff7276fc-942f-45cd-a199-e19a6e941db1\n" + ] + }, + { + "data": { + "text/html": [ + "Atlas Projection ECE408FA23_qr_index. Status Topic Modeling. view online" + ], + "text/plain": [ + "ECE408FA23_qr_index: https://atlas.nomic.ai/map/d8e02f68-501e-4a25-9060-502710c082ce/ff7276fc-942f-45cd-a199-e19a6e941db1" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# create an Atlas project\n", + "project_name = \"Query-Response Map for \" + course\n", + "index_name = course + \"_qr_index\"\n", + "project = atlas.map_embeddings(embeddings=np.array(embeddings),\n", + " data=metadata,\n", + " id_field='id',\n", + " build_topic_model=True,\n", + " topic_label_field='query',\n", + " name=project_name,\n", + " colorable_fields=['query'])\n", + "print(project.maps)\n", + "\n", + "project.create_index(index_name, build_topic_model=True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "ename": "ReadTimeout", + "evalue": "The read operation timed out", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mTimeoutError\u001b[0m Traceback (most recent call last)", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_exceptions.py:10\u001b[0m, in \u001b[0;36mmap_exceptions\u001b[1;34m(map)\u001b[0m\n\u001b[0;32m 9\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m---> 10\u001b[0m \u001b[39myield\u001b[39;00m\n\u001b[0;32m 11\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m exc: \u001b[39m# noqa: PIE786\u001b[39;00m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\backends\\sync.py:28\u001b[0m, in \u001b[0;36mSyncStream.read\u001b[1;34m(self, max_bytes, timeout)\u001b[0m\n\u001b[0;32m 27\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_sock\u001b[39m.\u001b[39msettimeout(timeout)\n\u001b[1;32m---> 28\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_sock\u001b[39m.\u001b[39;49mrecv(max_bytes)\n", + "File \u001b[1;32mC:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.10_3.10.3056.0_x64__qbz5n2kfra8p0\\lib\\ssl.py:1259\u001b[0m, in \u001b[0;36mSSLSocket.recv\u001b[1;34m(self, buflen, flags)\u001b[0m\n\u001b[0;32m 1256\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[0;32m 1257\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mnon-zero flags not allowed in calls to recv() on \u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m\"\u001b[39m \u001b[39m%\u001b[39m\n\u001b[0;32m 1258\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m)\n\u001b[1;32m-> 1259\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mread(buflen)\n\u001b[0;32m 1260\u001b[0m \u001b[39melse\u001b[39;00m:\n", + "File \u001b[1;32mC:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.10_3.10.3056.0_x64__qbz5n2kfra8p0\\lib\\ssl.py:1132\u001b[0m, in \u001b[0;36mSSLSocket.read\u001b[1;34m(self, len, buffer)\u001b[0m\n\u001b[0;32m 1131\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m-> 1132\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_sslobj\u001b[39m.\u001b[39;49mread(\u001b[39mlen\u001b[39;49m)\n\u001b[0;32m 1133\u001b[0m \u001b[39mexcept\u001b[39;00m SSLError \u001b[39mas\u001b[39;00m x:\n", + "\u001b[1;31mTimeoutError\u001b[0m: The read operation timed out", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[1;31mReadTimeout\u001b[0m Traceback (most recent call last)", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_transports\\default.py:60\u001b[0m, in \u001b[0;36mmap_httpcore_exceptions\u001b[1;34m()\u001b[0m\n\u001b[0;32m 59\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m---> 60\u001b[0m \u001b[39myield\u001b[39;00m\n\u001b[0;32m 61\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m exc: \u001b[39m# noqa: PIE-786\u001b[39;00m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_transports\\default.py:218\u001b[0m, in \u001b[0;36mHTTPTransport.handle_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 217\u001b[0m \u001b[39mwith\u001b[39;00m map_httpcore_exceptions():\n\u001b[1;32m--> 218\u001b[0m resp \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_pool\u001b[39m.\u001b[39;49mhandle_request(req)\n\u001b[0;32m 220\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39misinstance\u001b[39m(resp\u001b[39m.\u001b[39mstream, typing\u001b[39m.\u001b[39mIterable)\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_sync\\connection_pool.py:253\u001b[0m, in \u001b[0;36mConnectionPool.handle_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 252\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mresponse_closed(status)\n\u001b[1;32m--> 253\u001b[0m \u001b[39mraise\u001b[39;00m exc\n\u001b[0;32m 254\u001b[0m \u001b[39melse\u001b[39;00m:\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_sync\\connection_pool.py:237\u001b[0m, in \u001b[0;36mConnectionPool.handle_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 236\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> 237\u001b[0m response \u001b[39m=\u001b[39m connection\u001b[39m.\u001b[39;49mhandle_request(request)\n\u001b[0;32m 238\u001b[0m \u001b[39mexcept\u001b[39;00m ConnectionNotAvailable:\n\u001b[0;32m 239\u001b[0m \u001b[39m# The ConnectionNotAvailable exception is a special case, that\u001b[39;00m\n\u001b[0;32m 240\u001b[0m \u001b[39m# indicates we need to retry the request on a new connection.\u001b[39;00m\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 244\u001b[0m \u001b[39m# might end up as an HTTP/2 connection, but which actually ends\u001b[39;00m\n\u001b[0;32m 245\u001b[0m \u001b[39m# up as HTTP/1.1.\u001b[39;00m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_sync\\connection.py:90\u001b[0m, in \u001b[0;36mHTTPConnection.handle_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 88\u001b[0m \u001b[39mraise\u001b[39;00m ConnectionNotAvailable()\n\u001b[1;32m---> 90\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_connection\u001b[39m.\u001b[39;49mhandle_request(request)\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_sync\\http11.py:112\u001b[0m, in \u001b[0;36mHTTP11Connection.handle_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 111\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_response_closed()\n\u001b[1;32m--> 112\u001b[0m \u001b[39mraise\u001b[39;00m exc\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_sync\\http11.py:91\u001b[0m, in \u001b[0;36mHTTP11Connection.handle_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 83\u001b[0m \u001b[39mwith\u001b[39;00m Trace(\n\u001b[0;32m 84\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mhttp11.receive_response_headers\u001b[39m\u001b[39m\"\u001b[39m, request, kwargs\n\u001b[0;32m 85\u001b[0m ) \u001b[39mas\u001b[39;00m trace:\n\u001b[0;32m 86\u001b[0m (\n\u001b[0;32m 87\u001b[0m http_version,\n\u001b[0;32m 88\u001b[0m status,\n\u001b[0;32m 89\u001b[0m reason_phrase,\n\u001b[0;32m 90\u001b[0m headers,\n\u001b[1;32m---> 91\u001b[0m ) \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_receive_response_headers(\u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m 92\u001b[0m trace\u001b[39m.\u001b[39mreturn_value \u001b[39m=\u001b[39m (\n\u001b[0;32m 93\u001b[0m http_version,\n\u001b[0;32m 94\u001b[0m status,\n\u001b[0;32m 95\u001b[0m reason_phrase,\n\u001b[0;32m 96\u001b[0m headers,\n\u001b[0;32m 97\u001b[0m )\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_sync\\http11.py:155\u001b[0m, in \u001b[0;36mHTTP11Connection._receive_response_headers\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 154\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[1;32m--> 155\u001b[0m event \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_receive_event(timeout\u001b[39m=\u001b[39;49mtimeout)\n\u001b[0;32m 156\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(event, h11\u001b[39m.\u001b[39mResponse):\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_sync\\http11.py:191\u001b[0m, in \u001b[0;36mHTTP11Connection._receive_event\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m 190\u001b[0m \u001b[39mif\u001b[39;00m event \u001b[39mis\u001b[39;00m h11\u001b[39m.\u001b[39mNEED_DATA:\n\u001b[1;32m--> 191\u001b[0m data \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_network_stream\u001b[39m.\u001b[39;49mread(\n\u001b[0;32m 192\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mREAD_NUM_BYTES, timeout\u001b[39m=\u001b[39;49mtimeout\n\u001b[0;32m 193\u001b[0m )\n\u001b[0;32m 195\u001b[0m \u001b[39m# If we feed this case through h11 we'll raise an exception like:\u001b[39;00m\n\u001b[0;32m 196\u001b[0m \u001b[39m#\u001b[39;00m\n\u001b[0;32m 197\u001b[0m \u001b[39m# httpcore.RemoteProtocolError: can't handle event type\u001b[39;00m\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 201\u001b[0m \u001b[39m# perspective. Instead we handle this case distinctly and treat\u001b[39;00m\n\u001b[0;32m 202\u001b[0m \u001b[39m# it as a ConnectError.\u001b[39;00m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\backends\\sync.py:26\u001b[0m, in \u001b[0;36mSyncStream.read\u001b[1;34m(self, max_bytes, timeout)\u001b[0m\n\u001b[0;32m 25\u001b[0m exc_map: ExceptionMapping \u001b[39m=\u001b[39m {socket\u001b[39m.\u001b[39mtimeout: ReadTimeout, \u001b[39mOSError\u001b[39;00m: ReadError}\n\u001b[1;32m---> 26\u001b[0m \u001b[39mwith\u001b[39;00m map_exceptions(exc_map):\n\u001b[0;32m 27\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_sock\u001b[39m.\u001b[39msettimeout(timeout)\n", + "File \u001b[1;32mC:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.10_3.10.3056.0_x64__qbz5n2kfra8p0\\lib\\contextlib.py:153\u001b[0m, in \u001b[0;36m_GeneratorContextManager.__exit__\u001b[1;34m(self, typ, value, traceback)\u001b[0m\n\u001b[0;32m 152\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> 153\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mgen\u001b[39m.\u001b[39;49mthrow(typ, value, traceback)\n\u001b[0;32m 154\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mStopIteration\u001b[39;00m \u001b[39mas\u001b[39;00m exc:\n\u001b[0;32m 155\u001b[0m \u001b[39m# Suppress StopIteration *unless* it's the same exception that\u001b[39;00m\n\u001b[0;32m 156\u001b[0m \u001b[39m# was passed to throw(). This prevents a StopIteration\u001b[39;00m\n\u001b[0;32m 157\u001b[0m \u001b[39m# raised inside the \"with\" statement from being suppressed.\u001b[39;00m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpcore\\_exceptions.py:14\u001b[0m, in \u001b[0;36mmap_exceptions\u001b[1;34m(map)\u001b[0m\n\u001b[0;32m 13\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(exc, from_exc):\n\u001b[1;32m---> 14\u001b[0m \u001b[39mraise\u001b[39;00m to_exc(exc)\n\u001b[0;32m 15\u001b[0m \u001b[39mraise\u001b[39;00m\n", + "\u001b[1;31mReadTimeout\u001b[0m: The read operation timed out", + "\nThe above exception was the direct cause of the following exception:\n", + "\u001b[1;31mReadTimeout\u001b[0m Traceback (most recent call last)", + "\u001b[1;32mf:\\MSIM\\ML_Projects\\ai-ta-backend\\ai_ta_backend\\nomic_map_creation.ipynb Cell 19\u001b[0m line \u001b[0;36m3\n\u001b[0;32m 1\u001b[0m \u001b[39m# cell for all course map creation\u001b[39;00m\n\u001b[1;32m----> 3\u001b[0m response \u001b[39m=\u001b[39m supabase_client\u001b[39m.\u001b[39;49mtable(\u001b[39m\"\u001b[39;49m\u001b[39mllm-convo-monitor\u001b[39;49m\u001b[39m\"\u001b[39;49m)\u001b[39m.\u001b[39;49mselect(\u001b[39m\"\u001b[39;49m\u001b[39m*\u001b[39;49m\u001b[39m\"\u001b[39;49m)\u001b[39m.\u001b[39;49mexecute()\n\u001b[0;32m 4\u001b[0m data \u001b[39m=\u001b[39m response\u001b[39m.\u001b[39mdata\n\u001b[0;32m 5\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mlen\u001b[39m(data))\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\postgrest\\_sync\\request_builder.py:55\u001b[0m, in \u001b[0;36mSyncQueryRequestBuilder.execute\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 43\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mexecute\u001b[39m(\u001b[39mself\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m APIResponse:\n\u001b[0;32m 44\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"Execute the query.\u001b[39;00m\n\u001b[0;32m 45\u001b[0m \n\u001b[0;32m 46\u001b[0m \u001b[39m .. tip::\u001b[39;00m\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 53\u001b[0m \u001b[39m :class:`APIError` If the API raised an error.\u001b[39;00m\n\u001b[0;32m 54\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[1;32m---> 55\u001b[0m r \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msession\u001b[39m.\u001b[39;49mrequest(\n\u001b[0;32m 56\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mhttp_method,\n\u001b[0;32m 57\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mpath,\n\u001b[0;32m 58\u001b[0m json\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mjson,\n\u001b[0;32m 59\u001b[0m params\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mparams,\n\u001b[0;32m 60\u001b[0m headers\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mheaders,\n\u001b[0;32m 61\u001b[0m )\n\u001b[0;32m 62\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 63\u001b[0m \u001b[39mif\u001b[39;00m (\n\u001b[0;32m 64\u001b[0m \u001b[39m200\u001b[39m \u001b[39m<\u001b[39m\u001b[39m=\u001b[39m r\u001b[39m.\u001b[39mstatus_code \u001b[39m<\u001b[39m\u001b[39m=\u001b[39m \u001b[39m299\u001b[39m\n\u001b[0;32m 65\u001b[0m ): \u001b[39m# Response.ok from JS (https://developer.mozilla.org/en-US/docs/Web/API/Response/ok)\u001b[39;00m\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_client.py:821\u001b[0m, in \u001b[0;36mClient.request\u001b[1;34m(self, method, url, content, data, files, json, params, headers, cookies, auth, follow_redirects, timeout, extensions)\u001b[0m\n\u001b[0;32m 806\u001b[0m warnings\u001b[39m.\u001b[39mwarn(message, \u001b[39mDeprecationWarning\u001b[39;00m)\n\u001b[0;32m 808\u001b[0m request \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbuild_request(\n\u001b[0;32m 809\u001b[0m method\u001b[39m=\u001b[39mmethod,\n\u001b[0;32m 810\u001b[0m url\u001b[39m=\u001b[39murl,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 819\u001b[0m extensions\u001b[39m=\u001b[39mextensions,\n\u001b[0;32m 820\u001b[0m )\n\u001b[1;32m--> 821\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msend(request, auth\u001b[39m=\u001b[39;49mauth, follow_redirects\u001b[39m=\u001b[39;49mfollow_redirects)\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_client.py:908\u001b[0m, in \u001b[0;36mClient.send\u001b[1;34m(self, request, stream, auth, follow_redirects)\u001b[0m\n\u001b[0;32m 900\u001b[0m follow_redirects \u001b[39m=\u001b[39m (\n\u001b[0;32m 901\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mfollow_redirects\n\u001b[0;32m 902\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(follow_redirects, UseClientDefault)\n\u001b[0;32m 903\u001b[0m \u001b[39melse\u001b[39;00m follow_redirects\n\u001b[0;32m 904\u001b[0m )\n\u001b[0;32m 906\u001b[0m auth \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_build_request_auth(request, auth)\n\u001b[1;32m--> 908\u001b[0m response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_send_handling_auth(\n\u001b[0;32m 909\u001b[0m request,\n\u001b[0;32m 910\u001b[0m auth\u001b[39m=\u001b[39;49mauth,\n\u001b[0;32m 911\u001b[0m follow_redirects\u001b[39m=\u001b[39;49mfollow_redirects,\n\u001b[0;32m 912\u001b[0m history\u001b[39m=\u001b[39;49m[],\n\u001b[0;32m 913\u001b[0m )\n\u001b[0;32m 914\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 915\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m stream:\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_client.py:936\u001b[0m, in \u001b[0;36mClient._send_handling_auth\u001b[1;34m(self, request, auth, follow_redirects, history)\u001b[0m\n\u001b[0;32m 933\u001b[0m request \u001b[39m=\u001b[39m \u001b[39mnext\u001b[39m(auth_flow)\n\u001b[0;32m 935\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[1;32m--> 936\u001b[0m response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_send_handling_redirects(\n\u001b[0;32m 937\u001b[0m request,\n\u001b[0;32m 938\u001b[0m follow_redirects\u001b[39m=\u001b[39;49mfollow_redirects,\n\u001b[0;32m 939\u001b[0m history\u001b[39m=\u001b[39;49mhistory,\n\u001b[0;32m 940\u001b[0m )\n\u001b[0;32m 941\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 942\u001b[0m \u001b[39mtry\u001b[39;00m:\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_client.py:973\u001b[0m, in \u001b[0;36mClient._send_handling_redirects\u001b[1;34m(self, request, follow_redirects, history)\u001b[0m\n\u001b[0;32m 970\u001b[0m \u001b[39mfor\u001b[39;00m hook \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_event_hooks[\u001b[39m\"\u001b[39m\u001b[39mrequest\u001b[39m\u001b[39m\"\u001b[39m]:\n\u001b[0;32m 971\u001b[0m hook(request)\n\u001b[1;32m--> 973\u001b[0m response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_send_single_request(request)\n\u001b[0;32m 974\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 975\u001b[0m \u001b[39mfor\u001b[39;00m hook \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_event_hooks[\u001b[39m\"\u001b[39m\u001b[39mresponse\u001b[39m\u001b[39m\"\u001b[39m]:\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_client.py:1009\u001b[0m, in \u001b[0;36mClient._send_single_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 1004\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\n\u001b[0;32m 1005\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mAttempted to send an async request with a sync Client instance.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m 1006\u001b[0m )\n\u001b[0;32m 1008\u001b[0m \u001b[39mwith\u001b[39;00m request_context(request\u001b[39m=\u001b[39mrequest):\n\u001b[1;32m-> 1009\u001b[0m response \u001b[39m=\u001b[39m transport\u001b[39m.\u001b[39;49mhandle_request(request)\n\u001b[0;32m 1011\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39misinstance\u001b[39m(response\u001b[39m.\u001b[39mstream, SyncByteStream)\n\u001b[0;32m 1013\u001b[0m response\u001b[39m.\u001b[39mrequest \u001b[39m=\u001b[39m request\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_transports\\default.py:217\u001b[0m, in \u001b[0;36mHTTPTransport.handle_request\u001b[1;34m(self, request)\u001b[0m\n\u001b[0;32m 203\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39misinstance\u001b[39m(request\u001b[39m.\u001b[39mstream, SyncByteStream)\n\u001b[0;32m 205\u001b[0m req \u001b[39m=\u001b[39m httpcore\u001b[39m.\u001b[39mRequest(\n\u001b[0;32m 206\u001b[0m method\u001b[39m=\u001b[39mrequest\u001b[39m.\u001b[39mmethod,\n\u001b[0;32m 207\u001b[0m url\u001b[39m=\u001b[39mhttpcore\u001b[39m.\u001b[39mURL(\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 215\u001b[0m extensions\u001b[39m=\u001b[39mrequest\u001b[39m.\u001b[39mextensions,\n\u001b[0;32m 216\u001b[0m )\n\u001b[1;32m--> 217\u001b[0m \u001b[39mwith\u001b[39;00m map_httpcore_exceptions():\n\u001b[0;32m 218\u001b[0m resp \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_pool\u001b[39m.\u001b[39mhandle_request(req)\n\u001b[0;32m 220\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39misinstance\u001b[39m(resp\u001b[39m.\u001b[39mstream, typing\u001b[39m.\u001b[39mIterable)\n", + "File \u001b[1;32mC:\\Program Files\\WindowsApps\\PythonSoftwareFoundation.Python.3.10_3.10.3056.0_x64__qbz5n2kfra8p0\\lib\\contextlib.py:153\u001b[0m, in \u001b[0;36m_GeneratorContextManager.__exit__\u001b[1;34m(self, typ, value, traceback)\u001b[0m\n\u001b[0;32m 151\u001b[0m value \u001b[39m=\u001b[39m typ()\n\u001b[0;32m 152\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> 153\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mgen\u001b[39m.\u001b[39;49mthrow(typ, value, traceback)\n\u001b[0;32m 154\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mStopIteration\u001b[39;00m \u001b[39mas\u001b[39;00m exc:\n\u001b[0;32m 155\u001b[0m \u001b[39m# Suppress StopIteration *unless* it's the same exception that\u001b[39;00m\n\u001b[0;32m 156\u001b[0m \u001b[39m# was passed to throw(). This prevents a StopIteration\u001b[39;00m\n\u001b[0;32m 157\u001b[0m \u001b[39m# raised inside the \"with\" statement from being suppressed.\u001b[39;00m\n\u001b[0;32m 158\u001b[0m \u001b[39mreturn\u001b[39;00m exc \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m value\n", + "File \u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\httpx\\_transports\\default.py:77\u001b[0m, in \u001b[0;36mmap_httpcore_exceptions\u001b[1;34m()\u001b[0m\n\u001b[0;32m 74\u001b[0m \u001b[39mraise\u001b[39;00m\n\u001b[0;32m 76\u001b[0m message \u001b[39m=\u001b[39m \u001b[39mstr\u001b[39m(exc)\n\u001b[1;32m---> 77\u001b[0m \u001b[39mraise\u001b[39;00m mapped_exc(message) \u001b[39mfrom\u001b[39;00m \u001b[39mexc\u001b[39;00m\n", + "\u001b[1;31mReadTimeout\u001b[0m: The read operation timed out" + ] + } + ], + "source": [ + "# cell for all course map creation\n", + "\n", + "response = supabase_client.table(\"llm-convo-monitor\").select(\"*\").execute()\n", + "data = response.data\n", + "print(len(data))" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "126" + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = pd.DataFrame(data)\n", + "course_names = df['course_name'].unique()\n", + "len(course_names)" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: badm_550_ashley\n", + "(51, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:26:29.701 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for badm_550_ashley` in organization `dabholkar.asmita`\n", + "2023-08-30 15:26:31.242 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:26:31.255 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.03it/s]\n", + "2023-08-30 15:26:32.239 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:26:32.241 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:26:33.498 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for badm_550_ashley` in project `Query-Response Map for badm_550_ashley`: https://atlas.nomic.ai/map/e7fe1faf-be7a-4ca7-b89b-22863fffe432/79d4480a-c4e5-4920-95cd-ed831d45f5d0\n", + "2023-08-30 15:26:33.500 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for badm_550_ashley: https://atlas.nomic.ai/map/e7fe1faf-be7a-4ca7-b89b-22863fffe432/79d4480a-c4e5-4920-95cd-ed831d45f5d0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for badm_550_ashley: https://atlas.nomic.ai/map/e7fe1faf-be7a-4ca7-b89b-22863fffe432/79d4480a-c4e5-4920-95cd-ed831d45f5d0]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:26:35.272 | INFO | nomic.project:create_index:1111 - Created map `badm_550_ashley_qr_index` in project `Query-Response Map for badm_550_ashley`: https://atlas.nomic.ai/map/e7fe1faf-be7a-4ca7-b89b-22863fffe432/2b997f7c-0084-4db7-8e9a-76eeb62d715b\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: ece120\n", + "(298, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:26:42.765 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for ece120` in organization `dabholkar.asmita`\n", + "2023-08-30 15:26:43.831 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:26:43.850 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:01, 1.20s/it]\n", + "2023-08-30 15:26:45.059 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:26:45.063 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:26:46.221 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for ece120` in project `Query-Response Map for ece120`: https://atlas.nomic.ai/map/f35d2617-d029-4a29-ba4a-27a82088be24/0c8e9404-be65-4780-bba3-e3b44f14de9b\n", + "2023-08-30 15:26:46.230 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for ece120: https://atlas.nomic.ai/map/f35d2617-d029-4a29-ba4a-27a82088be24/0c8e9404-be65-4780-bba3-e3b44f14de9b\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for ece120: https://atlas.nomic.ai/map/f35d2617-d029-4a29-ba4a-27a82088be24/0c8e9404-be65-4780-bba3-e3b44f14de9b]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:26:48.040 | INFO | nomic.project:create_index:1111 - Created map `ece120_qr_index` in project `Query-Response Map for ece120`: https://atlas.nomic.ai/map/f35d2617-d029-4a29-ba4a-27a82088be24/61277149-b12d-4b59-8bcd-e9dd29fc58a4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: badm-567-v3\n", + "(27, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:26:52.367 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for badm-567-v3` in organization `dabholkar.asmita`\n", + "2023-08-30 15:26:53.227 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:26:53.236 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.07it/s]\n", + "2023-08-30 15:26:54.177 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:26:54.185 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:26:55.379 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for badm-567-v3` in project `Query-Response Map for badm-567-v3`: https://atlas.nomic.ai/map/022b4521-24df-401d-beff-bba4f881590f/adabe0a1-d147-462a-8992-72af89cb5ae2\n", + "2023-08-30 15:26:55.379 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for badm-567-v3: https://atlas.nomic.ai/map/022b4521-24df-401d-beff-bba4f881590f/adabe0a1-d147-462a-8992-72af89cb5ae2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for badm-567-v3: https://atlas.nomic.ai/map/022b4521-24df-401d-beff-bba4f881590f/adabe0a1-d147-462a-8992-72af89cb5ae2]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:26:57.012 | INFO | nomic.project:create_index:1111 - Created map `badm-567-v3_qr_index` in project `Query-Response Map for badm-567-v3`: https://atlas.nomic.ai/map/022b4521-24df-401d-beff-bba4f881590f/e6c9798b-c154-43e7-917e-dd5cb71f116f\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: new-weather\n", + "(98, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:27:02.087 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for new-weather` in organization `dabholkar.asmita`\n", + "2023-08-30 15:27:03.117 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:27:03.125 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.07it/s]\n", + "2023-08-30 15:27:04.071 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:27:04.071 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:27:05.459 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for new-weather` in project `Query-Response Map for new-weather`: https://atlas.nomic.ai/map/48db10b9-0a6e-4c43-83f7-f7fa2b406257/fd200444-86cb-4b1c-9be6-ebd0262e2428\n", + "2023-08-30 15:27:05.461 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for new-weather: https://atlas.nomic.ai/map/48db10b9-0a6e-4c43-83f7-f7fa2b406257/fd200444-86cb-4b1c-9be6-ebd0262e2428\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for new-weather: https://atlas.nomic.ai/map/48db10b9-0a6e-4c43-83f7-f7fa2b406257/fd200444-86cb-4b1c-9be6-ebd0262e2428]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:27:07.001 | INFO | nomic.project:create_index:1111 - Created map `new-weather_qr_index` in project `Query-Response Map for new-weather`: https://atlas.nomic.ai/map/48db10b9-0a6e-4c43-83f7-f7fa2b406257/ff717c79-50cd-468b-9fcc-b391c8c167df\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: gies-online-mba-v2\n", + "(52, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:27:10.946 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for gies-online-mba-v2` in organization `dabholkar.asmita`\n", + "2023-08-30 15:27:11.862 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:27:11.868 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.33it/s]\n", + "2023-08-30 15:27:12.630 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:27:12.634 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:27:13.627 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for gies-online-mba-v2` in project `Query-Response Map for gies-online-mba-v2`: https://atlas.nomic.ai/map/8699b44f-8e2a-40e6-9562-0d2fc2898c3d/87144fbd-48a3-4d00-9136-ddc177685aa1\n", + "2023-08-30 15:27:13.627 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for gies-online-mba-v2: https://atlas.nomic.ai/map/8699b44f-8e2a-40e6-9562-0d2fc2898c3d/87144fbd-48a3-4d00-9136-ddc177685aa1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for gies-online-mba-v2: https://atlas.nomic.ai/map/8699b44f-8e2a-40e6-9562-0d2fc2898c3d/87144fbd-48a3-4d00-9136-ddc177685aa1]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:27:15.546 | INFO | nomic.project:create_index:1111 - Created map `gies-online-mba-v2_qr_index` in project `Query-Response Map for gies-online-mba-v2`: https://atlas.nomic.ai/map/8699b44f-8e2a-40e6-9562-0d2fc2898c3d/db13ea7a-f93d-4f97-b922-c51216d3d6e9\n", + "2023-08-30 15:27:15,670:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:27:15,673:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:27:20,003:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:27:20,003:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: frontend\n", + "(24, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:27:28.373 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for frontend` in organization `dabholkar.asmita`\n", + "2023-08-30 15:27:29.396 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:27:29.405 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.11it/s]\n", + "2023-08-30 15:27:30.325 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:27:30.325 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:27:31.539 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for frontend` in project `Query-Response Map for frontend`: https://atlas.nomic.ai/map/3a8ff45a-ca2d-4750-ab1b-ea033d35580d/e427ecf0-339a-41bf-8f9a-dc7a2930ac70\n", + "2023-08-30 15:27:31.542 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for frontend: https://atlas.nomic.ai/map/3a8ff45a-ca2d-4750-ab1b-ea033d35580d/e427ecf0-339a-41bf-8f9a-dc7a2930ac70\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for frontend: https://atlas.nomic.ai/map/3a8ff45a-ca2d-4750-ab1b-ea033d35580d/e427ecf0-339a-41bf-8f9a-dc7a2930ac70]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:27:33.028 | INFO | nomic.project:create_index:1111 - Created map `frontend_qr_index` in project `Query-Response Map for frontend`: https://atlas.nomic.ai/map/3a8ff45a-ca2d-4750-ab1b-ea033d35580d/006903b0-bb82-4432-9975-ff7c9ca80af9\n", + "2023-08-30 15:27:33,166:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:27:33,166:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:27:37,279:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:27:37,281:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:27:41,477:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:27:41,481:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: ECE220FA23\n", + "(193, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:27:50.988 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for ECE220FA23` in organization `dabholkar.asmita`\n", + "2023-08-30 15:27:51.867 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:27:51.878 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:01, 1.01s/it]\n", + "2023-08-30 15:27:52.904 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:27:52.908 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:27:53.929 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for ECE220FA23` in project `Query-Response Map for ECE220FA23`: https://atlas.nomic.ai/map/29b6f987-ee46-40be-808e-3d79a56d1b04/6b9b60f6-e2b7-4863-911c-fe84a63b35a1\n", + "2023-08-30 15:27:53.929 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for ECE220FA23: https://atlas.nomic.ai/map/29b6f987-ee46-40be-808e-3d79a56d1b04/6b9b60f6-e2b7-4863-911c-fe84a63b35a1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for ECE220FA23: https://atlas.nomic.ai/map/29b6f987-ee46-40be-808e-3d79a56d1b04/6b9b60f6-e2b7-4863-911c-fe84a63b35a1]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:27:55.645 | INFO | nomic.project:create_index:1111 - Created map `ECE220FA23_qr_index` in project `Query-Response Map for ECE220FA23`: https://atlas.nomic.ai/map/29b6f987-ee46-40be-808e-3d79a56d1b04/d8163c57-a2e8-41ca-90fc-043c8a9469b3\n", + "2023-08-30 15:27:55,758:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:27:55,759:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:27:59,841:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:27:59,841:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: Snowmass\n", + "(23, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:28:08.067 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for Snowmass` in organization `dabholkar.asmita`\n", + "2023-08-30 15:28:09.006 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:28:09.014 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.20it/s]\n", + "2023-08-30 15:28:09.854 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:28:09.858 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:28:10.994 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for Snowmass` in project `Query-Response Map for Snowmass`: https://atlas.nomic.ai/map/aa415361-5d23-44ba-ada2-252ed1dfe9a0/51de6990-9d60-4cc4-9a8e-46bc7de97a0e\n", + "2023-08-30 15:28:10.994 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for Snowmass: https://atlas.nomic.ai/map/aa415361-5d23-44ba-ada2-252ed1dfe9a0/51de6990-9d60-4cc4-9a8e-46bc7de97a0e\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for Snowmass: https://atlas.nomic.ai/map/aa415361-5d23-44ba-ada2-252ed1dfe9a0/51de6990-9d60-4cc4-9a8e-46bc7de97a0e]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:28:12.968 | INFO | nomic.project:create_index:1111 - Created map `Snowmass_qr_index` in project `Query-Response Map for Snowmass`: https://atlas.nomic.ai/map/aa415361-5d23-44ba-ada2-252ed1dfe9a0/4fdea35b-cda2-434e-afd1-e46e01430a97\n", + "2023-08-30 15:28:13,066:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:28:13,068:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:28:17,200:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:28:17,200:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:28:21,297:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:28:21,297:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: NPRE247\n", + "(54, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:28:29.951 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for NPRE247` in organization `dabholkar.asmita`\n", + "2023-08-30 15:28:31.043 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:28:31.051 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:01, 1.64s/it]\n", + "2023-08-30 15:28:32.709 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:28:32.714 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:28:33.787 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for NPRE247` in project `Query-Response Map for NPRE247`: https://atlas.nomic.ai/map/3ae13050-5e00-4f2b-9dd4-f05e98945405/d06b7c47-850c-48f4-a5f9-984fbb0ca424\n", + "2023-08-30 15:28:33.790 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for NPRE247: https://atlas.nomic.ai/map/3ae13050-5e00-4f2b-9dd4-f05e98945405/d06b7c47-850c-48f4-a5f9-984fbb0ca424\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for NPRE247: https://atlas.nomic.ai/map/3ae13050-5e00-4f2b-9dd4-f05e98945405/d06b7c47-850c-48f4-a5f9-984fbb0ca424]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:28:35.347 | INFO | nomic.project:create_index:1111 - Created map `NPRE247_qr_index` in project `Query-Response Map for NPRE247`: https://atlas.nomic.ai/map/3ae13050-5e00-4f2b-9dd4-f05e98945405/03471048-46aa-473c-b599-0bc812c679c0\n", + "2023-08-30 15:28:35,479:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:28:35,484:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:28:39,590:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:28:39,594:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: your-awesome-course\n", + "(30, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:28:50.102 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for your-awesome-course` in organization `dabholkar.asmita`\n", + "2023-08-30 15:28:51.013 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:28:51.022 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.10it/s]\n", + "2023-08-30 15:28:51.943 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:28:51.945 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:28:52.904 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for your-awesome-course` in project `Query-Response Map for your-awesome-course`: https://atlas.nomic.ai/map/dd5dfe40-c810-4402-9684-a43783ea6e52/468942cc-d46a-45a2-82a6-11fac2f2be78\n", + "2023-08-30 15:28:52.907 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for your-awesome-course: https://atlas.nomic.ai/map/dd5dfe40-c810-4402-9684-a43783ea6e52/468942cc-d46a-45a2-82a6-11fac2f2be78\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for your-awesome-course: https://atlas.nomic.ai/map/dd5dfe40-c810-4402-9684-a43783ea6e52/468942cc-d46a-45a2-82a6-11fac2f2be78]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:28:54.405 | INFO | nomic.project:create_index:1111 - Created map `your-awesome-course_qr_index` in project `Query-Response Map for your-awesome-course`: https://atlas.nomic.ai/map/dd5dfe40-c810-4402-9684-a43783ea6e52/e58f20e3-fa19-4c4c-8764-a185e0691c85\n", + "2023-08-30 15:28:54,549:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:28:54,549:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:28:58,646:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:28:58,653:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: pract\n", + "(44, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:29:07.007 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for pract` in organization `dabholkar.asmita`\n", + "2023-08-30 15:29:08.243 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:29:08.251 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:01, 1.11s/it]\n", + "2023-08-30 15:29:09.368 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:29:09.368 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:29:10.392 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for pract` in project `Query-Response Map for pract`: https://atlas.nomic.ai/map/15b51f99-fe4b-41ec-97fd-5579884e39fb/669c8862-db60-41ad-94a9-fd815e1fdcd8\n", + "2023-08-30 15:29:10.392 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for pract: https://atlas.nomic.ai/map/15b51f99-fe4b-41ec-97fd-5579884e39fb/669c8862-db60-41ad-94a9-fd815e1fdcd8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for pract: https://atlas.nomic.ai/map/15b51f99-fe4b-41ec-97fd-5579884e39fb/669c8862-db60-41ad-94a9-fd815e1fdcd8]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:29:12.031 | INFO | nomic.project:create_index:1111 - Created map `pract_qr_index` in project `Query-Response Map for pract`: https://atlas.nomic.ai/map/15b51f99-fe4b-41ec-97fd-5579884e39fb/b15570eb-4db4-4b6f-9b4d-e80309d2dcb3\n", + "2023-08-30 15:29:12,113:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:29:12,115:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:29:16,201:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:29:16,209:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:29:20,282:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:29:20,285:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: ece120FL22\n", + "(53, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:29:28.994 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for ece120FL22` in organization `dabholkar.asmita`\n", + "2023-08-30 15:29:29.838 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:29:29.846 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.18it/s]\n", + "2023-08-30 15:29:30.708 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:29:30.710 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:29:31.828 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for ece120FL22` in project `Query-Response Map for ece120FL22`: https://atlas.nomic.ai/map/8e74c555-3192-451b-b4a1-2c86587df88b/888e5d82-2e4d-46ba-b920-915dc31b835b\n", + "2023-08-30 15:29:31.828 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for ece120FL22: https://atlas.nomic.ai/map/8e74c555-3192-451b-b4a1-2c86587df88b/888e5d82-2e4d-46ba-b920-915dc31b835b\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for ece120FL22: https://atlas.nomic.ai/map/8e74c555-3192-451b-b4a1-2c86587df88b/888e5d82-2e4d-46ba-b920-915dc31b835b]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:29:33.354 | INFO | nomic.project:create_index:1111 - Created map `ece120FL22_qr_index` in project `Query-Response Map for ece120FL22`: https://atlas.nomic.ai/map/8e74c555-3192-451b-b4a1-2c86587df88b/eb4e553d-ecd3-4b11-9d75-468108ab08e2\n", + "2023-08-30 15:29:33,458:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:29:33,458:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:29:37,544:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:29:37,545:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:29:41,634:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:29:41,642:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: Law794-TransactionalDraftingAlam\n", + "(21, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:29:49.618 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for Law794-TransactionalDraftingAlam` in organization `dabholkar.asmita`\n", + "2023-08-30 15:29:50.718 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:29:50.731 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:01, 1.11s/it]\n", + "2023-08-30 15:29:51.849 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:29:51.851 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:29:53.034 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for Law794-TransactionalDraftingAlam` in project `Query-Response Map for Law794-TransactionalDraftingAlam`: https://atlas.nomic.ai/map/c4328a33-8958-462e-ba30-50f342736dae/0da00b97-bf94-4805-935e-458fb5c3d76f\n", + "2023-08-30 15:29:53.034 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for Law794-TransactionalDraftingAlam: https://atlas.nomic.ai/map/c4328a33-8958-462e-ba30-50f342736dae/0da00b97-bf94-4805-935e-458fb5c3d76f\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for Law794-TransactionalDraftingAlam: https://atlas.nomic.ai/map/c4328a33-8958-462e-ba30-50f342736dae/0da00b97-bf94-4805-935e-458fb5c3d76f]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:29:54.605 | INFO | nomic.project:create_index:1111 - Created map `Law794-TransactionalDraftingAlam_qr_index` in project `Query-Response Map for Law794-TransactionalDraftingAlam`: https://atlas.nomic.ai/map/c4328a33-8958-462e-ba30-50f342736dae/9792fd4e-2196-4e39-bded-cc2bfd42abbf\n", + "2023-08-30 15:29:54,728:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:29:54,731:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:29:58,804:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:29:58,804:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: NCSA\n", + "(84, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:30:07.528 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for NCSA` in organization `dabholkar.asmita`\n", + "2023-08-30 15:30:08.422 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:30:08.431 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.09it/s]\n", + "2023-08-30 15:30:09.361 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:30:09.361 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:30:10.325 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for NCSA` in project `Query-Response Map for NCSA`: https://atlas.nomic.ai/map/237ff7ed-c987-4259-96e0-3c5ed6e21207/8f176512-6a1e-4c8f-9d94-5794b1d7e619\n", + "2023-08-30 15:30:10.325 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for NCSA: https://atlas.nomic.ai/map/237ff7ed-c987-4259-96e0-3c5ed6e21207/8f176512-6a1e-4c8f-9d94-5794b1d7e619\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for NCSA: https://atlas.nomic.ai/map/237ff7ed-c987-4259-96e0-3c5ed6e21207/8f176512-6a1e-4c8f-9d94-5794b1d7e619]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:30:11.917 | INFO | nomic.project:create_index:1111 - Created map `NCSA_qr_index` in project `Query-Response Map for NCSA`: https://atlas.nomic.ai/map/237ff7ed-c987-4259-96e0-3c5ed6e21207/2ce836ec-557b-4037-9ebd-d3e8982c0926\n", + "2023-08-30 15:30:12,004:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:30:12,004:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:30:16,092:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:30:16,092:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:30:20,157:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:30:20,164:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: NCSADelta\n", + "(22, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:30:28.362 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for NCSADelta` in organization `dabholkar.asmita`\n", + "2023-08-30 15:30:29.318 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:30:29.326 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.10it/s]\n", + "2023-08-30 15:30:30.246 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:30:30.251 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:30:31.253 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for NCSADelta` in project `Query-Response Map for NCSADelta`: https://atlas.nomic.ai/map/f0c381d5-e2e9-41fb-807a-ae61bc818122/78d29eb7-be96-47c1-a856-b684c6419e34\n", + "2023-08-30 15:30:31.254 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for NCSADelta: https://atlas.nomic.ai/map/f0c381d5-e2e9-41fb-807a-ae61bc818122/78d29eb7-be96-47c1-a856-b684c6419e34\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for NCSADelta: https://atlas.nomic.ai/map/f0c381d5-e2e9-41fb-807a-ae61bc818122/78d29eb7-be96-47c1-a856-b684c6419e34]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:30:33.104 | INFO | nomic.project:create_index:1111 - Created map `NCSADelta_qr_index` in project `Query-Response Map for NCSADelta`: https://atlas.nomic.ai/map/f0c381d5-e2e9-41fb-807a-ae61bc818122/b6e64fef-a829-435f-89b5-ed1a44c05202\n", + "2023-08-30 15:30:33,214:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:30:33,214:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:30:37,289:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:30:37,290:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:30:41,376:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:30:41,382:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: NuclGPT-v1\n", + "(25, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:30:49.297 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for NuclGPT-v1` in organization `dabholkar.asmita`\n", + "2023-08-30 15:30:50.216 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:30:50.222 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.23it/s]\n", + "2023-08-30 15:30:51.043 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:30:51.043 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:30:52.360 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for NuclGPT-v1` in project `Query-Response Map for NuclGPT-v1`: https://atlas.nomic.ai/map/76043322-073c-4d8c-8c90-8e563da8564f/335097b0-a80d-4f38-8559-c6075a46a8b2\n", + "2023-08-30 15:30:52.360 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for NuclGPT-v1: https://atlas.nomic.ai/map/76043322-073c-4d8c-8c90-8e563da8564f/335097b0-a80d-4f38-8559-c6075a46a8b2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for NuclGPT-v1: https://atlas.nomic.ai/map/76043322-073c-4d8c-8c90-8e563da8564f/335097b0-a80d-4f38-8559-c6075a46a8b2]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:30:53.900 | INFO | nomic.project:create_index:1111 - Created map `NuclGPT-v1_qr_index` in project `Query-Response Map for NuclGPT-v1`: https://atlas.nomic.ai/map/76043322-073c-4d8c-8c90-8e563da8564f/441b7ff6-00ef-47f3-98a9-e45d327a1414\n" + ] + } + ], + "source": [ + "low_volume_courses = []\n", + "high_volume_courses = []\n", + "for course in course_names:\n", + " if course is None or course == 'ece408':\n", + " continue\n", + " \n", + " user_queries = []\n", + " metadata = []\n", + " i = 1\n", + " course_df = df[df['course_name'] == course]['convo']\n", + " for convo in course_df: # iterate through all conversations in a course\n", + " messages = convo['messages']\n", + "\n", + " # form query-response pairs out of the messages\n", + " pair_check = 0\n", + " for message in messages:\n", + " if message['role'] == 'user' and pair_check == 0:\n", + " query = message['content']\n", + " metadata_row = {'course': course, 'query': message['content']}\n", + " #print(\"metadata row: \", metadata_row)\n", + " pair_check += 1\n", + " \n", + " if message['role'] == 'assistant' and pair_check == 1:\n", + " metadata_row['response'] = message['content']\n", + " metadata_row['id'] = i \n", + " #print(\"response metadata row: \", metadata_row)\n", + " i += 1\n", + " pair_check += 1\n", + "\n", + " if pair_check == 2:\n", + " # one conversation pair is complete\n", + " user_queries.append(query)\n", + " metadata.append(metadata_row)\n", + " pair_check = 0\n", + " # after iterating every conversation in a course, create the map\n", + " if len(user_queries) < 20:\n", + " low_volume_courses.append(course)\n", + " continue\n", + "\n", + " if len(user_queries) > 500:\n", + " high_volume_courses.append(course)\n", + " continue\n", + " \n", + " metadata = pd.DataFrame(metadata)\n", + " embeddings = embeddings_model.embed_documents(user_queries)\n", + " embeddings = np.array(embeddings)\n", + " print(\"course name: \", course)\n", + " print(embeddings.shape)\n", + "\n", + " # create an Atlas project\n", + " project_name = \"Query-Response Map for \" + course\n", + " index_name = course + \"_qr_index\"\n", + " project = atlas.map_embeddings(embeddings=np.array(embeddings),\n", + " data=metadata,\n", + " id_field='id',\n", + " build_topic_model=True,\n", + " topic_label_field='query',\n", + " name=project_name,\n", + " colorable_fields=['query'])\n", + " print(project.maps)\n", + "\n", + " project.create_index(index_name, build_topic_model=True)\n", + "\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "low volume courses: ['test-video-ingest', 'badm-567', 'test-video-ingest-28', 'pdeploy999', 'badm-350-summer', 'previewtesting1', 'localtest2', 'your-favorite-url', 'mantine', 'test-video-ingest-2', 'badm_567_v2', 'erpnext', 'mip', 'farmdoc_test_kastan-v1', 'personalstatement', 'hrc', 'csv', 'star_nox', 'badm_567', 'SPaRCEd', 'localdemo8', 'badm_567_thumbnails', 'chatbot', 'erp', 'extreme', 'rohan_atree', 'zotero-extreme', 'test-video-ingest-20', 'gies-online-mba2', 'gies-online-mba', 'careerassistant', 'weather', 'lillian-wang-blog', 'local-test5', 'demo-for-vyriad', 'ag-gpt-beta', 'rtest', 'previewdeploy', 'r2test', 'personal-statement', 'rohan_excel', 'langchain-python', 'langchain', 'ncsa-live-demo', 'rohan_atree_individual', 'meta11-test', 'HealthyLivingGuide', 'rohan', 'babreu', 'test-video-ingest-31', 'p', 'test-video-ingest-17', 'summary', 'test-video-ingest-3', 'test-video-ingest-27', 'lillian-wang-blog-2', 'python-magic', 'ansible2', 'ece408fa23', 'farmdoc_test_josh_v2', 'local-test3', 'automata', 'SpaceFlorida-GT', 'GBSI-GT', 'newnew_ncsa', 'canvas', 'gbsi-gt', 'meditation-tutorial', 'profit', 'ansible', 'langchain-docs', 'testing_url_metadata_josh', 'test-india-biodiversity', 'vyriad', 'irc-workplans', 'kastanasdfasdfasdf', 'BADM-567-GT', 'mdt', 'vercel', 'gies-graduate-hub', 'test-video-ingest-12', 'test-video-ingest-13', 'Gies-graduate-hub', 'test_new_supa_scrape', 'doe-ascr-2023', 'arize', 'final-meta-test', 'preview-meta-test', 'gies-online-mba-V3', 'FoF-Drawdown-from-INVPEIV-5-24-23', 'FIN574-GT', 'test-video-ingest-30', 'test', 'NCSA-v2', 'conversational', 'clowder-docs', 'DA', 'test-video-ingest-21', 'test-video-ingest-25', 'test-ingest-10', 'eric-test-course', 'farmdoc-test', 'test-video-ingest-22', 'Academic-paper', 'starwars', 'AdamDemo']\n", + "high volume courses: ['gpt4', 'ECE408FA23']\n" + ] + } + ], + "source": [ + "print(\"low volume courses: \", low_volume_courses)\n", + "print(\"high volume courses: \", high_volume_courses)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +}