diff --git a/ai_ta_backend/nomic_logging.py b/ai_ta_backend/nomic_logging.py index d5ca3c70..7da46e89 100644 --- a/ai_ta_backend/nomic_logging.py +++ b/ai_ta_backend/nomic_logging.py @@ -21,6 +21,8 @@ def log_convo_to_nomic(course_name: str, conversation) -> str: 3. Keep current logic for map doesn't exist - update metadata """ print("in log_convo_to_nomic()") + + print("conversation: ", conversation) messages = conversation['conversation']['messages'] user_email = conversation['conversation']['user_email'] @@ -64,7 +66,12 @@ def log_convo_to_nomic(course_name: str, conversation) -> str: # select the last 2 messages and append new convo to prev convo messages_to_be_logged = messages[-2:] for message in messages_to_be_logged: - prev_convo += "\n>>> " + message['role'] + ": " + message['content'] + "\n" + if message['role'] == 'user': + emoji = "🙋" + else: + emoji = "🤖" + + prev_convo += "\n>>> " + emoji + message['role'] + ": " + message['content'] + "\n" # update metadata metadata = [{"course": course_name, "conversation": prev_convo, "conversation_id": conversation_id, @@ -79,7 +86,7 @@ def log_convo_to_nomic(course_name: str, conversation) -> str: user_queries.append(first_message) for message in messages: - conversation_string += "\n>>> " + message['role'] + ": " + message['content'] + "\n" + conversation_string += "\n>>> " + emoji + message['role'] + ": " + message['content'] + "\n" metadata = [{"course": course_name, "conversation": conversation_string, "conversation_id": conversation_id, "id": last_id+1, "user_email": user_email, "first_query": first_message}] @@ -177,15 +184,23 @@ def create_nomic_map(course_name: str, log_data: list): # create metadata for multi-turn conversation conversation = "" + if message['role'] == 'user': + emoji = "🙋" + else: + emoji = "🤖" for message in messages: # string of role: content, role: content, ... - conversation += "\n>>> " + message['role'] + ": " + message['content'] + "\n" + conversation += "\n>>> " + emoji + message['role'] + ": " + message['content'] + "\n" # append current chat to previous chat if convo already exists if convo['id'] == log_conversation_id: conversation_exists = True + if m['role'] == 'user': + emoji = "🙋" + else: + emoji = "🤖" for m in log_messages: - conversation += "\n>>> " + m['role'] + ": " + m['content'] + "\n" + conversation += "\n>>> " + emoji + m['role'] + ": " + m['content'] + "\n" # add to metadata metadata_row = {"course": row['course_name'], "conversation": conversation, "conversation_id": convo['id'], @@ -198,7 +213,11 @@ def create_nomic_map(course_name: str, log_data: list): user_queries.append(log_messages[0]['content']) conversation = "" for message in log_messages: - conversation += "\n>>> " + message['role'] + ": " + message['content'] + "\n" + if message['role'] == 'user': + emoji = "🙋" + else: + emoji = "🤖" + conversation += "\n>>> " + emoji + message['role'] + ": " + message['content'] + "\n" metadata_row = {"course": course_name, "conversation": conversation, "conversation_id": log_conversation_id, "id": i, "user_email": log_user_email, "first_query": log_messages[0]['content']} metadata.append(metadata_row) diff --git a/ai_ta_backend/nomic_map_creation.ipynb b/ai_ta_backend/nomic_map_creation.ipynb new file mode 100644 index 00000000..23924157 --- /dev/null +++ b/ai_ta_backend/nomic_map_creation.ipynb @@ -0,0 +1,1556 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-09-07 11:57:29,274:INFO - Note: NumExpr detected 16 cores but \"NUMEXPR_MAX_THREADS\" not set, so enforcing safe limit of 8.\n", + "2023-09-07 11:57:29,274:INFO - NumExpr defaulting to 8 threads.\n" + ] + } + ], + "source": [ + "# import required libraries\n", + "\n", + "import os\n", + "import supabase\n", + "from nomic import atlas\n", + "from dotenv import load_dotenv\n", + "from langchain.embeddings import OpenAIEmbeddings\n", + "import numpy as np\n", + "import time\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# loading environment variables\n", + "\n", + "env_path = \"../.env\"\n", + "load_dotenv(dotenv_path=env_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# initialize supabase client\n", + "\n", + "url = os.environ.get(\"SUPABASE_URL\")\n", + "key = os.environ.get(\"SUPABASE_API_KEY\")\n", + "\n", + "supabase_client = supabase.create_client(url, key)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idcreated_atconvoconvo_idcourse_nameuser_email
052002023-09-07T17:03:47.705812+00:00{'id': 'abd2e044-fbff-455e-8c60-755cc7635182',...abd2e044-fbff-455e-8c60-755cc7635182cropwizardavd6@illinois.edu
152012023-09-07T17:05:25.863567+00:00{'id': '3e5d4861-b128-4c64-96ac-87c74f3217e5',...3e5d4861-b128-4c64-96ac-87c74f3217e5cropwizardavd6@illinois.edu
252162023-09-07T17:18:32.197379+00:00{'id': '43ee631a-cb58-43f5-b2af-a5b91b7585cd',...43ee631a-cb58-43f5-b2af-a5b91b7585cdcropwizardavd6@illinois.edu
352122023-09-07T17:16:34.267931+00:00{'id': '0129ea46-207f-47e3-be90-da143857000f',...0129ea46-207f-47e3-be90-da143857000fcropwizardavd6@illinois.edu
452172023-09-07T17:19:00.681823+00:00{'id': 'c6b4e4d8-4de7-4387-b4e9-411084dffea6',...c6b4e4d8-4de7-4387-b4e9-411084dffea6cropwizardavd6@illinois.edu
552232023-09-07T17:22:38.970643+00:00{'id': 'b5500763-7e7b-4b23-9031-cc320a51ccbf',...b5500763-7e7b-4b23-9031-cc320a51ccbfcropwizardavd6@illinois.edu
652272023-09-07T17:24:10.362647+00:00{'id': 'd410955f-4398-4869-b395-e6b659cc2d06',...d410955f-4398-4869-b395-e6b659cc2d06cropwizardavd6@illinois.edu
752092023-09-07T17:14:43.518617+00:00{'id': '0ecd2c05-772a-42aa-b29a-0a892bd0e9ab',...0ecd2c05-772a-42aa-b29a-0a892bd0e9abcropwizardavd6@illinois.edu
852222023-09-07T17:21:29.223343+00:00{'id': 'c82056a0-2d67-4ce8-82e3-86a30f1f6dc0',...c82056a0-2d67-4ce8-82e3-86a30f1f6dc0cropwizardavd6@illinois.edu
952242023-09-07T17:22:54.856839+00:00{'id': '2316bbd7-61f3-44aa-a79e-bb42bd688c47',...2316bbd7-61f3-44aa-a79e-bb42bd688c47cropwizardavd6@illinois.edu
1052262023-09-07T17:23:27.644745+00:00{'id': '66abfe85-bb04-456e-8709-89f9aafe5508',...66abfe85-bb04-456e-8709-89f9aafe5508cropwizardavd6@illinois.edu
1152282023-09-07T17:24:41.32465+00:00{'id': '175ad6b2-3bf2-4889-b2de-a18961ee8ecb',...175ad6b2-3bf2-4889-b2de-a18961ee8ecbcropwizardavd6@illinois.edu
1252322023-09-07T17:30:05.770146+00:00{'id': 'f9859e36-bf76-40ab-9413-91ef6663dbd6',...f9859e36-bf76-40ab-9413-91ef6663dbd6cropwizardavd6@illinois.edu
1352332023-09-07T17:30:52.749867+00:00{'id': 'bab32d0b-8e2b-4eaa-a46e-069be0d0c3a2',...bab32d0b-8e2b-4eaa-a46e-069be0d0c3a2cropwizardavd6@illinois.edu
1452342023-09-07T17:31:19.801611+00:00{'id': 'ecaf3228-78f3-49f7-b46d-3a5c3d5b62fd',...ecaf3228-78f3-49f7-b46d-3a5c3d5b62fdcropwizardavd6@illinois.edu
1552372023-09-07T17:36:14.68431+00:00{'id': 'edead825-12df-417c-af40-059e83067c69',...edead825-12df-417c-af40-059e83067c69cropwizardavd6@illinois.edu
1652382023-09-07T17:36:42.984907+00:00{'id': 'bc44d229-327a-452d-a386-8868216a1bd2',...bc44d229-327a-452d-a386-8868216a1bd2cropwizardavd6@illinois.edu
1752412023-09-07T17:37:22.134543+00:00{'id': 'ff7a1c27-e126-49db-be79-6deaefcffec3',...ff7a1c27-e126-49db-be79-6deaefcffec3cropwizardavd6@illinois.edu
1853042023-09-07T19:45:21.73541+00:00{'id': '6226b153-356a-408c-9483-49ef5808538c',...6226b153-356a-408c-9483-49ef5808538ccropwizardavd6@illinois.edu
1953052023-09-07T19:46:03.626639+00:00{'id': 'e9edae6b-b7e1-46a8-b5e8-6215890a2a01',...e9edae6b-b7e1-46a8-b5e8-6215890a2a01cropwizardavd6@illinois.edu
2053062023-09-07T19:46:36.076704+00:00{'id': 'b2116035-da7b-4136-878d-66a10098a756',...b2116035-da7b-4136-878d-66a10098a756cropwizardavd6@illinois.edu
2151952023-09-06T23:43:38.201481+00:00{'id': '543ee10e-faf0-47a8-bb1c-c040aec44ed1',...543ee10e-faf0-47a8-bb1c-c040aec44ed1cropwizarddabholkar.asmita@gmail.com
\n", + "
" + ], + "text/plain": [ + " id created_at \\\n", + "0 5200 2023-09-07T17:03:47.705812+00:00 \n", + "1 5201 2023-09-07T17:05:25.863567+00:00 \n", + "2 5216 2023-09-07T17:18:32.197379+00:00 \n", + "3 5212 2023-09-07T17:16:34.267931+00:00 \n", + "4 5217 2023-09-07T17:19:00.681823+00:00 \n", + "5 5223 2023-09-07T17:22:38.970643+00:00 \n", + "6 5227 2023-09-07T17:24:10.362647+00:00 \n", + "7 5209 2023-09-07T17:14:43.518617+00:00 \n", + "8 5222 2023-09-07T17:21:29.223343+00:00 \n", + "9 5224 2023-09-07T17:22:54.856839+00:00 \n", + "10 5226 2023-09-07T17:23:27.644745+00:00 \n", + "11 5228 2023-09-07T17:24:41.32465+00:00 \n", + "12 5232 2023-09-07T17:30:05.770146+00:00 \n", + "13 5233 2023-09-07T17:30:52.749867+00:00 \n", + "14 5234 2023-09-07T17:31:19.801611+00:00 \n", + "15 5237 2023-09-07T17:36:14.68431+00:00 \n", + "16 5238 2023-09-07T17:36:42.984907+00:00 \n", + "17 5241 2023-09-07T17:37:22.134543+00:00 \n", + "18 5304 2023-09-07T19:45:21.73541+00:00 \n", + "19 5305 2023-09-07T19:46:03.626639+00:00 \n", + "20 5306 2023-09-07T19:46:36.076704+00:00 \n", + "21 5195 2023-09-06T23:43:38.201481+00:00 \n", + "\n", + " convo \\\n", + "0 {'id': 'abd2e044-fbff-455e-8c60-755cc7635182',... \n", + "1 {'id': '3e5d4861-b128-4c64-96ac-87c74f3217e5',... \n", + "2 {'id': '43ee631a-cb58-43f5-b2af-a5b91b7585cd',... \n", + "3 {'id': '0129ea46-207f-47e3-be90-da143857000f',... \n", + "4 {'id': 'c6b4e4d8-4de7-4387-b4e9-411084dffea6',... \n", + "5 {'id': 'b5500763-7e7b-4b23-9031-cc320a51ccbf',... \n", + "6 {'id': 'd410955f-4398-4869-b395-e6b659cc2d06',... \n", + "7 {'id': '0ecd2c05-772a-42aa-b29a-0a892bd0e9ab',... \n", + "8 {'id': 'c82056a0-2d67-4ce8-82e3-86a30f1f6dc0',... \n", + "9 {'id': '2316bbd7-61f3-44aa-a79e-bb42bd688c47',... \n", + "10 {'id': '66abfe85-bb04-456e-8709-89f9aafe5508',... \n", + "11 {'id': '175ad6b2-3bf2-4889-b2de-a18961ee8ecb',... \n", + "12 {'id': 'f9859e36-bf76-40ab-9413-91ef6663dbd6',... \n", + "13 {'id': 'bab32d0b-8e2b-4eaa-a46e-069be0d0c3a2',... \n", + "14 {'id': 'ecaf3228-78f3-49f7-b46d-3a5c3d5b62fd',... \n", + "15 {'id': 'edead825-12df-417c-af40-059e83067c69',... \n", + "16 {'id': 'bc44d229-327a-452d-a386-8868216a1bd2',... \n", + "17 {'id': 'ff7a1c27-e126-49db-be79-6deaefcffec3',... \n", + "18 {'id': '6226b153-356a-408c-9483-49ef5808538c',... \n", + "19 {'id': 'e9edae6b-b7e1-46a8-b5e8-6215890a2a01',... \n", + "20 {'id': 'b2116035-da7b-4136-878d-66a10098a756',... \n", + "21 {'id': '543ee10e-faf0-47a8-bb1c-c040aec44ed1',... \n", + "\n", + " convo_id course_name \\\n", + "0 abd2e044-fbff-455e-8c60-755cc7635182 cropwizard \n", + "1 3e5d4861-b128-4c64-96ac-87c74f3217e5 cropwizard \n", + "2 43ee631a-cb58-43f5-b2af-a5b91b7585cd cropwizard \n", + "3 0129ea46-207f-47e3-be90-da143857000f cropwizard \n", + "4 c6b4e4d8-4de7-4387-b4e9-411084dffea6 cropwizard \n", + "5 b5500763-7e7b-4b23-9031-cc320a51ccbf cropwizard \n", + "6 d410955f-4398-4869-b395-e6b659cc2d06 cropwizard \n", + "7 0ecd2c05-772a-42aa-b29a-0a892bd0e9ab cropwizard \n", + "8 c82056a0-2d67-4ce8-82e3-86a30f1f6dc0 cropwizard \n", + "9 2316bbd7-61f3-44aa-a79e-bb42bd688c47 cropwizard \n", + "10 66abfe85-bb04-456e-8709-89f9aafe5508 cropwizard \n", + "11 175ad6b2-3bf2-4889-b2de-a18961ee8ecb cropwizard \n", + "12 f9859e36-bf76-40ab-9413-91ef6663dbd6 cropwizard \n", + "13 bab32d0b-8e2b-4eaa-a46e-069be0d0c3a2 cropwizard \n", + "14 ecaf3228-78f3-49f7-b46d-3a5c3d5b62fd cropwizard \n", + "15 edead825-12df-417c-af40-059e83067c69 cropwizard \n", + "16 bc44d229-327a-452d-a386-8868216a1bd2 cropwizard \n", + "17 ff7a1c27-e126-49db-be79-6deaefcffec3 cropwizard \n", + "18 6226b153-356a-408c-9483-49ef5808538c cropwizard \n", + "19 e9edae6b-b7e1-46a8-b5e8-6215890a2a01 cropwizard \n", + "20 b2116035-da7b-4136-878d-66a10098a756 cropwizard \n", + "21 543ee10e-faf0-47a8-bb1c-c040aec44ed1 cropwizard \n", + "\n", + " user_email \n", + "0 avd6@illinois.edu \n", + "1 avd6@illinois.edu \n", + "2 avd6@illinois.edu \n", + "3 avd6@illinois.edu \n", + "4 avd6@illinois.edu \n", + "5 avd6@illinois.edu \n", + "6 avd6@illinois.edu \n", + "7 avd6@illinois.edu \n", + "8 avd6@illinois.edu \n", + "9 avd6@illinois.edu \n", + "10 avd6@illinois.edu \n", + "11 avd6@illinois.edu \n", + "12 avd6@illinois.edu \n", + "13 avd6@illinois.edu \n", + "14 avd6@illinois.edu \n", + "15 avd6@illinois.edu \n", + "16 avd6@illinois.edu \n", + "17 avd6@illinois.edu \n", + "18 avd6@illinois.edu \n", + "19 avd6@illinois.edu \n", + "20 avd6@illinois.edu \n", + "21 dabholkar.asmita@gmail.com " + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# query data for one course for testing\n", + "course = 'cropwizard'\n", + "response = supabase_client.table(\"llm-convo-monitor\").select(\"*\").eq(\"course_name\", course).execute()\n", + "data = response.data\n", + "df = pd.DataFrame(data)\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0 {'id': 'abd2e044-fbff-455e-8c60-755cc7635182',...\n", + "1 {'id': '3e5d4861-b128-4c64-96ac-87c74f3217e5',...\n", + "2 {'id': '43ee631a-cb58-43f5-b2af-a5b91b7585cd',...\n", + "3 {'id': '0129ea46-207f-47e3-be90-da143857000f',...\n", + "4 {'id': 'c6b4e4d8-4de7-4387-b4e9-411084dffea6',...\n", + "Name: convo, dtype: object" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "course_df = df[df['course_name'] == course]['convo']\n", + "course_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'role': 'assistant', 'content': \"The U.S. Environmental Protection Agency (EPA)'s mission is to protect human health and the environment. As part of this mission, the EPA is involved in efforts such as developing strategies to protect endangered and threatened species from potential impacts of agricultural practices, including the use of herbicides. For instance, the EPA has released a draft Herbicide Strategy for public comment, aimed at proposing early mitigations for more than 900 listed species and designated critical habitats to reduce potential impacts from the agricultural use of herbicides(1^,2^,3^,4^).\\n\\n1. University of Illinois Extension\\n2. EPA releases draft herbicide strategy\\n3. EPA releases draft herbicide strategy\\n4. extension.pdf, page: 3\", 'contexts': [{'base_url': 'https://extension.illinois.edu/newsletters/illinois-pesticide-review-newsletter/julyaugust-2023', 'course_name ': 'cropwizard', 'pagenumber': '', 'readable_filename': 'University of Illinois Extension', 's3_path': 'courses/cropwizard/University_of_Illinois_Extension.html', 'text': \". — The U.S. Environmental Protection Agency, in collaboration with the U.S. Department of Energy, is funding the creation of Thriving Communities Technical Assistance Centers to help rural and underserved communities access state and federal funding for environmental infrastructure…\\n\\n\\n \\nAddress health challenges with holistic solutions\\nSeptember 1, 2023\\n\\nURBANA, Ill. — The University of Illinois, along with the Interdisciplinary Health Sciences Institute, and in collaboration with Illinois Extension, has developed the Autumn Health Picks 2023 webinar series. This series is part of the Community Seminar Series, and it provides an opportunity for…\\n\\n\\n \\nDo artificial roosts help bats? Illinois experts say more research needed\\nSeptember 1, 2023\\n\\nURBANA, Ill.\\xa0— Artificial roosts for bats come in many forms — bat boxes, condos, bark mimics, clay roosts, and cinder block structures, to name a few — but a new conservation practice and policy article from researchers at the\\xa0University of Illinois Urbana-Champaign\\xa0suggests the structures…\\n\\n\\nMore news\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\nUpcoming Events\\n\\n\\n\\n\\n \\n\\nRead Before You Sign: Renting & Leasing \\n\\n\\nSeptember 6, 2023\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\nTechnology & Soil Health Field Day\\n\\n\\nSeptember 6, 2023\\n\\n\\nCounty\\n\\nHenry\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\nPollinator Plants to Span the Season\\n\\n\\nSeptember 6, 2023\\n\\n\\nCounty\\n\\nMacoupin\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\nOrr Beef Research Center Field Day\\n\\n\\nSeptember 6, 2023\\n\\n\\n\\n\\n\\n\\nMore Events\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nSubtitle\\nOpens the Door to New Opportunities\\n\\n\\n\\n\\nTitle\\nLearn @ Illinois Extension\\n\\n\\n\\n\\nBody\\nThere is so much you want to do, but the challenges of time, distance, and cost create barriers to achieving those goals. You need a program that's flexible to your schedule.\\xa0Learn @ Illinois Extension\\xa0helps remove those challenge by offering\\xa0flexible online learning programs that meet your personal interests and continuing education requirements. We provide learning on your terms so you can be who you were meant to be.\\xa0\\n\\n\\n\\nOnline Courses\\n\\n\\n\\n\\n\\n \\n\\n\\nLatest Podcast\\n\\n\\n\\n\\nGood Growing\\n\\n\\nGardenbite: Three tips for a healthier lawn | #GoodGrowingThis week on the Good Growing podcast Chris shares a Gardenbite of when retired horticulture educator Richard Hentschel visited the show in 2021 to talk about fall lawn care. During the show, Richard spoke about three things we could all do to reduce our lawn inputs.\\xa0Want to see or...\\n\\n\\n Your browser does not support iframes, but you can visit \\n\\n\\n\\n\\n\\nMore Podcasts\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\nDig Deeper\\n\\n\\nTopics we address\\n\\n4-H Youth Development\\nBeef Cattle\\nClimate\\nCommunity Gardens\\nCommunity Planning\\nCottage Food\\nCrops\\nDisasters\\nEconomic Development\\nEnergy\\nEnvironmental Quality\\nFamily\\nFinances\\nFirewood\\nFlowers\\nFood\\nForestry\\nHealth\\nHemp\\nHerbs\\nHome Vegetable Gardening\\nIllinois Grasses\\nInsects\\nInvasives\\nLivestock\\nLocal Food Systems and Small Farms\\nLocal Government Education\\nMental Health\\nMushrooms\\nNatural Resources\\nPlant Problems\\nPlants\\nRainfall Management\\nSoil\\nSpecialty Crops\\nVaccines\\nWeather\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nShare with us\\n\\n\\n \\n\\nBody\\n\\n\\n\\n\\xa0\\n\\n\\xa0\\n\\xa0\\n\\n\\n\\xa0\\n\\n\\nView this profile on Instagram\\n\\n\\xa0\\n\\n\\n\\xa0\\n\\xa0\\n\\xa0\\n\\n\\n\\xa0\\n\\xa0\\n\\n\\n\\xa0\\n\\xa0\\n\\xa0\\n\\n\\n\\n\\xa0\\n\\xa0\\n\\nIllinois Extension (@ilextension) • Instagram photos and videos\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\nBody\\n\\xa0\\n\\nUniversity of Illinois Extension\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\nVolunteer with Extension\\nLearn Something New\\nRead a Blog\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIllinois Extension\\n\\n 101 Mumford Hall (MC-710)\\n \\n1301 W\", 'url': 'https://extension.illinois.edu/'}, {'base_url': 'https://extension.illinois.edu/newsletters/illinois-pesticide-review-newsletter/julyaugust-2023', 'course_name ': 'cropwizard', 'pagenumber': '', 'readable_filename': 'EPA releases draft herbicide strategy', 's3_path': 'courses/cropwizard/EPA_releases_draft_herbicide_strategy.html', 'text': '. The draft framework describes how EPA would apply the mitigations in the Strategy compared to mitigations in the other initiatives.\\nWhat can you do? Submit comments! Learn more!\\nThe draft herbicide framework and accompanying documents are available in docket EPA-HQ-OPP-2023-0365 for public comment for 60 days.\\xa0Comments are due September 22, 2023. Agricultural pesticide users are encouraged to learn about EPA’s plan and to start thinking about how these mitigation measures could apply to herbicide use in their operation. While extensive recordkeeping is not currently required for the mitigation factors described in the strategy, it is highly recommended that users begin thinking about how to incorporate these new elements into their current record systems. If you are applying according to label directions, proper records can only assist your defense should you need it. To help guide you, watch for shared comments from professional organizations such as the Weed Science Society of America (WSSA). In April, a WSSA press release linked their comments to EPA and encouraged growers to act now to understand the impact of ESA’s new compliance initiatives. One good suggestion they offered to growers is to learn how to use EPA’s Bulletins Live! Two which is where important application instructions will be found.\\nEPA’s Office of Pesticide Programs will present a webinar on this draft herbicide Strategy on August 10th at Noon Central Time. EPA plans to walk through the framework and take questions from grower groups and other stakeholders. Register today. Questions may be submitted in advance of the webinar by emailing sm.opmp.pesticides@usda.gov.\\nTo learn more about EPA’s comprehensive ESA workplan Check out our article, “Change Coming to How EPA Protects Endangered Species from Pesticides – Feedback Needed” in the November/December 2022 issue of this newsletter. Proposed mitigation measures are discussed in more general terms in this comprehensive workplan. Please note that the comment period discussed there has ended.\\nVisit EPA’s website to learn more about how EPA’s pesticide program is protecting endangered species.\\nAdapted slightly from an EPA press release, “EPA Releases Draft Strategy to Better Protect Endangered Species from Herbicide Use” and related EPA documents. \\nABOUT THE AUTHOR: Michelle Wiesbrook\\xa0provides subject matter expertise and training in pesticide safety with an emphasis on horticultural weed science. She serves as the Illinois Pesticide Review newsletter editor, collecting and organizing material; and co-coordinates social media information for the PSEP program and ensures its timely publication.\\n\\nPesticide News\\n\\n\\n\\n\\nKeywords\\n\\nPesticide\\nHerbicide\\nInsecticide\\nFungicide\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nRelated Content\\n\\nUnintended herbicide injury on trees: A growing concernAugust 28, 2023\\n\\nTips to help employees succeedAugust 2, 2023\\n\\nParaquat certification valid 3 years: Are you due for training?August 2, 2023\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nIllinois Extension\\n\\n 101 Mumford Hall (MC-710)\\n \\n1301 W. Gregory Dr.\\n Urbana,\\n IL\\n 61801\\n \\nEmail: extension@illinois.edu\\n\\n\\n\\nInstagram\\nFacebook\\nTwitter\\nYouTube\\nLinkedIn\\n\\n\\n\\nCollege of Agricultural, Consumer & Environmental Sciences\\n\\n\\n\\n\\n\\nDig Deeper\\n\\n\\nTake an Online Course\\n\\n\\nRead a Blog\\n\\n\\nRead a Newsletter\\n\\n\\nListen to a Podcast\\n\\n\\nWatch a Video\\n\\n\\nBuy a Publication\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nExtension Network\\n\\n\\nEat.Move.Save.\\n\\n\\nIllinois 4-H\\n\\n\\nIllini Science Policy Program\\n\\n\\nIllinois Indiana Sea Grant\\n\\n\\nIllinois Master Gardeners\\n\\n\\nIllinois Master Naturalists\\n\\n\\nIllinois Nutrition Education Programs\\n\\n\\nPesticide Safety Education Program\\n\\n\\nResearch Centers\\n\\n\\nSafe Electricity\\n\\n\\nU of I Plant Clinic\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nAdditional links\\n\\nAbout Cookies\\nPrivacy Policy\\n© 2023 University of Illinois Board of Trustees\\nEEO\\nAccessibility\\nmyExtension\\nLogin', 'url': 'https://extension.illinois.edu/blogs/pesticide-news/2023-08-02-epa-releases-draft-herbicide-strategy-public-comment-period-open'}, {'base_url': 'https://extension.illinois.edu/newsletters/illinois-pesticide-review-newsletter/julyaugust-2023', 'course_name ': 'cropwizard', 'pagenumber': '', 'readable_filename': 'EPA releases draft herbicide strategy', 's3_path': 'courses/cropwizard/EPA_releases_draft_herbicide_strategy.html', 'text': \". The Strategy — which is primarily designed to provide early mitigations that minimize impacts to over 900 listed species — is one of EPA’s most significant proposals to help overcome these challenges.\\nEPA focused the Strategy on agricultural crop uses in the lower 48 states because hundreds of millions of pounds of herbicides (and plant growth regulators) are applied each year, which is substantially more than for non-agricultural uses of herbicides and for other pesticide classes (e.g., insecticides, fungicides). Additionally, hundreds of listed species in the lower 48 states live in habitats adjacent to agricultural areas. The proposed mitigations in the Strategy would address the most common ways that conventional agricultural herbicides might impact these listed species. More specifically, EPA developed potential mitigation options for conventional agricultural herbicides to reduce pesticide transport via spray drift and runoff/erosion that could result in exposure to listed plants and listed animals that depend on plants.\\nEPA expects that the Strategy will increase the efficiency of future ESA consultations on herbicides with the U.S. Fish and Wildlife Service (FWS), which has authority over most listed species that could benefit from the proposed mitigations. Under the Strategy, EPA proposes to identify and begin mitigating for potential impacts even before EPA completes ESA consultations. These early mitigations should expedite EPA’s ability to fully comply with the ESA by reducing impacts to listed species before EPA conducts most of its ESA analysis. Adopting mitigations earlier will also allow EPA and FWS to use their resources more efficiently in ESA consultations.\\nThe Strategy’s proposed mitigations to reduce spray drift, runoff, and erosion and thereby reduce the potential exposure reflect practices that can be readily implemented by growers and identified by pesticide applicators and that provide flexibility for growers to select the mitigations that work best for them. The Strategy also gives credit to landowners who are already implementing certain measures to reduce pesticide runoff. For example, existing vegetated ditches and water retention ponds will qualify for credits that reduce the need for additional mitigation. Similarly, the Strategy would require less mitigation on flat lands, which are less prone to runoff, and in many western states, which typically experience less rain to carry pesticides off fields. The Strategy also describes how the Agency could add other mitigation practices to the menu of mitigation options in the future, particularly to incorporate emerging technology or new information on the effectiveness of specific practices.\\nDraft Herbicide Framework Document\\nThe draft framework document titled, “Draft Herbicide Strategy Framework to Reduce Exposure of Federally Listed Endangered and Threatened Species and Designated Critical Habitats from the Use of Conventional Agricultural Herbicides” is 97 pages long and includes a discussion of both the proposed scope of the Herbicide Strategy and the proposed decision framework to determine the level of mitigation needed for a particular conventional agricultural herbicide. The draft framework document also includes examples of how the proposed herbicide mitigation would apply to some of the herbicides for which EPA has conducted case studies as well as EPA's proposed implementation plan.\\nSome of the accompanying documents are quite lengthy. The “Herbicide Strategy Case Study Summary and Process” is 666 pages!\\xa0 Coincidence on the number? I’m not sure. I haven’t made it through it all yet. The primary thing I gathered from perusing through the spreadsheet files was that managing these complexities must be a nightmare. The document, “Application of EPA’s Draft Herbicide Strategy Framework Through Scenarios that Represent Crop Production Systems” is only 17 pages long and includes possible scenarios. Examples 1 and 2 would be particularly fitting for Illinois corn and soybean producers. These are shared to help producers better understand how these mitigation practices may be used.\\nIn its ESA Workplan and ESA Workplan Update, EPA outlined this and other ESA initiatives to develop early mitigations that provide listed species with practical protections from pesticides. The Strategy complements those other initiatives, such as targeted mitigations for listed species particularly vulnerable to pesticides and Interim Ecological Mitigations that EPA has begun incorporating under the Federal Insecticide, Fungicide, and Rodenticide Act. The draft framework describes how EPA would apply the mitigations in the Strategy compared to mitigations in the other initiatives.\\nWhat can you do? Submit comments! Learn more!\\nThe draft herbicide framework and accompanying documents are available in docket EPA-HQ-OPP-2023-0365 for public comment for 60 days.\\xa0Comments are due September 22, 2023. Agricultural pesticide users are encouraged to learn about EPA’s plan and to start thinking about how these mitigation measures could apply to herbicide use in their operation. While extensive recordkeeping is not currently required for the mitigation factors described in the strategy, it is highly recommended that users begin thinking about how to incorporate these new elements into their current record systems\", 'url': 'https://extension.illinois.edu/blogs/pesticide-news/2023-08-02-epa-releases-draft-herbicide-strategy-public-comment-period-open'}, {'base_url': 'https://extension.illinois.edu/newsletters/illinois-pesticide-review-newsletter/julyaugust-2023', 'course_name ': 'cropwizard', 'pagenumber': '', 'readable_filename': 'EPA releases draft herbicide strategy', 's3_path': 'courses/cropwizard/EPA_releases_draft_herbicide_strategy.html', 'text': 'EPA releases draft herbicide strategy; public comment period open | Illinois Extension | UIUC\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n Skip to main content\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\nCollege of Agricultural, Consumer & Environmental Sciences\\n\\nIllinois Extension\\n\\n\\n\\n\\n\\nGive\\nVolunteer\\nCareers\\n\\n\\n\\n\\nTopics\\n\\n\\nAll Topics\\n\\n\\nCottage Food\\n\\n\\nFood\\n\\n\\nForestry\\n\\n\\nLocal Government Education\\n\\n\\nPlants\\n\\n\\nRainfall Management\\n\\n\\nSoil\\n\\n\\nVaccines\\n\\n\\nVegetable Gardening\\n\\n\\n\\n\\nLearn\\n\\n\\nOnline Courses\\n\\n\\nBlogs\\n\\n\\nNewsletters\\n\\n\\nPodcasts\\n\\n\\nVideos\\n\\n\\nPublications\\n\\n\\nSummer Resources\\n\\n\\n\\n\\nEvents\\n\\n\\nStatewide Webinars\\n\\n\\n\\n\\nNews\\n\\n\\nConnect\\n\\n\\nContact Staff\\n\\n\\nFind an Office\\n\\n\\nSocial Media\\n\\n\\nAdministration and Educator Teams\\n\\n\\nCommunications and Information Technology\\n\\n\\nIllini Science Policy Program\\n\\n\\nIllinois Indiana Sea Grant\\n\\n\\nMaster Gardeners\\n\\n\\nMaster Naturalists\\n\\n\\nPlant Clinic\\n\\n\\nResearch and Education Centers\\n\\n\\nSea Grant\\n\\n\\nEnergy Education Council\\n\\n\\nHome and Community Education\\n\\n\\nPlanning, Reporting, and Evaluation\\n\\n\\n\\n\\nImpact\\n\\n\\n2024 Extension Collaboration Grants\\n\\n\\nEconomic and Functional Impact\\n\\n\\nOur Impact in Agriculture and AgriBusiness\\n\\n\\nSNAP-Education Impact\\n\\n\\nExtension Funded Research Projects\\n\\n\\nOur Impact in Agriculture and Natural Resources\\n\\n\\nOur Impact in Community & Economic Development\\n\\n\\nOur Impact in Family and Consumer Sciences\\n\\n\\nOur Impact in Integrated Health Disparities\\n\\n\\n\\n\\nAbout\\n\\n\\nStrategic Planning\\n\\n\\nExtension Councils\\n\\n\\nCareers\\n\\n\\nProfessional Associations\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nBreadcrumb\\n\\n\\nHome\\n\\n\\nBlogs\\n\\n\\nPesticide News\\n\\n\\n EPA releases draft herbicide strategy; public comment period open \\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nPesticide News\\n\\n\\nEPA releases draft herbicide strategy; public comment period open \\n\\n\\n\\n\\n\\n\\nAugust 2, 2023\\n\\n\\n\\nMichelle Wiesbrook\\n\\n\\n \\n\\nStrategy aims to increase efficiencies while supporting farmers, herbicide users with continued use of important pesticide tools. \\xa0\\nThe U.S. Environmental Protection Agency (EPA) released the draft Herbicide Strategy for public comment, a major milestone in the Agency’s work to protect federally endangered and threatened (listed) species from conventional agricultural herbicides. The Strategy describes proposed early mitigations for more than 900 listed species and designated critical habitats to reduce potential impacts from the agricultural use of these herbicides while helping to ensure the continued availability of these important pesticide tools.\\n“Ensuring safe use of herbicides is an important part of EPA’s mission to protect the environment,” said Deputy Assistant Administrator for Pesticide Programs for the Office of Chemical Safety and Pollution Prevention Jake Li. “This strategy reflects one of our biggest steps to support farmers and other herbicide users with tools for managing weeds, while accelerating EPA’s ability to protect many endangered species that live near agricultural areas.”\\nThe Strategy is part of EPA’s ongoing efforts to develop a multichemical, multispecies approach toward meeting its obligations under the Endangered Species Act (ESA). EPA’s traditional chemical-by-chemical, species-by-species approach to meeting these obligations is slow and costly.\\xa0 As a result, EPA has completed its ESA obligations for less than 5% of its actions, creating legal vulnerabilities for the Agency, increased litigation, and uncertainty for farmers and other pesticide users about their continued ability to use many pesticides. The Strategy — which is primarily designed to provide early mitigations that minimize impacts to over 900 listed species — is one of EPA’s most significant proposals to help overcome these challenges.\\nEPA focused the Strategy on agricultural crop uses in the lower 48 states because hundreds of millions of pounds of herbicides (and plant growth regulators) are applied each year, which is substantially more than for non-agricultural uses of herbicides and for other pesticide classes (e.g., insecticides, fungicides). Additionally, hundreds of listed species in the lower 48 states live in habitats adjacent to agricultural areas. The proposed mitigations in the Strategy would address the most common ways that conventional agricultural herbicides might impact these listed species', 'url': 'https://extension.illinois.edu/blogs/pesticide-news/2023-08-02-epa-releases-draft-herbicide-strategy-public-comment-period-open'}, {'base_url': 'https://extension.illinois.edu/newsletters/illinois-pesticide-review-newsletter/julyaugust-2023', 'course_name ': 'cropwizard', 'pagenumber': '', 'readable_filename': 'News', 's3_path': 'courses/cropwizard/News.html', 'text': \". — The U.S. Environmental Protection Agency, in collaboration with the U.S. Department of Energy, is funding the creation of Thriving Communities Technical Assistance Centers to help rural and underserved communities access state and federal funding for environmental infrastructure…\\n\\n\\n \\nAddress health challenges with holistic solutions\\nSeptember 1, 2023\\n\\nURBANA, Ill. — The University of Illinois, along with the Interdisciplinary Health Sciences Institute, and in collaboration with Illinois Extension, has developed the Autumn Health Picks 2023 webinar series. This series is part of the Community Seminar Series, and it provides an opportunity for…\\n\\n\\n \\nDo artificial roosts help bats? Illinois experts say more research needed\\nSeptember 1, 2023\\n\\nURBANA, Ill.\\xa0— Artificial roosts for bats come in many forms — bat boxes, condos, bark mimics, clay roosts, and cinder block structures, to name a few — but a new conservation practice and policy article from researchers at the\\xa0University of Illinois Urbana-Champaign\\xa0suggests the structures…\\n\\n\\nMore news\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\nBlock Reference\\n\\nNews\\n\\n\\n\\nBy topic\\n- Any -4-H4-H alumni4-H FoundationAccessibilityAdministrationAgingAgricultural safetyAgricultureAnimal scienceAnnie's ProjectAnnualsAutismBeefBeef CattleBeekeepingBeesBeneficial InsectsBirdsBrain healthBulbsBullyingBusiness developmentBusiness retention and expansionButterflyCampingCaregivingCensusCertified Crop AdvisorCertified Livestock ManagerChild nutritionChronic diseasesCitizen ScienceCivic engagementClimateCollege ReadinessCommercial agricultureCommercial Vegetable ProductionCommunicationCommunity developmentCommunity gardenCommunity healthCommunity planningCommunity resiliencyCompostingConservationConsumer economicsCornCover cropsCreditCrop diseaseCropsDairy CattleDebt managementDementia Alzheimer’s diseaseDiabetesDicambaDisaster preparednessDiversity Equity InclusionDowntown developmentDrainageDronesEarly childhoodEconomic developmentEDEN Ready BusinessEFNEPElder careEmergency foodEnergyEnergy conservationEnergy efficiencyEntomologyEntrepreneurshipEnvironmentEstate planningExpensesFacultyFamily lifeFarm business managementFarm safetyFarmers marketsFinancial ExploitationFinancial planningFinancial wellnessFlowersFood accessFood PreservationFood safetyFood sanitationForestryFruitsFungicideGardeningGrassesHayHealthHealth CareHealthy cookingHealthy eatingHempHerbicideHerbsHolidaysHome OwnershipHorticultureHouseplantsIdentity TheftInclusionINEPInformation TechnologyInsect PestsInsecticideInsects and pestsInsuranceIntegrated Health DisparitiesIntegrated pest managementInvasive speciesInvestingLandscape architectureLandscape designLawn careLeadershipLeadership developmentLife skillsLivestockLocal foods and small farmsLocal governmentManaging stressManure managementMarketingMaster GardenersMaster NaturalistMeeting ManagementMental healthMindfulnessMoney MentorsMyPINative plantsNavigating differenceNutritionNutrition educationObesity preventionOrnamentalOutdoor SkillsParentingPasturePerennialsPesticidePesticide LabelPhysical ActivityPlant ClinicPlant diseasePlant health carePollinator HabitatPondsPoultryPoverty simulationPrivate/Commercial ApplicatorProfessional Development CEU CPDUPSEP trainingReal ColorsRecyclingRelationshipsResilienceRoboticsRosesSafetyShooting sportsShrubsSmall farmsSmart MeterSNAP-EdSocial-emotional healthSoilSoybeansSpecialty CropsSpendingState 4-H OfficeSTEMSubstance UseSustainable agricultureSwineTaxesTeam buildingTeenagersTime managementTrauma informed Adverse Childhood ExperiencesTree fruitsTreesTurfUrban AgricultureUrban gardeningVegetable gardeningVegetablesVolunteersWaterWeatherWeedsWellnessWheatWhole grainsWildlifeWorkforce developmentWorkplace wellnessYouth and MoneyYouth development\\n\\n\\nSearch\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\nSeptember 6, 2023\\n\\nIllinois Extension selected to establish environmental assistance center to help Illinois communities\\n\\n \\n URBANA, Ill. — The U.S. Environmental Protection Agency, in collaboration with the U.S. Department of Energy, is funding the creation of Thriving Communities Technical Assistance Centers to help rural and underserved communities access state and federal funding for environmental...\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\nSeptember 1, 2023\\n\\nAddress health challenges with holistic solutions\\n\\n \\n URBANA, Ill\", 'url': 'https://extension.illinois.edu/global/news-releases'}, {'base_url': 'https://extension.illinois.edu/newsletters/illinois-pesticide-review-newsletter/julyaugust-2023', 'course_name ': 'cropwizard', 'pagenumber': 4, 'readable_filename': 'extension.pdf', 's3_path': 'courses/cropwizard/extension.pdf', 'text': \"Illinois Pesticide Review \\nVolume 36, No. 4, July/August \\n4\\nproposed mitigations. Under the Strategy, EPA \\nproposes to identify and begin mitigating for \\npotential impacts even before EPA completes \\nESA consultations. These early mitigations \\nshould expedite EPAs ability to fully comply \\nwith the ESA by reducing impacts to listed \\nspecies before EPA conducts most of its ESA \\nanalysis. Adopting mitigations earlier will \\nalso allow EPA and FWS to use their resources \\nmore efficiently in ESA consultations.\\nThe Strategys proposed mitigations to reduce \\nspray drift, runoff, and erosion and thereby \\nreduce the potential exposure reflect practices \\nthat can be readily implemented by growers \\nand identified by pesticide applicators and \\nthat provide flexibility for growers to select \\nthe mitigations that work best for them. The \\nStrategy also gives credit to landowners who \\nare already implementing certain measures to \\nreduce pesticide runoff. For example, existing \\nvegetated ditches and water retention ponds \\nwill qualify for credits that reduce the need for \\nadditional mitigation. Similarly, the Strategy \\nwould require less mitigation on flat lands, \\nwhich are less prone to runoff, and in many \\nwestern states, which typically experience \\nless rain to carry pesticides off fields. The \\nStrategy also describes how the Agency could \\nadd other mitigation practices to the menu of \\nmitigation options in the future, particularly \\nto incorporate emerging technology or new \\ninformation on the effectiveness of specific \\npractices.\\nDraft Herbicide Framework \\nDocument\\nThe draft framework document titled, Draft \\nHerbicide Strategy Framework to Reduce \\nExposure of Federally Listed Endangered \\nand Threatened Species and Designated \\nCritical Habitats from the Use of Conventional \\nAgricultural Herbicides is 97 pages long and \\nincludes a discussion of both the proposed \\nscope of the Herbicide Strategy and the \\nproposed decision framework to determine \\nthe level of mitigation needed for a particular \\nconventional agricultural herbicide. The draft \\nframework document also includes examples \\nof how the proposed herbicide mitigation \\nwould apply to some of the herbicides for \\nwhich EPA has conducted case studies as well \\nas EPA's proposed implementation plan.\\nSome of the accompanying documents are \\nquite lengthy. The Herbicide Strategy Case \\nStudy Summary and Process is 666 pages! \\nCoincidence on the number? Im not sure. I \\nhavent made it through it all yet. The primary \\nthing I gathered from perusing through \\nthe spreadsheet files was that managing \\nthese complexities must be a nightmare. \\nThe document, Application of EPAs Draft \\nHerbicide Strategy Framework Through \\nScenarios that Represent Crop Production \\nSystems is only 17 pages long and includes \\npossible scenarios. Examples 1 and 2 would \\nbe particularly fitting for Illinois corn and \\nsoybean producers. These are shared to \\nhelp producers better understand how these \\nmitigation practices may be used. \\nIn its ESA Workplan and ESA Workplan \\nUpdate, EPA outlined this and other ESA \\ninitiatives to develop early mitigations \\nthat provide listed species with practical \\nprotections from pesticides. The Strategy \\ncomplements those other initiatives, such \\nas targeted mitigations for listed species \\nparticularly vulnerable to pesticides and \\nInterim Ecological Mitigations that EPA \\nhas begun incorporating under the Federal \\nInsecticide, Fungicide, and Rodenticide Act. \\nThe draft framework describes how EPA would \\napply the mitigations in the Strategy compared \\nto mitigations in the other initiatives. \\nWhat can you do? Submit \\ncomments! Learn more!\\nThe draft herbicide framework and \\naccompanying documents are available in \\ndocket EPA-HQ-OPP-2023-0365 for public \\ncomment for 60 days. Comments are due \\nSeptember 22, 2023. Agricultural pesticide \\nusers are encouraged to learn about EPAs\", 'url': 'https://extension.illinois.edu/sites/default/files/2023-08/IPR%20Volume%2036%20Issue%204%20July%20August%20SECURE.pdf'}, {'base_url': 'https://extension.illinois.edu/newsletters/illinois-pesticide-review-newsletter/julyaugust-2023', 'course_name ': 'cropwizard', 'pagenumber': 3, 'readable_filename': 'extension.pdf', 's3_path': 'courses/cropwizard/extension.pdf', 'text': 'Illinois Pesticide Review \\nVolume 36, No. 4, July/August \\n3\\nIts important to consider that one should \\ntake the test in the language he or she is \\nmost comfortable with. If someone has \\nbeen studying the material in English, dont \\nbe surprised if they opt to take the exam in \\nEnglish too. \\nIn the end, it all comes down to good \\ncommunication between you and your \\nemployees. It could be that they dont know \\nyet which learning method would work best \\nfor them and theyll need to try a few things. \\nTheyll appreciate you taking the time to ask \\nthem and work with them to help ensure their \\nsuccess.\\nMichelle Wiesbrook \\nEPA Releases Draft \\nHerbicide Strategy, Public \\nComment Period Open \\nStrategy aims to increase \\nefficiencies while supporting \\nfarmers, herbicide users with \\ncontinued use of important \\npesticide tools \\nThe U.S. Environmental Protection Agency \\n(EPA) released the draft Herbicide Strategy \\nfor public comment, a major milestone in the \\nAgencys work to protect federally endangered \\nand threatened (listed) species from conven-\\ntional agricultural herbicides. The Strategy \\ndescribes proposed early mitigations for more \\nthan 900 listed species and designated criti-\\ncal habitats to reduce potential impacts from \\nthe agricultural use of these herbicides while \\nhelping to ensure the continued availability of \\nthese important pesticide tools.\\nEnsuring safe use of herbicides is an \\nimportant part of EPAs mission to protect \\nthe environment, said Deputy Assistant \\nAdministrator for Pesticide Programs for \\nthe Office of Chemical Safety and Pollution \\nPrevention Jake Li. This strategy reflects one \\nof our biggest steps to support farmers and \\nother herbicide users with tools for managing \\nweeds, while accelerating EPAs ability to \\nprotect many endangered species that live near \\nagricultural areas.\\nThe Strategy is part of EPAs ongoing efforts \\nto develop a multichemical, multispecies \\napproach toward meeting its obligations \\nunder the Endangered Species Act (ESA). \\nEPAs traditional chemical-by-chemical, \\nspecies-by-species approach to meeting these \\nobligations is slow and costly. As a result, EPA \\nhas completed its ESA obligations for less than \\n5% of its actions, creating legal vulnerabilities \\nfor the Agency, increased litigation, and \\nuncertainty for farmers and other pesticide \\nusers about their continued ability to use many \\npesticides. The Strategy which is primarily \\ndesigned to provide early mitigations that \\nminimize impacts to over 900 listed species \\nis one of EPAs most significant proposals to \\nhelp overcome these challenges.\\nEPA focused the Strategy on agricultural crop \\nuses in the lower 48 states because hundreds \\nof millions of pounds of herbicides (and plant \\ngrowth regulators) are applied each year, \\nwhich is substantially more than for non-\\nagricultural uses of herbicides and for other \\npesticide classes (e.g., insecticides, fungicides). \\nAdditionally, hundreds of listed species in \\nthe lower 48 states live in habitats adjacent to \\nagricultural areas. The proposed mitigations \\nin the Strategy would address the most \\ncommon ways that conventional agricultural \\nherbicides might impact these listed \\nspecies. More specifically, EPA developed \\npotential mitigation options for conventional \\nagricultural herbicides to reduce pesticide \\ntransport via spray drift and runoff/erosion \\nthat could result in exposure to listed plants \\nand listed animals that depend on plants.\\nEPA expects that the Strategy will increase \\nthe efficiency of future ESA consultations \\non herbicides with the U.S. Fish and Wildlife \\nService (FWS), which has authority over most \\nlisted species that could benefit from the', 'url': 'https://extension.illinois.edu/sites/default/files/2023-08/IPR%20Volume%2036%20Issue%204%20July%20August%20SECURE.pdf'}, {'base_url': 'https://extension.illinois.edu/newsletters/illinois-pesticide-review-newsletter/julyaugust-2023', 'course_name ': 'cropwizard', 'pagenumber': '', 'readable_filename': 'News', 's3_path': 'courses/cropwizard/News.html', 'text': \". — The U.S. Environmental Protection Agency, in collaboration with the U.S. Department of Energy, is funding the creation of Thriving Communities Technical Assistance Centers to help rural and underserved communities access state and federal funding for environmental...\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\nSeptember 1, 2023\\n\\nAddress health challenges with holistic solutions\\n\\n \\n URBANA, Ill. — The University of Illinois, along with the Interdisciplinary Health Sciences Institute, and in collaboration with Illinois Extension, has developed...\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\nSeptember 1, 2023\\n\\nDo artificial roosts help bats? Illinois experts say more research needed\\n\\n \\n URBANA, Ill.\\xa0— Artificial roosts for bats come in many forms — bat boxes, condos, bark mimics, clay roosts, and cinder block structures, to name a few — but a new conservation practice and policy article from researchers at the\\xa0University of...\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\nAugust 29, 2023\\n\\nButterflies can eat to live, live to eat in a balanced garden\\n\\n \\n URBANA, Ill. — A favorite thing about visiting gardens in the summer is catching sight of a butterfly enjoying nectar from a brightly colored zinnia or a monarch caterpillar munching on a milkweed leaf. When designing a butterfly garden, expand and balance plant selection to provide more than...\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\nAugust 24, 2023\\n\\nField Day event plans to keep beef cattle producers up on trends for their herds\\n\\n \\n URBANA, Ill. — Beef cattle producers will gain insights and stay up to date on current research from cow/calf patterns to alternative forages and more at the Orr Beef Research Center's Field Day on September 6.\\xa0The meeting will be held at the John Wood Community College Ag Center located west of...\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\nAugust 23, 2023\\n\\nBuild drought-tolerant gardens this fall for next summer’s blooms\\n\\n \\n URBANA, Ill. — Many Illinois gardens are entering the fall stressed from the lack of summer rains combined with scorching hot temperatures. These conditions easily stress some plants; however, many plants quickly adapt to hot, dry conditions. Drought-tolerant plants are not only tough and...\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\nAugust 23, 2023\\n\\nIllinois Extension exhibits research, programs, innovation at 2023 Farm Progress Show\\n\\n \\n DECATUR, Ill. — The Farm Progress Show returns to Decatur, Aug. 29-31, and\\xa0University of Illinois Extension will be on-site in the College of...\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\nAugust 21, 2023\\n\\nBuild privacy with plants for secret gardens\\n\\n \\n URBANA, Ill.\\xa0— Plants serve a lot of purposes in the landscape. One of which is to add some privacy. Screening plants can help define and give purpose to a space. Homeowners may wish to screen a particular area or transparency in the landscape, creating interest in what lies beyond.\\xa0\\n\\n...\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\nAugust 21, 2023\\n\\nIllinois Extension investing in the future of farming across the state\\n\\n \\n URBANA, Ill. — Helping Illinois farmers grow thriving crops and livestock has always been at the heart of University of Illinois Extension’s mission. Using feedback received from farmers and other agricultural stakeholders through a 2022 survey,...\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\nAugust 21, 2023\\n\\nExtraordinary 4-H volunteers honored\\n\\n \\n SPRINGFIELD, Ill\", 'url': 'https://extension.illinois.edu/global/news-releases'}, {'base_url': 'https://extension.illinois.edu/newsletters/illinois-pesticide-review-newsletter/julyaugust-2023', 'course_name ': 'cropwizard', 'pagenumber': '', 'readable_filename': 'Extension Funded Research Projects', 's3_path': 'courses/cropwizard/Extension_Funded_Research_Projects.html', 'text': '. Today’s most pressing issues are related to climate change. An understanding of how human activity impacts the environment helps us make sense of how a zoonotic transfer of a virus to humans causes a global pandemic, and how rising temperatures increase the frequency and intensity of wildfires and storms. Environmental Education (EE) connects people to their environments, instilling values and knowledge that leads to better environmental stewardship of local environments and natural resources. There are several well-established EE programs offered throughout Cook County by Illinois\\xa0Extension and the Forest Preserve of Cook County (FPCC). Missing from these opportunities are programs available to middle school-aged children, the age best suited for EE experiences because their environmental sensitivities are still developing and early experiences in nature have been shown to have impacts throughout adulthood (Boom, 2017). This proposal seeks to develop a program, Illinois Inquiry Adventures in Nature (IIAN), for middle school children and their families, suitable for small groups during the pandemic\\xa0and expanding in scope to include classrooms when safe. A series of four seasonal activities\\xa0and teacher workshops\\xa0will be created to bring groups to their local green spaces, including FPCC sites. Groups will engage in open-ended investigations based on their own observations and questions, complete activities at home\\xa0and enact local community conservation projects. Research will be conducted to examine how individuals’ connections to nature and environmental stewardship change over the course of their participation. This program fills a local need in Cook County, creating a continuum of opportunities across ages, and will be made available to all residents in Illinois, and nationwide, encouraging the next generation of environmental leaders.\\n\\n\\nAssessing the Needs and Connecting Young & Beginning Farmers with Extension Resources in Northern Illinois\\nAwarded to: Illinois Extension in the College of ACES\\nPrincipal Investigator: Joseph Malual\\nCo-Investigators:\\nNikki Keltner, Extension program coordinator, Illinois Extension\\nGrant McCarty, Extension educator, Illinois Extension\\nHope Michelson, assistant professor,\\xa0Department of Agricultural & Consumer Economics\\nPROJECT SUMMARY\\nMore and more young people are engaging in small-scale farming, with many focusing on specialty crops and sustainable agricultural production. Despite this trend, entry into farming, which is a complex business, is challenging. Beginning farmers face serious obstacles in accessing critical assets, including startup capital to acquire land, farm equipment\\xa0and agricultural technical knowledge needed to develop a\\xa0successful agricultural practice and profitable business. The situation is complicated by lack of adequate research to understand the unique challenges facing this generation of farmers. In Illinois, there is limited research to understand how people new to farming navigate access to critical resources. This research project aims to provide a comprehensive assessment of the needs and opportunities facing young and beginning\\xa0farmers in northern Illinois. We will identify and map farms owned by young and beginning farmers, examine their experiences and strategies used to leverage critical startup assets, including farmland and equipment, financial capital\\xa0and agricultural technical assistance, as well as strategies for marketing agricultural products. This project will build relations and connect this new audience with Extension resources, which can help\\xa0beginning farmers develop the knowledge and skills necessary for solving critical problems. Through interdisciplinary collaboration between Extension educators and specialists with faculty at the University of Illinois at Urbana-Champaign, this research will generate useful knowledge that can help beginning farmers, businesses\\xa0and communities make informed decisions and plan for future support of those new to farming. The\\xa0knowledge and practices discovered and identified through this project will be shared with Extension across the state. Extension educators can build on this knowledge to plan and deliver educational programming that empowers farmers to develop financially viable and sustainable farms. Those successful endeavors will, in turn, help to revitalize their rural communities.\\n\\n\\nNew Immigrant Foodways\\nAwarded to: Department of History in the College of Liberal Arts and Sciences\\nPrincipal Investigator: Teresa Barnes\\nCo-Investigators:\\nGisela Sin, director, Center for Latin American and Caribbean Studies\\nMargarita Teran-Garcia, Extension specialist, Illinois Extension\\nPROJECT SUMMARY\\nThis project will leverage new and existing research with immigrant communities about challenges and strategies in adapting home foodways to American food systems to create short instructional videos related to nutrition and cooking. The project addresses a complex issue at the intersection of three critical areas of Extension’s mission: food, health\\xa0and environment. It addresses the public need of new immigrant families to access information and expertise and develop sustainable strategies when faced with the bewildering array of often unhealthy food options in the USA', 'url': 'https://extension.illinois.edu/global/extension-funded-research-projects'}]}\n" + ] + } + ], + "source": [ + "print(course_df[0]['messages'][1])" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "# user email is in DF, outside of convo" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Mapping Conversations to Nomic" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "22\n", + "22\n" + ] + } + ], + "source": [ + "user_queries = []\n", + "metadata = []\n", + "i = 1\n", + "\n", + "# log conversation instead of individual messages\n", + "for index, row in df.iterrows():\n", + " user_email = row['user_email']\n", + " convo = row['convo']\n", + " messages = convo['messages']\n", + " first_message = messages[0]['content']\n", + " user_queries.append(first_message)\n", + " # create metadata for multi-turn conversation\n", + " conversation = \"\"\n", + " for message in messages:\n", + " # string of role: content, role: content, ...\n", + " conversation += \"\\n>>> \" + message['role'] + \": \" + message['content'] + \"\\n\"\n", + " # add to metadata\n", + " metadata_row = {\"course\": row['course_name'], \"conversation\": conversation, \"conversation_id\": convo['id'], \n", + " \"id\": i, \"user_email\": user_email, \"first_query\": first_message}\n", + " metadata.append(metadata_row)\n", + " i += 1\n", + "\n", + "print(len(user_queries))\n", + "print(len(metadata))" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'course': 'cropwizard', 'conversation': '\\n>>> user: What type of pest is most harmful to corn crops in Illinois and how can I prevent it?\\n\\n>>> assistant: The most harmful pest to corn crops in Illinois is the corn rootworm. Corn rootworms, specifically the western corn rootworm (Diabrotica virgifera virgifera), are the most impactful species in Illinois and the North Central U.S. These pests can cause significant damage to corn plants by feeding on the roots, leading to reduced plant vigor and yield.\\n\\nTo prevent corn rootworm infestations, several measures can be taken:\\n\\n1. Crop rotation: Implementing a crop rotation strategy by alternating corn with non-host crops, such as soybeans, can help reduce corn rootworm populations. The larvae are not capable of surviving on soybean or other broadleaf plants, so rotating crops disrupts their life cycle.\\n\\n2. Utilize Bt hybrids: Planting corn hybrids that have been genetically modified to express Bacillus thuringiensis (Bt) proteins targeting corn rootworm can provide effective control. These Bt hybrids produce toxins that are toxic to the pests, reducing their feeding and damage to corn plants.\\n\\n3. Insecticide treatments: Seed-applied insecticides containing neonicotinoid or diamide insecticides can be used to protect corn seedlings from early-season pests, including corn rootworms. Soil-applied insecticides can also be applied at planting to control corn rootworms and other early-season pests.\\n\\n4. Monitoring and scouting: Regular monitoring of corn fields for signs of corn rootworm damage, such as root feeding and lodging, is essential. Scouting helps determine if pest populations have reached economic thresholds that require action, such as insecticide application.\\n\\n5. Integrated Pest Management (IPM): Implementing an IPM approach involves combining multiple pest management strategies, including cultural practices, biological control, and chemical control, to minimize the impact of corn rootworms and other pests. This holistic approach aims to reduce reliance on chemical treatments and promote sustainable pest management.\\n\\nIt is important to consult with local agricultural extension services, seed dealers, or agricultural experts for specific recommendations tailored to your region and field conditions to effectively manage corn rootworm infestations.\\n', 'conversation_id': '3e5d4861-b128-4c64-96ac-87c74f3217e5', 'id': 2, 'user_email': 'avd6@illinois.edu', 'first_query': 'What type of pest is most harmful to corn crops in Illinois and how can I prevent it?'}\n" + ] + } + ], + "source": [ + "print(metadata[1])" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(22, 1536)\n" + ] + } + ], + "source": [ + "# initialize langchain OpenAI embeddings model\n", + "embeddings_model = OpenAIEmbeddings()\n", + "metadata = pd.DataFrame(metadata)\n", + "embeddings = embeddings_model.embed_documents(user_queries)\n", + "embeddings = np.array(embeddings)\n", + "print(embeddings.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-09-07 14:53:44.900 | INFO | nomic.project:_create_project:779 - Creating project `Conversation Map for cropwizard` in organization `dabholkar.asmita`\n", + "2023-09-07 14:53:45.794 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-09-07 14:53:45.798 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.07it/s]\n", + "2023-09-07 14:53:46.743 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-09-07 14:53:46.744 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-09-07 14:53:47.568 | INFO | nomic.project:create_index:1111 - Created map `Conversation Map for cropwizard` in project `Conversation Map for cropwizard`: https://atlas.nomic.ai/map/27ba0013-71ba-4fd6-a710-187d114442ab/f74b45b3-7213-45df-a4cb-c16ec9feb8b7\n", + "2023-09-07 14:53:47.570 | INFO | nomic.atlas:map_embeddings:139 - Conversation Map for cropwizard: https://atlas.nomic.ai/map/27ba0013-71ba-4fd6-a710-187d114442ab/f74b45b3-7213-45df-a4cb-c16ec9feb8b7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Conversation Map for cropwizard: https://atlas.nomic.ai/map/27ba0013-71ba-4fd6-a710-187d114442ab/f74b45b3-7213-45df-a4cb-c16ec9feb8b7]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-09-07 14:53:48.872 | INFO | nomic.project:create_index:1111 - Created map `cropwizard_convo_index_2` in project `Conversation Map for cropwizard`: https://atlas.nomic.ai/map/27ba0013-71ba-4fd6-a710-187d114442ab/3336aa7f-5995-4f02-831b-7161fd0c0b71\n" + ] + }, + { + "data": { + "text/html": [ + "Atlas Projection cropwizard_convo_index_2. Status Topic Modeling. view online" + ], + "text/plain": [ + "cropwizard_convo_index_2: https://atlas.nomic.ai/map/27ba0013-71ba-4fd6-a710-187d114442ab/3336aa7f-5995-4f02-831b-7161fd0c0b71" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# create an Atlas project\n", + "project_name = \"Conversation Map for \" + course\n", + "index_name = course + \"_convo_index_2\"\n", + "project = atlas.map_embeddings(embeddings=np.array(embeddings),\n", + " data=metadata,\n", + " id_field='id',\n", + " build_topic_model=True,\n", + " topic_label_field='first_query',\n", + " name=project_name,\n", + " colorable_fields=['conversation_id', 'first_query'])\n", + "print(project.maps)\n", + "\n", + "project.create_index(index_name, build_topic_model=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Mapping Query-Response Pairs to Nomic" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2\n", + "user queries: 1\n", + "metadata 1\n", + "------------------------\n" + ] + } + ], + "source": [ + "user_queries = []\n", + "metadata = []\n", + "i = 1\n", + "for convo in course_df:\n", + " messages = convo['messages']\n", + " print(len(messages))\n", + " pair_check = 0\n", + " for message in messages:\n", + " if message['role'] == 'user' and pair_check == 0:\n", + " query = message['content']\n", + " metadata_row = {'course': course, 'query': message['content']}\n", + " #print(\"metadata row: \", metadata_row)\n", + " pair_check += 1\n", + " if message['role'] == 'assistant' and pair_check == 1:\n", + " metadata_row['response'] = message['content']\n", + " metadata_row['id'] = i \n", + " #print(\"response metadata row: \", metadata_row)\n", + " i += 1\n", + " pair_check += 1\n", + " if pair_check == 2:\n", + " # one conversation pair is complete\n", + " user_queries.append(query)\n", + " metadata.append(metadata_row)\n", + " pair_check = 0\n", + "\n", + " \n", + "print(\"user queries: \", len(user_queries))\n", + "print(\"metadata\", len(metadata))\n", + "print(\"------------------------\")\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(590, 1536)\n" + ] + } + ], + "source": [ + "# initialize langchain OpenAI embeddings model\n", + "embeddings_model = OpenAIEmbeddings()\n", + "metadata = pd.DataFrame(metadata)\n", + "embeddings = embeddings_model.embed_documents(user_queries)\n", + "embeddings = np.array(embeddings)\n", + "print(embeddings.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-31 19:55:40.276 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for ECE408FA23` in organization `dabholkar.asmita`\n", + "2023-08-31 19:55:41.466 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-31 19:55:41.491 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:02, 2.43s/it]\n", + "2023-08-31 19:55:43.932 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-31 19:55:43.932 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-31 19:55:45.475 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for ECE408FA23` in project `Query-Response Map for ECE408FA23`: https://atlas.nomic.ai/map/d8e02f68-501e-4a25-9060-502710c082ce/b55cdca3-3ff4-454e-8f1b-a73ebc22cdff\n", + "2023-08-31 19:55:45.480 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for ECE408FA23: https://atlas.nomic.ai/map/d8e02f68-501e-4a25-9060-502710c082ce/b55cdca3-3ff4-454e-8f1b-a73ebc22cdff\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for ECE408FA23: https://atlas.nomic.ai/map/d8e02f68-501e-4a25-9060-502710c082ce/b55cdca3-3ff4-454e-8f1b-a73ebc22cdff]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-31 19:55:47.813 | INFO | nomic.project:create_index:1111 - Created map `ECE408FA23_qr_index` in project `Query-Response Map for ECE408FA23`: https://atlas.nomic.ai/map/d8e02f68-501e-4a25-9060-502710c082ce/ff7276fc-942f-45cd-a199-e19a6e941db1\n" + ] + }, + { + "data": { + "text/html": [ + "Atlas Projection ECE408FA23_qr_index. Status Topic Modeling. view online" + ], + "text/plain": [ + "ECE408FA23_qr_index: https://atlas.nomic.ai/map/d8e02f68-501e-4a25-9060-502710c082ce/ff7276fc-942f-45cd-a199-e19a6e941db1" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# create an Atlas project\n", + "project_name = \"Query-Response Map for \" + course\n", + "index_name = course + \"_qr_index\"\n", + "project = atlas.map_embeddings(embeddings=np.array(embeddings),\n", + " data=metadata,\n", + " id_field='id',\n", + " build_topic_model=True,\n", + " topic_label_field='query',\n", + " name=project_name,\n", + " colorable_fields=['query'])\n", + "print(project.maps)\n", + "\n", + "project.create_index(index_name, build_topic_model=True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1571\n" + ] + } + ], + "source": [ + "# cell for all course map creation\n", + "\n", + "response = supabase_client.table(\"llm-convo-monitor\").select(\"*\").execute()\n", + "data = response.data\n", + "print(len(data))" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "126" + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = pd.DataFrame(data)\n", + "course_names = df['course_name'].unique()\n", + "len(course_names)" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: badm_550_ashley\n", + "(51, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:26:29.701 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for badm_550_ashley` in organization `dabholkar.asmita`\n", + "2023-08-30 15:26:31.242 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:26:31.255 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.03it/s]\n", + "2023-08-30 15:26:32.239 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:26:32.241 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:26:33.498 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for badm_550_ashley` in project `Query-Response Map for badm_550_ashley`: https://atlas.nomic.ai/map/e7fe1faf-be7a-4ca7-b89b-22863fffe432/79d4480a-c4e5-4920-95cd-ed831d45f5d0\n", + "2023-08-30 15:26:33.500 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for badm_550_ashley: https://atlas.nomic.ai/map/e7fe1faf-be7a-4ca7-b89b-22863fffe432/79d4480a-c4e5-4920-95cd-ed831d45f5d0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for badm_550_ashley: https://atlas.nomic.ai/map/e7fe1faf-be7a-4ca7-b89b-22863fffe432/79d4480a-c4e5-4920-95cd-ed831d45f5d0]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:26:35.272 | INFO | nomic.project:create_index:1111 - Created map `badm_550_ashley_qr_index` in project `Query-Response Map for badm_550_ashley`: https://atlas.nomic.ai/map/e7fe1faf-be7a-4ca7-b89b-22863fffe432/2b997f7c-0084-4db7-8e9a-76eeb62d715b\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: ece120\n", + "(298, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:26:42.765 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for ece120` in organization `dabholkar.asmita`\n", + "2023-08-30 15:26:43.831 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:26:43.850 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:01, 1.20s/it]\n", + "2023-08-30 15:26:45.059 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:26:45.063 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:26:46.221 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for ece120` in project `Query-Response Map for ece120`: https://atlas.nomic.ai/map/f35d2617-d029-4a29-ba4a-27a82088be24/0c8e9404-be65-4780-bba3-e3b44f14de9b\n", + "2023-08-30 15:26:46.230 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for ece120: https://atlas.nomic.ai/map/f35d2617-d029-4a29-ba4a-27a82088be24/0c8e9404-be65-4780-bba3-e3b44f14de9b\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for ece120: https://atlas.nomic.ai/map/f35d2617-d029-4a29-ba4a-27a82088be24/0c8e9404-be65-4780-bba3-e3b44f14de9b]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:26:48.040 | INFO | nomic.project:create_index:1111 - Created map `ece120_qr_index` in project `Query-Response Map for ece120`: https://atlas.nomic.ai/map/f35d2617-d029-4a29-ba4a-27a82088be24/61277149-b12d-4b59-8bcd-e9dd29fc58a4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: badm-567-v3\n", + "(27, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:26:52.367 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for badm-567-v3` in organization `dabholkar.asmita`\n", + "2023-08-30 15:26:53.227 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:26:53.236 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.07it/s]\n", + "2023-08-30 15:26:54.177 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:26:54.185 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:26:55.379 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for badm-567-v3` in project `Query-Response Map for badm-567-v3`: https://atlas.nomic.ai/map/022b4521-24df-401d-beff-bba4f881590f/adabe0a1-d147-462a-8992-72af89cb5ae2\n", + "2023-08-30 15:26:55.379 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for badm-567-v3: https://atlas.nomic.ai/map/022b4521-24df-401d-beff-bba4f881590f/adabe0a1-d147-462a-8992-72af89cb5ae2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for badm-567-v3: https://atlas.nomic.ai/map/022b4521-24df-401d-beff-bba4f881590f/adabe0a1-d147-462a-8992-72af89cb5ae2]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:26:57.012 | INFO | nomic.project:create_index:1111 - Created map `badm-567-v3_qr_index` in project `Query-Response Map for badm-567-v3`: https://atlas.nomic.ai/map/022b4521-24df-401d-beff-bba4f881590f/e6c9798b-c154-43e7-917e-dd5cb71f116f\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: new-weather\n", + "(98, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:27:02.087 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for new-weather` in organization `dabholkar.asmita`\n", + "2023-08-30 15:27:03.117 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:27:03.125 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.07it/s]\n", + "2023-08-30 15:27:04.071 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:27:04.071 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:27:05.459 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for new-weather` in project `Query-Response Map for new-weather`: https://atlas.nomic.ai/map/48db10b9-0a6e-4c43-83f7-f7fa2b406257/fd200444-86cb-4b1c-9be6-ebd0262e2428\n", + "2023-08-30 15:27:05.461 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for new-weather: https://atlas.nomic.ai/map/48db10b9-0a6e-4c43-83f7-f7fa2b406257/fd200444-86cb-4b1c-9be6-ebd0262e2428\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for new-weather: https://atlas.nomic.ai/map/48db10b9-0a6e-4c43-83f7-f7fa2b406257/fd200444-86cb-4b1c-9be6-ebd0262e2428]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:27:07.001 | INFO | nomic.project:create_index:1111 - Created map `new-weather_qr_index` in project `Query-Response Map for new-weather`: https://atlas.nomic.ai/map/48db10b9-0a6e-4c43-83f7-f7fa2b406257/ff717c79-50cd-468b-9fcc-b391c8c167df\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: gies-online-mba-v2\n", + "(52, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:27:10.946 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for gies-online-mba-v2` in organization `dabholkar.asmita`\n", + "2023-08-30 15:27:11.862 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:27:11.868 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.33it/s]\n", + "2023-08-30 15:27:12.630 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:27:12.634 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:27:13.627 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for gies-online-mba-v2` in project `Query-Response Map for gies-online-mba-v2`: https://atlas.nomic.ai/map/8699b44f-8e2a-40e6-9562-0d2fc2898c3d/87144fbd-48a3-4d00-9136-ddc177685aa1\n", + "2023-08-30 15:27:13.627 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for gies-online-mba-v2: https://atlas.nomic.ai/map/8699b44f-8e2a-40e6-9562-0d2fc2898c3d/87144fbd-48a3-4d00-9136-ddc177685aa1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for gies-online-mba-v2: https://atlas.nomic.ai/map/8699b44f-8e2a-40e6-9562-0d2fc2898c3d/87144fbd-48a3-4d00-9136-ddc177685aa1]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:27:15.546 | INFO | nomic.project:create_index:1111 - Created map `gies-online-mba-v2_qr_index` in project `Query-Response Map for gies-online-mba-v2`: https://atlas.nomic.ai/map/8699b44f-8e2a-40e6-9562-0d2fc2898c3d/db13ea7a-f93d-4f97-b922-c51216d3d6e9\n", + "2023-08-30 15:27:15,670:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:27:15,673:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:27:20,003:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:27:20,003:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: frontend\n", + "(24, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:27:28.373 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for frontend` in organization `dabholkar.asmita`\n", + "2023-08-30 15:27:29.396 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:27:29.405 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.11it/s]\n", + "2023-08-30 15:27:30.325 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:27:30.325 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:27:31.539 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for frontend` in project `Query-Response Map for frontend`: https://atlas.nomic.ai/map/3a8ff45a-ca2d-4750-ab1b-ea033d35580d/e427ecf0-339a-41bf-8f9a-dc7a2930ac70\n", + "2023-08-30 15:27:31.542 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for frontend: https://atlas.nomic.ai/map/3a8ff45a-ca2d-4750-ab1b-ea033d35580d/e427ecf0-339a-41bf-8f9a-dc7a2930ac70\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for frontend: https://atlas.nomic.ai/map/3a8ff45a-ca2d-4750-ab1b-ea033d35580d/e427ecf0-339a-41bf-8f9a-dc7a2930ac70]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:27:33.028 | INFO | nomic.project:create_index:1111 - Created map `frontend_qr_index` in project `Query-Response Map for frontend`: https://atlas.nomic.ai/map/3a8ff45a-ca2d-4750-ab1b-ea033d35580d/006903b0-bb82-4432-9975-ff7c9ca80af9\n", + "2023-08-30 15:27:33,166:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:27:33,166:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:27:37,279:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:27:37,281:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:27:41,477:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:27:41,481:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: ECE220FA23\n", + "(193, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:27:50.988 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for ECE220FA23` in organization `dabholkar.asmita`\n", + "2023-08-30 15:27:51.867 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:27:51.878 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:01, 1.01s/it]\n", + "2023-08-30 15:27:52.904 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:27:52.908 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:27:53.929 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for ECE220FA23` in project `Query-Response Map for ECE220FA23`: https://atlas.nomic.ai/map/29b6f987-ee46-40be-808e-3d79a56d1b04/6b9b60f6-e2b7-4863-911c-fe84a63b35a1\n", + "2023-08-30 15:27:53.929 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for ECE220FA23: https://atlas.nomic.ai/map/29b6f987-ee46-40be-808e-3d79a56d1b04/6b9b60f6-e2b7-4863-911c-fe84a63b35a1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for ECE220FA23: https://atlas.nomic.ai/map/29b6f987-ee46-40be-808e-3d79a56d1b04/6b9b60f6-e2b7-4863-911c-fe84a63b35a1]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:27:55.645 | INFO | nomic.project:create_index:1111 - Created map `ECE220FA23_qr_index` in project `Query-Response Map for ECE220FA23`: https://atlas.nomic.ai/map/29b6f987-ee46-40be-808e-3d79a56d1b04/d8163c57-a2e8-41ca-90fc-043c8a9469b3\n", + "2023-08-30 15:27:55,758:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:27:55,759:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:27:59,841:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:27:59,841:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: Snowmass\n", + "(23, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:28:08.067 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for Snowmass` in organization `dabholkar.asmita`\n", + "2023-08-30 15:28:09.006 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:28:09.014 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.20it/s]\n", + "2023-08-30 15:28:09.854 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:28:09.858 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:28:10.994 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for Snowmass` in project `Query-Response Map for Snowmass`: https://atlas.nomic.ai/map/aa415361-5d23-44ba-ada2-252ed1dfe9a0/51de6990-9d60-4cc4-9a8e-46bc7de97a0e\n", + "2023-08-30 15:28:10.994 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for Snowmass: https://atlas.nomic.ai/map/aa415361-5d23-44ba-ada2-252ed1dfe9a0/51de6990-9d60-4cc4-9a8e-46bc7de97a0e\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for Snowmass: https://atlas.nomic.ai/map/aa415361-5d23-44ba-ada2-252ed1dfe9a0/51de6990-9d60-4cc4-9a8e-46bc7de97a0e]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:28:12.968 | INFO | nomic.project:create_index:1111 - Created map `Snowmass_qr_index` in project `Query-Response Map for Snowmass`: https://atlas.nomic.ai/map/aa415361-5d23-44ba-ada2-252ed1dfe9a0/4fdea35b-cda2-434e-afd1-e46e01430a97\n", + "2023-08-30 15:28:13,066:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:28:13,068:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:28:17,200:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:28:17,200:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:28:21,297:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:28:21,297:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: NPRE247\n", + "(54, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:28:29.951 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for NPRE247` in organization `dabholkar.asmita`\n", + "2023-08-30 15:28:31.043 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:28:31.051 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:01, 1.64s/it]\n", + "2023-08-30 15:28:32.709 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:28:32.714 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:28:33.787 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for NPRE247` in project `Query-Response Map for NPRE247`: https://atlas.nomic.ai/map/3ae13050-5e00-4f2b-9dd4-f05e98945405/d06b7c47-850c-48f4-a5f9-984fbb0ca424\n", + "2023-08-30 15:28:33.790 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for NPRE247: https://atlas.nomic.ai/map/3ae13050-5e00-4f2b-9dd4-f05e98945405/d06b7c47-850c-48f4-a5f9-984fbb0ca424\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for NPRE247: https://atlas.nomic.ai/map/3ae13050-5e00-4f2b-9dd4-f05e98945405/d06b7c47-850c-48f4-a5f9-984fbb0ca424]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:28:35.347 | INFO | nomic.project:create_index:1111 - Created map `NPRE247_qr_index` in project `Query-Response Map for NPRE247`: https://atlas.nomic.ai/map/3ae13050-5e00-4f2b-9dd4-f05e98945405/03471048-46aa-473c-b599-0bc812c679c0\n", + "2023-08-30 15:28:35,479:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:28:35,484:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:28:39,590:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:28:39,594:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: your-awesome-course\n", + "(30, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:28:50.102 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for your-awesome-course` in organization `dabholkar.asmita`\n", + "2023-08-30 15:28:51.013 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:28:51.022 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.10it/s]\n", + "2023-08-30 15:28:51.943 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:28:51.945 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:28:52.904 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for your-awesome-course` in project `Query-Response Map for your-awesome-course`: https://atlas.nomic.ai/map/dd5dfe40-c810-4402-9684-a43783ea6e52/468942cc-d46a-45a2-82a6-11fac2f2be78\n", + "2023-08-30 15:28:52.907 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for your-awesome-course: https://atlas.nomic.ai/map/dd5dfe40-c810-4402-9684-a43783ea6e52/468942cc-d46a-45a2-82a6-11fac2f2be78\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for your-awesome-course: https://atlas.nomic.ai/map/dd5dfe40-c810-4402-9684-a43783ea6e52/468942cc-d46a-45a2-82a6-11fac2f2be78]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:28:54.405 | INFO | nomic.project:create_index:1111 - Created map `your-awesome-course_qr_index` in project `Query-Response Map for your-awesome-course`: https://atlas.nomic.ai/map/dd5dfe40-c810-4402-9684-a43783ea6e52/e58f20e3-fa19-4c4c-8764-a185e0691c85\n", + "2023-08-30 15:28:54,549:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:28:54,549:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:28:58,646:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:28:58,653:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: pract\n", + "(44, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:29:07.007 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for pract` in organization `dabholkar.asmita`\n", + "2023-08-30 15:29:08.243 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:29:08.251 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:01, 1.11s/it]\n", + "2023-08-30 15:29:09.368 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:29:09.368 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:29:10.392 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for pract` in project `Query-Response Map for pract`: https://atlas.nomic.ai/map/15b51f99-fe4b-41ec-97fd-5579884e39fb/669c8862-db60-41ad-94a9-fd815e1fdcd8\n", + "2023-08-30 15:29:10.392 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for pract: https://atlas.nomic.ai/map/15b51f99-fe4b-41ec-97fd-5579884e39fb/669c8862-db60-41ad-94a9-fd815e1fdcd8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for pract: https://atlas.nomic.ai/map/15b51f99-fe4b-41ec-97fd-5579884e39fb/669c8862-db60-41ad-94a9-fd815e1fdcd8]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:29:12.031 | INFO | nomic.project:create_index:1111 - Created map `pract_qr_index` in project `Query-Response Map for pract`: https://atlas.nomic.ai/map/15b51f99-fe4b-41ec-97fd-5579884e39fb/b15570eb-4db4-4b6f-9b4d-e80309d2dcb3\n", + "2023-08-30 15:29:12,113:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:29:12,115:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:29:16,201:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:29:16,209:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:29:20,282:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:29:20,285:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: ece120FL22\n", + "(53, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:29:28.994 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for ece120FL22` in organization `dabholkar.asmita`\n", + "2023-08-30 15:29:29.838 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:29:29.846 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.18it/s]\n", + "2023-08-30 15:29:30.708 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:29:30.710 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:29:31.828 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for ece120FL22` in project `Query-Response Map for ece120FL22`: https://atlas.nomic.ai/map/8e74c555-3192-451b-b4a1-2c86587df88b/888e5d82-2e4d-46ba-b920-915dc31b835b\n", + "2023-08-30 15:29:31.828 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for ece120FL22: https://atlas.nomic.ai/map/8e74c555-3192-451b-b4a1-2c86587df88b/888e5d82-2e4d-46ba-b920-915dc31b835b\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for ece120FL22: https://atlas.nomic.ai/map/8e74c555-3192-451b-b4a1-2c86587df88b/888e5d82-2e4d-46ba-b920-915dc31b835b]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:29:33.354 | INFO | nomic.project:create_index:1111 - Created map `ece120FL22_qr_index` in project `Query-Response Map for ece120FL22`: https://atlas.nomic.ai/map/8e74c555-3192-451b-b4a1-2c86587df88b/eb4e553d-ecd3-4b11-9d75-468108ab08e2\n", + "2023-08-30 15:29:33,458:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:29:33,458:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:29:37,544:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:29:37,545:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:29:41,634:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:29:41,642:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: Law794-TransactionalDraftingAlam\n", + "(21, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:29:49.618 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for Law794-TransactionalDraftingAlam` in organization `dabholkar.asmita`\n", + "2023-08-30 15:29:50.718 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:29:50.731 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:01, 1.11s/it]\n", + "2023-08-30 15:29:51.849 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:29:51.851 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:29:53.034 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for Law794-TransactionalDraftingAlam` in project `Query-Response Map for Law794-TransactionalDraftingAlam`: https://atlas.nomic.ai/map/c4328a33-8958-462e-ba30-50f342736dae/0da00b97-bf94-4805-935e-458fb5c3d76f\n", + "2023-08-30 15:29:53.034 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for Law794-TransactionalDraftingAlam: https://atlas.nomic.ai/map/c4328a33-8958-462e-ba30-50f342736dae/0da00b97-bf94-4805-935e-458fb5c3d76f\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for Law794-TransactionalDraftingAlam: https://atlas.nomic.ai/map/c4328a33-8958-462e-ba30-50f342736dae/0da00b97-bf94-4805-935e-458fb5c3d76f]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:29:54.605 | INFO | nomic.project:create_index:1111 - Created map `Law794-TransactionalDraftingAlam_qr_index` in project `Query-Response Map for Law794-TransactionalDraftingAlam`: https://atlas.nomic.ai/map/c4328a33-8958-462e-ba30-50f342736dae/9792fd4e-2196-4e39-bded-cc2bfd42abbf\n", + "2023-08-30 15:29:54,728:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:29:54,731:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:29:58,804:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:29:58,804:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: NCSA\n", + "(84, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:30:07.528 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for NCSA` in organization `dabholkar.asmita`\n", + "2023-08-30 15:30:08.422 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:30:08.431 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.09it/s]\n", + "2023-08-30 15:30:09.361 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:30:09.361 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:30:10.325 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for NCSA` in project `Query-Response Map for NCSA`: https://atlas.nomic.ai/map/237ff7ed-c987-4259-96e0-3c5ed6e21207/8f176512-6a1e-4c8f-9d94-5794b1d7e619\n", + "2023-08-30 15:30:10.325 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for NCSA: https://atlas.nomic.ai/map/237ff7ed-c987-4259-96e0-3c5ed6e21207/8f176512-6a1e-4c8f-9d94-5794b1d7e619\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for NCSA: https://atlas.nomic.ai/map/237ff7ed-c987-4259-96e0-3c5ed6e21207/8f176512-6a1e-4c8f-9d94-5794b1d7e619]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:30:11.917 | INFO | nomic.project:create_index:1111 - Created map `NCSA_qr_index` in project `Query-Response Map for NCSA`: https://atlas.nomic.ai/map/237ff7ed-c987-4259-96e0-3c5ed6e21207/2ce836ec-557b-4037-9ebd-d3e8982c0926\n", + "2023-08-30 15:30:12,004:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:30:12,004:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:30:16,092:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:30:16,092:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:30:20,157:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:30:20,164:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: NCSADelta\n", + "(22, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:30:28.362 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for NCSADelta` in organization `dabholkar.asmita`\n", + "2023-08-30 15:30:29.318 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:30:29.326 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.10it/s]\n", + "2023-08-30 15:30:30.246 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:30:30.251 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:30:31.253 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for NCSADelta` in project `Query-Response Map for NCSADelta`: https://atlas.nomic.ai/map/f0c381d5-e2e9-41fb-807a-ae61bc818122/78d29eb7-be96-47c1-a856-b684c6419e34\n", + "2023-08-30 15:30:31.254 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for NCSADelta: https://atlas.nomic.ai/map/f0c381d5-e2e9-41fb-807a-ae61bc818122/78d29eb7-be96-47c1-a856-b684c6419e34\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for NCSADelta: https://atlas.nomic.ai/map/f0c381d5-e2e9-41fb-807a-ae61bc818122/78d29eb7-be96-47c1-a856-b684c6419e34]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:30:33.104 | INFO | nomic.project:create_index:1111 - Created map `NCSADelta_qr_index` in project `Query-Response Map for NCSADelta`: https://atlas.nomic.ai/map/f0c381d5-e2e9-41fb-807a-ae61bc818122/b6e64fef-a829-435f-89b5-ed1a44c05202\n", + "2023-08-30 15:30:33,214:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:30:33,214:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:30:37,289:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:30:37,290:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n", + "2023-08-30 15:30:41,376:INFO - error_code=rate_limit_exceeded error_message='Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method.' error_param=None error_type=requests message='OpenAI API error received' stream_error=False\n", + "2023-08-30 15:30:41,382:WARNING - Retrying langchain.embeddings.openai.embed_with_retry.._embed_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-text-embedding-ada-002 in organization org-4dafnkfdjctuHfuAPPkPsN46 on requests per min. Limit: 3 / min. Please try again in 20s. Contact us through our help center at help.openai.com if you continue to have issues. Please add a payment method to your account to increase your rate limit. Visit https://platform.openai.com/account/billing to add a payment method..\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "course name: NuclGPT-v1\n", + "(25, 1536)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:30:49.297 | INFO | nomic.project:_create_project:779 - Creating project `Query-Response Map for NuclGPT-v1` in organization `dabholkar.asmita`\n", + "2023-08-30 15:30:50.216 | INFO | nomic.atlas:map_embeddings:107 - Uploading embeddings to Atlas.\n", + "2023-08-30 15:30:50.222 | WARNING | nomic.project:_validate_and_correct_arrow_upload:261 - id_field is not a string. Converting to string from int32\n", + "1it [00:00, 1.23it/s]\n", + "2023-08-30 15:30:51.043 | INFO | nomic.project:_add_data:1401 - Upload succeeded.\n", + "2023-08-30 15:30:51.043 | INFO | nomic.atlas:map_embeddings:126 - Embedding upload succeeded.\n", + "2023-08-30 15:30:52.360 | INFO | nomic.project:create_index:1111 - Created map `Query-Response Map for NuclGPT-v1` in project `Query-Response Map for NuclGPT-v1`: https://atlas.nomic.ai/map/76043322-073c-4d8c-8c90-8e563da8564f/335097b0-a80d-4f38-8559-c6075a46a8b2\n", + "2023-08-30 15:30:52.360 | INFO | nomic.atlas:map_embeddings:139 - Query-Response Map for NuclGPT-v1: https://atlas.nomic.ai/map/76043322-073c-4d8c-8c90-8e563da8564f/335097b0-a80d-4f38-8559-c6075a46a8b2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Query-Response Map for NuclGPT-v1: https://atlas.nomic.ai/map/76043322-073c-4d8c-8c90-8e563da8564f/335097b0-a80d-4f38-8559-c6075a46a8b2]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-08-30 15:30:53.900 | INFO | nomic.project:create_index:1111 - Created map `NuclGPT-v1_qr_index` in project `Query-Response Map for NuclGPT-v1`: https://atlas.nomic.ai/map/76043322-073c-4d8c-8c90-8e563da8564f/441b7ff6-00ef-47f3-98a9-e45d327a1414\n" + ] + } + ], + "source": [ + "low_volume_courses = []\n", + "high_volume_courses = []\n", + "for course in course_names:\n", + " if course is None or course == 'ece408':\n", + " continue\n", + " \n", + " user_queries = []\n", + " metadata = []\n", + " i = 1\n", + " course_df = df[df['course_name'] == course]['convo']\n", + " for convo in course_df: # iterate through all conversations in a course\n", + " messages = convo['messages']\n", + "\n", + " # form query-response pairs out of the messages\n", + " pair_check = 0\n", + " for message in messages:\n", + " if message['role'] == 'user' and pair_check == 0:\n", + " query = message['content']\n", + " metadata_row = {'course': course, 'query': message['content']}\n", + " #print(\"metadata row: \", metadata_row)\n", + " pair_check += 1\n", + " \n", + " if message['role'] == 'assistant' and pair_check == 1:\n", + " metadata_row['response'] = message['content']\n", + " metadata_row['id'] = i \n", + " #print(\"response metadata row: \", metadata_row)\n", + " i += 1\n", + " pair_check += 1\n", + "\n", + " if pair_check == 2:\n", + " # one conversation pair is complete\n", + " user_queries.append(query)\n", + " metadata.append(metadata_row)\n", + " pair_check = 0\n", + " # after iterating every conversation in a course, create the map\n", + " if len(user_queries) < 20:\n", + " low_volume_courses.append(course)\n", + " continue\n", + "\n", + " if len(user_queries) > 500:\n", + " high_volume_courses.append(course)\n", + " continue\n", + " \n", + " metadata = pd.DataFrame(metadata)\n", + " embeddings = embeddings_model.embed_documents(user_queries)\n", + " embeddings = np.array(embeddings)\n", + " print(\"course name: \", course)\n", + " print(embeddings.shape)\n", + "\n", + " # create an Atlas project\n", + " project_name = \"Query-Response Map for \" + course\n", + " index_name = course + \"_qr_index\"\n", + " project = atlas.map_embeddings(embeddings=np.array(embeddings),\n", + " data=metadata,\n", + " id_field='id',\n", + " build_topic_model=True,\n", + " topic_label_field='query',\n", + " name=project_name,\n", + " colorable_fields=['query'])\n", + " print(project.maps)\n", + "\n", + " project.create_index(index_name, build_topic_model=True)\n", + "\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "low volume courses: ['test-video-ingest', 'badm-567', 'test-video-ingest-28', 'pdeploy999', 'badm-350-summer', 'previewtesting1', 'localtest2', 'your-favorite-url', 'mantine', 'test-video-ingest-2', 'badm_567_v2', 'erpnext', 'mip', 'farmdoc_test_kastan-v1', 'personalstatement', 'hrc', 'csv', 'star_nox', 'badm_567', 'SPaRCEd', 'localdemo8', 'badm_567_thumbnails', 'chatbot', 'erp', 'extreme', 'rohan_atree', 'zotero-extreme', 'test-video-ingest-20', 'gies-online-mba2', 'gies-online-mba', 'careerassistant', 'weather', 'lillian-wang-blog', 'local-test5', 'demo-for-vyriad', 'ag-gpt-beta', 'rtest', 'previewdeploy', 'r2test', 'personal-statement', 'rohan_excel', 'langchain-python', 'langchain', 'ncsa-live-demo', 'rohan_atree_individual', 'meta11-test', 'HealthyLivingGuide', 'rohan', 'babreu', 'test-video-ingest-31', 'p', 'test-video-ingest-17', 'summary', 'test-video-ingest-3', 'test-video-ingest-27', 'lillian-wang-blog-2', 'python-magic', 'ansible2', 'ece408fa23', 'farmdoc_test_josh_v2', 'local-test3', 'automata', 'SpaceFlorida-GT', 'GBSI-GT', 'newnew_ncsa', 'canvas', 'gbsi-gt', 'meditation-tutorial', 'profit', 'ansible', 'langchain-docs', 'testing_url_metadata_josh', 'test-india-biodiversity', 'vyriad', 'irc-workplans', 'kastanasdfasdfasdf', 'BADM-567-GT', 'mdt', 'vercel', 'gies-graduate-hub', 'test-video-ingest-12', 'test-video-ingest-13', 'Gies-graduate-hub', 'test_new_supa_scrape', 'doe-ascr-2023', 'arize', 'final-meta-test', 'preview-meta-test', 'gies-online-mba-V3', 'FoF-Drawdown-from-INVPEIV-5-24-23', 'FIN574-GT', 'test-video-ingest-30', 'test', 'NCSA-v2', 'conversational', 'clowder-docs', 'DA', 'test-video-ingest-21', 'test-video-ingest-25', 'test-ingest-10', 'eric-test-course', 'farmdoc-test', 'test-video-ingest-22', 'Academic-paper', 'starwars', 'AdamDemo']\n", + "high volume courses: ['gpt4', 'ECE408FA23']\n" + ] + } + ], + "source": [ + "print(\"low volume courses: \", low_volume_courses)\n", + "print(\"high volume courses: \", high_volume_courses)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +}