diff --git a/examples/notebooks/remote/Saving_and_Loading_ExperimentHarness_to_Remote.ipynb b/examples/notebooks/remote/Saving_and_Loading_ExperimentHarness_to_Remote.ipynb index 6d92cc0..c843f51 100644 --- a/examples/notebooks/remote/Saving_and_Loading_ExperimentHarness_to_Remote.ipynb +++ b/examples/notebooks/remote/Saving_and_Loading_ExperimentHarness_to_Remote.ipynb @@ -81,14 +81,11 @@ " \n", " \n", " \n", + " template_index\n", + " templates\n", " user_inputs\n", " model\n", " messages\n", - " temperature\n", - " top_p\n", - " n\n", - " presence_penalty\n", - " frequency_penalty\n", " response\n", " response_usage\n", " latency\n", @@ -97,65 +94,65 @@ " \n", " \n", " 0\n", + " 0\n", + " [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} president?'}]\n", " {'input': 'first'}\n", " gpt-3.5-turbo\n", " [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the first president?'}]\n", - " 1.0\n", - " 1.0\n", - " 1\n", - " 0.0\n", - " 0.0\n", - " The first president of the United States was George Washington. He served two terms from 1789 to 1797.\n", - " {'completion_tokens': 24, 'prompt_tokens': 23, 'total_tokens': 47}\n", - " 1.251051\n", + " The first president of the United States was George Washington. He served from 1789 to 1797.\n", + " {'completion_tokens': 22, 'prompt_tokens': 23, 'total_tokens': 45}\n", + " 1.620099\n", " \n", " \n", " 1\n", + " 1\n", + " [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} vice president?'}]\n", " {'input': 'second'}\n", " gpt-3.5-turbo\n", " [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the second president?'}]\n", - " 1.0\n", - " 1.0\n", - " 1\n", - " 0.0\n", - " 0.0\n", " The second president of the United States was John Adams. He served from 1797 to 1801.\n", " {'completion_tokens': 22, 'prompt_tokens': 23, 'total_tokens': 45}\n", - " 0.949733\n", + " 0.657733\n", " \n", " \n", " 2\n", + " 0\n", + " [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} president?'}]\n", " {'input': 'first'}\n", " gpt-3.5-turbo\n", " [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the first vice president?'}]\n", - " 1.0\n", - " 1.0\n", - " 1\n", - " 0.0\n", - " 0.0\n", - " The first Vice President of the United States was John Adams. He served as Vice President under President George Washington from 1789 to 1797.\n", - " {'completion_tokens': 30, 'prompt_tokens': 24, 'total_tokens': 54}\n", - " 0.936664\n", + " The first Vice President of the United States was John Adams. He served under President George Washington from 1789 to 1797.\n", + " {'completion_tokens': 27, 'prompt_tokens': 24, 'total_tokens': 51}\n", + " 0.809008\n", " \n", " \n", " 3\n", + " 1\n", + " [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} vice president?'}]\n", " {'input': 'second'}\n", " gpt-3.5-turbo\n", " [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the second vice president?'}]\n", - " 1.0\n", - " 1.0\n", - " 1\n", - " 0.0\n", - " 0.0\n", - " The second vice president of the United States was Thomas Jefferson. He served as vice president from 1797 to 1801 under President John Adams.\n", + " The second Vice President of the United States was Thomas Jefferson. He served as Vice President under President John Adams from 1797 to 1801.\n", " {'completion_tokens': 30, 'prompt_tokens': 24, 'total_tokens': 54}\n", - " 0.971230\n", + " 0.922675\n", " \n", " \n", "\n", "" ], "text/plain": [ + " template_index \\\n", + "0 0 \n", + "1 1 \n", + "2 0 \n", + "3 1 \n", + "\n", + " templates \\\n", + "0 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} president?'}] \n", + "1 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} vice president?'}] \n", + "2 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} president?'}] \n", + "3 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} vice president?'}] \n", + "\n", " user_inputs model \\\n", "0 {'input': 'first'} gpt-3.5-turbo \n", "1 {'input': 'second'} gpt-3.5-turbo \n", @@ -168,29 +165,23 @@ "2 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the first vice president?'}] \n", "3 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the second vice president?'}] \n", "\n", - " temperature top_p n presence_penalty frequency_penalty \\\n", - "0 1.0 1.0 1 0.0 0.0 \n", - "1 1.0 1.0 1 0.0 0.0 \n", - "2 1.0 1.0 1 0.0 0.0 \n", - "3 1.0 1.0 1 0.0 0.0 \n", - "\n", " response \\\n", - "0 The first president of the United States was George Washington. He served two terms from 1789 to 1797. \n", + "0 The first president of the United States was George Washington. He served from 1789 to 1797. \n", "1 The second president of the United States was John Adams. He served from 1797 to 1801. \n", - "2 The first Vice President of the United States was John Adams. He served as Vice President under President George Washington from 1789 to 1797. \n", - "3 The second vice president of the United States was Thomas Jefferson. He served as vice president from 1797 to 1801 under President John Adams. \n", + "2 The first Vice President of the United States was John Adams. He served under President George Washington from 1789 to 1797. \n", + "3 The second Vice President of the United States was Thomas Jefferson. He served as Vice President under President John Adams from 1797 to 1801. \n", "\n", " response_usage \\\n", - "0 {'completion_tokens': 24, 'prompt_tokens': 23, 'total_tokens': 47} \n", + "0 {'completion_tokens': 22, 'prompt_tokens': 23, 'total_tokens': 45} \n", "1 {'completion_tokens': 22, 'prompt_tokens': 23, 'total_tokens': 45} \n", - "2 {'completion_tokens': 30, 'prompt_tokens': 24, 'total_tokens': 54} \n", + "2 {'completion_tokens': 27, 'prompt_tokens': 24, 'total_tokens': 51} \n", "3 {'completion_tokens': 30, 'prompt_tokens': 24, 'total_tokens': 54} \n", "\n", " latency \n", - "0 1.251051 \n", - "1 0.949733 \n", - "2 0.936664 \n", - "3 0.971230 " + "0 1.620099 \n", + "1 0.657733 \n", + "2 0.809008 \n", + "3 0.922675 " ] }, "metadata": {}, @@ -203,6 +194,217 @@ "harness.visualize()" ] }, + { + "cell_type": "markdown", + "id": "f246c958", + "metadata": {}, + "source": [ + "### Examples of `aggregation` API" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "88d81001", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
user_inputslatency
0((input, first),)1.214553
1((input, second),)0.790204
\n", + "
" + ], + "text/plain": [ + " user_inputs latency\n", + "0 ((input, first),) 1.214553\n", + "1 ((input, second),) 0.790204" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "agg_df = harness.aggregate(groupby_column='user_inputs', aggregate_columns='latency', method='mean')\n", + "agg_df" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "349bec6c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
template_indexlatencytemplates
001.214553[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} president?'}]
110.790204[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} vice president?'}]
\n", + "
" + ], + "text/plain": [ + " template_index latency \\\n", + "0 0 1.214553 \n", + "1 1 0.790204 \n", + "\n", + " templates \n", + "0 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} president?'}] \n", + "1 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} vice president?'}] " + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "agg_df = harness.aggregate(groupby_column='templates', aggregate_columns='latency', method='mean')\n", + "agg_df" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "18419516", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
template_indexlatencylatencytemplates
001.2145531.214553[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} president?'}]
110.7902040.790204[{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} vice president?'}]
\n", + "
" + ], + "text/plain": [ + " template_index latency latency \\\n", + "0 0 1.214553 1.214553 \n", + "1 1 0.790204 0.790204 \n", + "\n", + " templates \n", + "0 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} president?'}] \n", + "1 [{'role': 'system', 'content': 'You are a helpful assistant.'}, {'role': 'user', 'content': 'Who was the {{input}} vice president?'}] " + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "agg_df = harness.aggregate(groupby_column='templates', aggregate_columns=['latency', 'latency'], method='mean')\n", + "agg_df" + ] + }, { "cell_type": "code", "execution_count": 4,