notebooks/02_GPT3_metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>auprc</th>\n",
       "      <th>validation_token_accuracy</th>\n",
       "      <th>recall</th>\n",
       "      <th>precision</th>\n",
       "      <th>validation_sequence_accuracy</th>\n",
       "      <th>elapsed_tokens</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>auroc</th>\n",
       "      <th>training_token_accuracy</th>\n",
       "      <th>training_sequence_accuracy</th>\n",
       "      <th>f1.0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>adav1</th>\n",
       "      <td>0.604722</td>\n",
       "      <td>0.613281</td>\n",
       "      <td>0.603622</td>\n",
       "      <td>0.587084</td>\n",
       "      <td>0.613281</td>\n",
       "      <td>3500800</td>\n",
       "      <td>0.592</td>\n",
       "      <td>0.631859</td>\n",
       "      <td>0.664062</td>\n",
       "      <td>0.664062</td>\n",
       "      <td>0.595238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>adav2</th>\n",
       "      <td>0.609321</td>\n",
       "      <td>0.578125</td>\n",
       "      <td>0.581489</td>\n",
       "      <td>0.591002</td>\n",
       "      <td>0.578125</td>\n",
       "      <td>3523328</td>\n",
       "      <td>0.592</td>\n",
       "      <td>0.627381</td>\n",
       "      <td>0.699219</td>\n",
       "      <td>0.699219</td>\n",
       "      <td>0.586207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>adav3</th>\n",
       "      <td>0.595404</td>\n",
       "      <td>0.574219</td>\n",
       "      <td>0.631791</td>\n",
       "      <td>0.576147</td>\n",
       "      <td>0.574219</td>\n",
       "      <td>3498752</td>\n",
       "      <td>0.586</td>\n",
       "      <td>0.619944</td>\n",
       "      <td>0.656250</td>\n",
       "      <td>0.656250</td>\n",
       "      <td>0.602687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>babbage</th>\n",
       "      <td>0.600456</td>\n",
       "      <td>0.652344</td>\n",
       "      <td>0.591549</td>\n",
       "      <td>0.604938</td>\n",
       "      <td>0.652344</td>\n",
       "      <td>3508992</td>\n",
       "      <td>0.605</td>\n",
       "      <td>0.626093</td>\n",
       "      <td>0.671875</td>\n",
       "      <td>0.671875</td>\n",
       "      <td>0.598169</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            auprc  validation_token_accuracy    recall  precision  \\\n",
       "adav1    0.604722                   0.613281  0.603622   0.587084   \n",
       "adav2    0.609321                   0.578125  0.581489   0.591002   \n",
       "adav3    0.595404                   0.574219  0.631791   0.576147   \n",
       "babbage  0.600456                   0.652344  0.591549   0.604938   \n",
       "\n",
       "         validation_sequence_accuracy  elapsed_tokens  accuracy     auroc  \\\n",
       "adav1                        0.613281         3500800     0.592  0.631859   \n",
       "adav2                        0.578125         3523328     0.592  0.627381   \n",
       "adav3                        0.574219         3498752     0.586  0.619944   \n",
       "babbage                      0.652344         3508992     0.605  0.626093   \n",
       "\n",
       "         training_token_accuracy  training_sequence_accuracy      f1.0  \n",
       "adav1                   0.664062                    0.664062  0.595238  \n",
       "adav2                   0.699219                    0.699219  0.586207  \n",
       "adav3                   0.656250                    0.656250  0.602687  \n",
       "babbage                 0.671875                    0.671875  0.598169  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import wandb\n",
    "api = wandb.Api()\n",
    "import pandas as pd\n",
    "\n",
    "runs = [\n",
    "    (\"adav1\", \"ft-uSwqlpAX9RlIcFfrYkyNI8g1\"),\n",
    "    (\"adav2\", \"ft-2ooqd5t0ycMwvf8TM3MUuyAt\"),\n",
    "    (\"adav3\", \"ft-cNrLBjqB9V6g0WLayqpRxcGK\"),\n",
    "    (\"babbage\", \"ft-3Svmi2GPLKTLnZAQCmHlEYql\"),\n",
    "]\n",
    "\n",
    "df = pd.DataFrame(\n",
    "    [\n",
    "        api.run(f\"benneo/GPT-3 blog title/{run_id}\").summary_metrics\n",
    "        for project_name, run_id in runs\n",
    "    ],\n",
    "    index=[project_name for project_name, _ in runs],\n",
    ")\n",
    "\n",
    "\n",
    "# remove \"classification/\" from column names\n",
    "df.columns = [col.split(\"/\")[-1] for col in df.columns]\n",
    "\n",
    "# filter auprc, aorc, precision, validation token accuarcy, validation sequence accuracy\n",
    "df = df.filter(regex=\"auprc|f1.0|auroc|accuracy|precision|recall|token|sequence\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>f1.0</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>recall</th>\n",
       "      <th>precision</th>\n",
       "      <th>auprc</th>\n",
       "      <th>auroc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>adav1</th>\n",
       "      <td>0.595238</td>\n",
       "      <td>0.592</td>\n",
       "      <td>0.603622</td>\n",
       "      <td>0.587084</td>\n",
       "      <td>0.604722</td>\n",
       "      <td>0.631859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>adav2</th>\n",
       "      <td>0.586207</td>\n",
       "      <td>0.592</td>\n",
       "      <td>0.581489</td>\n",
       "      <td>0.591002</td>\n",
       "      <td>0.609321</td>\n",
       "      <td>0.627381</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>adav3</th>\n",
       "      <td>0.602687</td>\n",
       "      <td>0.586</td>\n",
       "      <td>0.631791</td>\n",
       "      <td>0.576147</td>\n",
       "      <td>0.595404</td>\n",
       "      <td>0.619944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>babbage</th>\n",
       "      <td>0.598169</td>\n",
       "      <td>0.605</td>\n",
       "      <td>0.591549</td>\n",
       "      <td>0.604938</td>\n",
       "      <td>0.600456</td>\n",
       "      <td>0.626093</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             f1.0  accuracy    recall  precision     auprc     auroc\n",
       "adav1    0.595238     0.592  0.603622   0.587084  0.604722  0.631859\n",
       "adav2    0.586207     0.592  0.581489   0.591002  0.609321  0.627381\n",
       "adav3    0.602687     0.586  0.631791   0.576147  0.595404  0.619944\n",
       "babbage  0.598169     0.605  0.591549   0.604938  0.600456  0.626093"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['f1.0', 'accuracy', 'recall', 'precision', \"auprc\", \"auroc\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>training_token_accuracy</th>\n",
       "      <th>validation_token_accuracy</th>\n",
       "      <th>training_sequence_accuracy</th>\n",
       "      <th>validation_sequence_accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>adav1</th>\n",
       "      <td>0.664062</td>\n",
       "      <td>0.613281</td>\n",
       "      <td>0.664062</td>\n",
       "      <td>0.613281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>adav2</th>\n",
       "      <td>0.699219</td>\n",
       "      <td>0.578125</td>\n",
       "      <td>0.699219</td>\n",
       "      <td>0.578125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>adav3</th>\n",
       "      <td>0.656250</td>\n",
       "      <td>0.574219</td>\n",
       "      <td>0.656250</td>\n",
       "      <td>0.574219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>babbage</th>\n",
       "      <td>0.671875</td>\n",
       "      <td>0.652344</td>\n",
       "      <td>0.671875</td>\n",
       "      <td>0.652344</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         training_token_accuracy  validation_token_accuracy  \\\n",
       "adav1                   0.664062                   0.613281   \n",
       "adav2                   0.699219                   0.578125   \n",
       "adav3                   0.656250                   0.574219   \n",
       "babbage                 0.671875                   0.652344   \n",
       "\n",
       "         training_sequence_accuracy  validation_sequence_accuracy  \n",
       "adav1                      0.664062                      0.613281  \n",
       "adav2                      0.699219                      0.578125  \n",
       "adav3                      0.656250                      0.574219  \n",
       "babbage                    0.671875                      0.652344  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['training_token_accuracy', 'validation_token_accuracy', 'training_sequence_accuracy', 'validation_sequence_accuracy']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 ('gpt3-blog')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "98388095db23f0d270f6cd358fd032ff4feb9c08e3a138f6a6bcb71953dc310f"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}