random_experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "import matplotlib.pyplot as plt\n",
    "from run_model import get_dataloader\n",
    "from utils import get_all_amino_acids, get_wild_type_amino_acid_sequence, count_substring_mismatch\n",
    "from rnn import GenerativeRNN\n",
    "from hmm import GenerativeHMM\n",
    "from vae import GenerativeVAE\n",
    "import numpy as np\n",
    "import time\n",
    "import pandas as pd\n",
    "from scipy.stats import skewnorm\n",
    "from utils import hmm_default_args, vae_default_args, rnn_default_args\n",
    "from utils import hmm_default_small_args, vae_default_small_args, rnn_default_small_args\n",
    "from utils import translate, load_data, get_all_dataset_names\n",
    "from sklearn.model_selection import train_test_split\n",
    "import os\n",
    "import subprocess\n",
    "import sys\n",
    "import traceback"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating Random Sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_mutations_df(base_sequence, mutations_lst, mutation_count_lst, alphabet, \n",
    "                          start_mutation_index=0, end_mutation_index=0, verbose=False):\n",
    "    \"\"\"\n",
    "    Generates a mutated dataframe\n",
    "    :param base_sequence: string, the sequence to be mutated\n",
    "    :param mutations_lst: list, the number of mutations from the base sequence\n",
    "    :param mutation_count_lst: list, defines the number of each mutation count to generate\n",
    "    :param alphabet: string, the entire set of letters in the vocabulary\n",
    "    :param start_mutation_index: int, how many characters to skip at the beginning of the mutation\n",
    "    :param end_mutation_index: int, how many characters to skip at the end of the mutation\n",
    "    :return: a dataframe comprised of three columns: the mutated strings, the number of mutations, the base_sequence\n",
    "    >>> mutations_lst, base_sequence = [1, 2, 3], get_wild_type_amino_acid_sequence()\n",
    "    >>> alphabet, num_per_mutation_count = get_all_amino_acids(), 100\n",
    "    >>> mutated_df = generate_mutations_df(mutations_lst, base_sequence, alphabet, num_per_mutation_count, verbose=False)\n",
    "    >>> for i, row in mutated_df.iterrows():\n",
    "    ...     assert(count_substring_mismatch(row[\"mutated_string\"], row[\"base_sequence\"]) == row[\"mutation_count\"])\n",
    "    \"\"\"\n",
    "    start_time = time.time()\n",
    "    # get index maps\n",
    "    characters_to_index = dict(zip(alphabet, range(len(alphabet))))\n",
    "    index_to_characters = dict(zip(range(len(alphabet)), alphabet))\n",
    "    num_characters = len(alphabet)\n",
    "    total_data_points = sum(mutation_count_lst)        \n",
    "    mutations_per_sequence_lst = np.array([[mutation_count for _ in range(num_mutations)] \n",
    "                                for mutation_count, num_mutations in zip(mutations_lst, mutation_count_lst)]).flatten()\n",
    "    assert(total_data_points < 200000)\n",
    "    assert(len(mutations_lst) == len(mutation_count_lst))\n",
    "    assert(len(mutations_per_sequence_lst) == total_data_points)\n",
    "    mutated_strings_lst = []\n",
    "    for i, mutation_count in enumerate(mutations_per_sequence_lst):\n",
    "        mutated_string = get_mutation(string=base_sequence,\n",
    "                                      num_mutations=mutation_count,\n",
    "                                      num_characters=num_characters,\n",
    "                                      characters_to_index=characters_to_index,\n",
    "                                      index_to_characters=index_to_characters,\n",
    "                                      start_mutation_index=start_mutation_index,\n",
    "                                      end_mutation_index=end_mutation_index)\n",
    "        mutated_strings_lst.append(mutated_string)\n",
    "        if i % 1000 == 0 and verbose:\n",
    "            print(\n",
    "                \"Generated {0}/{1} mutations in {2:.2f} seconds\".format(i, total_data_points, time.time() - start_time))\n",
    "    base_sequences_lst = [base_sequence] * total_data_points\n",
    "    mutated_df = pd.DataFrame.from_dict({\n",
    "        'mutated_string': mutated_strings_lst,\n",
    "        'mutation_count': mutations_per_sequence_lst,\n",
    "        'base_sequence': base_sequences_lst\n",
    "    })\n",
    "    if verbose:\n",
    "        print(\"{0:.2f} seconds to generate the mutated df\".format(time.time() - start_time))\n",
    "    return mutated_df\n",
    "\n",
    "def get_mutation(string, num_mutations, num_characters, characters_to_index, index_to_characters,\n",
    "                 start_mutation_index=0, end_mutation_index=0):\n",
    "    \"\"\"\n",
    "    get a mutation of a string\n",
    "    :param string: the string to be mutated\n",
    "    :param num_mutations: the number of mutations\n",
    "    :param num_characters: the number of characters in alphabet\n",
    "    :param characters_to_index: map from characters in alphabet to index\n",
    "    :param index_to_characters: map from index in alphabet to characters\n",
    "    :param start_mutation_index: how many characters to skip at the beginning of the mutation\n",
    "    :param end_mutation_index: how many characters to skip at the end of the mutation\n",
    "    :return: mutated string\n",
    "    >>> get_mutation(\"AAC\", num_mutations=1, num_characters=2, characters_to_index={\"A\": 0, \"C\": 1}, \\\n",
    "        index_to_characters={0: \"A\", 1: \"C\"}, start_mutation_index=1, end_mutation_index=1)\n",
    "    'ACC'\n",
    "    >>> np.random.seed(1)\n",
    "    >>> get_mutation(\"ACTGA\", num_mutations=2, num_characters=4, characters_to_index={\"A\": 0, \"C\": 1, \"T\": 2, \"G\": 3}, \\\n",
    "        index_to_characters={0: \"A\", 1: \"C\", 2: \"T\", 3: \"G\"}, start_mutation_index=1, end_mutation_index=1)\n",
    "    'ATTAA'\n",
    "    \"\"\"\n",
    "    mutation = list(string)\n",
    "    indexes = np.random.choice(range(start_mutation_index, len(string) - end_mutation_index),\n",
    "                            num_mutations, replace=False)\n",
    "    for i in indexes:\n",
    "        original_c = string[i]\n",
    "        original_index = characters_to_index[original_c]\n",
    "        # mutation by moving the character somewhere else along the possible sequences.\n",
    "        new_index = (original_index + np.random.randint(1, num_characters)) % num_characters\n",
    "        mutation[i] = index_to_characters[new_index]\n",
    "    return \"\".join(mutation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "alphabet = get_all_amino_acids()\n",
    "characters_to_index = dict(zip(alphabet, range(len(alphabet))))\n",
    "index_to_characters = dict(zip(range(len(alphabet)), alphabet))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Cannot take a larger sample than population when 'replace=False'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-e60e9be9bf35>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mwild_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_wild_type_amino_acid_sequence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0ms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mget_mutation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwild_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m238\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malphabet\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcharacters_to_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex_to_characters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-2-437b825eb814>\u001b[0m in \u001b[0;36mget_mutation\u001b[0;34m(string, num_mutations, num_characters, characters_to_index, index_to_characters, start_mutation_index, end_mutation_index)\u001b[0m\n\u001b[1;32m     72\u001b[0m     \u001b[0mmutation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstring\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m     indexes = np.random.choice(range(start_mutation_index, len(string) - end_mutation_index),\n\u001b[0;32m---> 74\u001b[0;31m                             num_mutations, replace=False)\n\u001b[0m\u001b[1;32m     75\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mindexes\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m         \u001b[0moriginal_c\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstring\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mmtrand.pyx\u001b[0m in \u001b[0;36mmtrand.RandomState.choice\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Cannot take a larger sample than population when 'replace=False'"
     ]
    }
   ],
   "source": [
    "wild_type = get_wild_type_amino_acid_sequence()\n",
    "s=get_mutation(wild_type, 238, len(alphabet), characters_to_index, index_to_characters, 0, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mismatches: 238\n",
      "AFRLANQFAAAITPLFPI*LWKEVRPFGCLWKD*GQNSAMNPGHNEIQWAVKGNLADNIFRVGPVRMPESQMVNKVGGRSRGPHQDAGNPECWSYFP*YYFP*SRWFDG*CNRRKHKIQRQWVVASD*MFSVTAAMKMFGTC*VRK*CPIWDDLYFMWFWFDMVT**KFNACMFDSQPVTWLHKQDSHITQYTMCWARSKGWRFGRETTKQSHEMEYEPSFHTLMIKACVVN*DPWPK\n"
     ]
    }
   ],
   "source": [
    "print(\"Mismatches:\", count_substring_mismatch(s, wild_type))\n",
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generated 0/1000 mutations in 0.00 seconds\n",
      "0.13 seconds to generate the mutated df\n"
     ]
    }
   ],
   "source": [
    "mutations_lst = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n",
    "mutations_count = [100] * 10\n",
    "base_sequence = wild_type\n",
    "alphabet = get_all_amino_acids() \n",
    "num_per_mutation_count = 1000\n",
    "start_mutation_index = 5\n",
    "end_mutation_index = 5\n",
    "mutated_df = generate_mutations_df(base_sequence, mutations_lst, mutations_count, alphabet, start_mutation_index, end_mutation_index, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, row in mutated_df.iterrows():\n",
    "    assert(count_substring_mismatch(row[\"mutated_string\"], row[\"base_sequence\"]) == row[\"mutation_count\"])\n",
    "    assert(count_substring_mismatch(row[\"mutated_string\"][:start_mutation_index], row[\"base_sequence\"][:start_mutation_index]) == 0)\n",
    "    assert(count_substring_mismatch(row[\"mutated_string\"][-end_mutation_index:], row[\"base_sequence\"][-end_mutation_index:]) == 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1000, 3)\n"
     ]
    }
   ],
   "source": [
    "print(mutated_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generated 0/10000 mutations in 0.00 seconds\n",
      "Generated 1000/10000 mutations in 0.08 seconds\n",
      "Generated 2000/10000 mutations in 0.14 seconds\n",
      "Generated 3000/10000 mutations in 0.20 seconds\n",
      "Generated 4000/10000 mutations in 0.26 seconds\n",
      "Generated 5000/10000 mutations in 0.33 seconds\n",
      "Generated 6000/10000 mutations in 0.39 seconds\n",
      "Generated 7000/10000 mutations in 0.45 seconds\n",
      "Generated 8000/10000 mutations in 0.54 seconds\n",
      "Generated 9000/10000 mutations in 0.62 seconds\n",
      "0.69 seconds to generate the mutated df\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "mutations_lst = range(10)\n",
    "mutations_count = [1000] * 10\n",
    "name = \"synthetic_data_length_20_uniform\"\n",
    "base_sequence = \"GYSSASKIIFGSGTRLSIRP\"\n",
    "alphabet = get_all_amino_acids()\n",
    "start_mutation_index = 1\n",
    "end_mutation_index = 1\n",
    "mutated_df = generate_mutations_df(base_sequence, mutations_lst, mutations_count, alphabet,\n",
    "                                   start_mutation_index, end_mutation_index, verbose=True)\n",
    "mutated_df = mutated_df.sample(frac=1).reset_index(drop=True)\n",
    "mutated_df.to_csv(name + \".csv\")\n",
    "x_train, x_test = train_test_split(mutated_df, test_size = 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(name + \"_x_train.npy\", x_train)\n",
    "np.save(name + \"_x_test.npy\", x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Discrete Gaussian Distributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_discrete_gaussian_distribution(num_values, mean=5.5, std=2, low=1, high=10):\n",
    "    \"\"\"\n",
    "    generates a discrete gaussian distribution\n",
    "    :param num_values: int, number of samples to take from discrete distribution\n",
    "    :param mean: float, mean of gaussian\n",
    "    :param std: float, standard deviation of gaussian\n",
    "    :param low: int, lowest value the discrete distribution will have\n",
    "    :param high: int, highest value the discrete distribution will have\n",
    "    :return: gaussian_lst, list, list of values in distributions\n",
    "            bins, dict, map between the values and counts in distribution\n",
    "    >>> np.random.seed(1)\n",
    "    >>> generate_discrete_gaussian_distribution(10, 0, 1, -4, 4)[0]\n",
    "    [2.0, -1.0, -1.0, -1.0, 1.0, -2.0, 2.0, -1.0, 0.0, -0.0]\n",
    "    >>> generate_discrete_gaussian_distribution(10, 0, 1, -4, 4)[1]\n",
    "    {-4: 0.0, -3: 0.0, -2: 1.0, -1: 2.0, 0: 4.0, 1: 3.0, 2: 0.0, 3: 0.0, 4: 0.0}\n",
    "    \"\"\"\n",
    "    normal_lst = np.random.normal(loc=mean, scale=std, size=num_values)\n",
    "    bins = dict(zip(list(range(low, high + 1)), np.zeros(high - low + 1)))\n",
    "    gaussian_lst = []\n",
    "    for x in normal_lst:\n",
    "        if x <= low:\n",
    "            gaussian_lst.append(low)\n",
    "            bins[low] += 1\n",
    "        elif x >= high:\n",
    "            gaussian_lst.append(high)\n",
    "            bins[high] += 1\n",
    "        else:\n",
    "            gaussian_lst.append(round(x))\n",
    "            bins[round(x)] += 1\n",
    "    return gaussian_lst, bins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{1: 246.0, 2: 426.0, 3: 848.0, 4: 1465.0, 5: 1973.0, 6: 1935.0, 7: 1537.0, 8: 888.0, 9: 453.0, 10: 229.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1111267f0>]"
      ]
     },
     "execution_count": 266,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAD8CAYAAACRkhiPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xlc1VX+x/HXhx0RFBUVEWVxX9HITHNrtU2dFstm2qbJFmtqarb6/ZpsapqmaZlps2zvN5VZ1mjWZE6ulaVouOAKiAIioCiK7HB+f/C1sFQQuJy7fJ6Px33cy7nf771vrsKHc873+z1ijEEppZRv8rMdQCmllD1aBJRSyodpEVBKKR+mRUAppXyYFgGllPJhWgSUUsqHaRFQSikfpkVAKaV8mBYBpZTyYQG2AzSkU6dOJi4uznYMpZTyGGvXrt1njIlqzLZuXwTi4uJISUmxHUMppTyGiOxq7LY6HKSUUj5Mi4BSSvmwBouAiMSKyFIR2SwiaSJyl9PeQUQWi8gO5z7SaRcReUZE0kVkg4gMr/da1zvb7xCR6133bSmllGqMxvQEqoF7jTEDgJHADBEZAPwR+MIY0xv4wvka4EKgt3ObDsyCuqIBPAicAYwAHjxaOJRSStnRYBEwxuQZY9Y5jw8DW4AYYDLwprPZm8AU5/Fk4C1T5xugvYhEAxcAi40xRcaYA8BiYGKLfjdKKaVOySnNCYhIHDAM+BboYozJc57aC3RxHscA2fV2y3HaTtR+vPeZLiIpIpJSWFh4KhGVUkqdgkYXARFpC8wD7jbGHKr/nKlbnqzFligzxsw2xiQbY5Kjohp1qKtSSqkmaNR5AiISSF0BeNsY86HTnC8i0caYPGe4p8BpzwVi6+3e3WnLBcb/qH1Z06Mr1TTrdh9gzc4iuke2IbZDKN0j2xDZJhARsR1NqVbXYBGQup+MV4Etxpin6j21ALgeeMy5n1+v/Q4RmUPdJHCxUygWAY/Wmww+H7ivZb4NpRrncHkV099ay76SimPaw4L8jykK3SNDie3ww31ESKClxEq5VmN6AqOBa4GNIpLqtN1P3S//uSJyE7ALmOo89ylwEZAOlAI3AhhjikTkYWCNs92fjTFFLfJdKNVIz3yxg/1HKnj35pG0Cw0k50Ap2QfK6u6L6u5XZeznSGXNMfu1Cw2sKwj1CkX9gtEmyO1PvlfquBr8n2uM+RI4UT/5nONsb4AZJ3it14DXTiWgUi0lvaCE17/KYuppsZyZ2BGAAd0ifrKdMYaDpVXkHCgj+0Ap2UWl3z9OLyxh2fYCyqtqj9mnU9sgYiLbEBv5Q4GIdQpETGQowQH+rfI9KnWq9M8X5ROMMTz0cRqhQf78bmLfk24rIkSGBREZFsTg7u2O+1qFJRV1hcEpEEd7Eptyi1mUtpeqGlPv9aBzeLDTi2jzfY+ie4dQhveIJCRQC4SyR4uA8gmLN+ezcsc+/nTJADq1DW7Wa4kIncND6BwewvAePz3fsabWUHC4nOyismN6ETkHSlm9s4j5qWXUOjXi9LhI3rl5JIH+egUXZYcWAeX1yqtqePiTzfTu3JZrz+zp8vfz9xOi24US3S6UEfEdfvJ8VU0te4vL+WJLPjM/3syTn2/njxf2c3kupY5H//xQXu/lFZlkF5Uxc9JAt/iLO9Dfj9gObbhhdDzXnNGDF5dnsHRrQcM7KuUC9n8ilHKhPQfLeH5ZOhcO6sroXp1sx/mJP10ygP7REdwzN5U9B8tsx1E+SIuA8mqPfroFY+D+i/rbjnJcIYH+PH/NMCqra7nz3e+oqqlteCelWpAWAeW1VmXsZ+GGPG4bn0hshza245xQQlRbHr1sMGt3HeDJz7fbjqN8jBYB5ZWqa2p56OM0YtqHcuu4RNtxGjQ5KYZpI5z5gW06P6BajxYB5ZXeWb2brXsP878X9/eY4/AfvHQA/bqGc897qeQV6/yAah1aBJTXKTpSyZOfb2d0r45MHNTVdpxGCwn05/mfD6+bH3jnO6p1fkC1Ai0Cyus88fk2SiqqefDSgR53ZdBEZ34gZdcBnlys8wPK9bQIKK+yKbeYd1fv5roze9KnS7jtOE1ydH5g1jKdH1Cup0VAeQ1jDDMXpNGhTRB3n9vHdpxm0fkB1Vq0CCivMT91Dym7DvD7iX1pF+rZ1//X+QHVWrQIKK9QUlHNo59uYUj3dlx5WmzDO3gAnR9QrUGLgPIKzy1Jp+BwBTMnDcTPz7Mmg09G5weUq2kRUB5v574jvPplJpcP737cSzt7Op0fUK6kRUB5vIcXbiY4wJ8/XHjyxWI8lc4PKFfSIqA82pKt+SzZWsBd5/Smc3iI7Tguo/MDylUaLAIi8pqIFIjIpnpt74lIqnPLOroAvYjEiUhZvederLfPaSKyUUTSReQZ8bSzeJTbqaiu4c8fbyYhKozrR8XZjuNydfMDscxalsEynR9QLaQxPYE3gIn1G4wxVxljkowxScA84MN6T2ccfc4Yc2u99lnAzUBv53bMayp1ql77Mous/aXMvHQgQQG+0al98NKBdfMDc9fr/IBqEQ3+5BhjVgBFx3vO+Wt+KvDuyV5DRKKBCGPMN8YYA7wFTDn1uErV2VtczrNLdnDegC6M7RNlO06rOTo/UF5Vw6/f1fkB1XzN/fNpDJBvjNlRry1eRL4TkeUiMsZpiwFy6m2T47Qp1SSP/WcL1bWGBy4eYDtKq0uMasujPxvMmqwDPKXzA6qZmlsEpnFsLyAP6GGMGQbcA7wjIhGn+qIiMl1EUkQkpbCwsJkRlbdZk1XEv1P3cMvYBHp0dN/FYlxpyrC6+YEXdH5ANVOTi4CIBACXAe8dbTPGVBhj9juP1wIZQB8gF+heb/fuTttxGWNmG2OSjTHJUVG+09VXDaupNTw4P43odiHcNt79F4txJZ0fUC2hOT2Bc4Gtxpjvh3lEJEpE/J3HCdRNAGcaY/KAQyIy0plHuA6Y34z3Vj5qzprdbM47xP0X9adNUIDtOFbp/IBqCY05RPRdYBXQV0RyROQm56mr+emE8Fhgg3PI6AfArcaYo5PKtwOvAOnU9RD+0wL5lQ85WFrJE4u2cUZ8By4ZEm07jlvQ+QHVXA3+KWWMmXaC9huO0zaPukNGj7d9CjDoFPMp9b2nFm+nuKyKmZM8b7EYV5oyLIZvd+7nhWUZjIjvwPi+nW1HUh7ENw6uVh5vS94h/vXNLn4xsif9o0/5WAOvp/MDqqm0CCi3Z4zhwQVptAsN5J7zPHuxGFfR+QHVVFoElNtbuCGP1TuL+O0FfWnfJsh2HLdVf37g6f/q/IBqHC0Cyq2VVtYtFjOwWwRXn97Ddhy3d/T8geeXZrB8u55joxqmRUC5tReWZpBXXM5Dkwbi70WLxbjS0fmB37yXyt7icttxlJvTIqDc1q79R5i9IpMpSd1IjutgO47H0PkBdSq0CCi39cgnWwjwF+67qL/tKB7n6PzA6qwinR9QJ6VFQLml5dsLWbw5nzvP7k2XCO9dLMaVpgyL4erTdX5AnZwWAeV2KqtreejjNOI7hfHLs+Jsx/FoMyfp/IA6OS0Cyu28+XUWmYVH+NMlAwgO8Lcdx6OFBPrz3DU6P6BOTIuAcisFh8r55xc7OLtfZyb008sftIRenXV+QJ2YFgHlVv722TYqq2t54BLfWyzGlXR+QJ2IFgHlNtbtPsC8dTncNCae+E5htuN4HZ0fUMejRUC5hdpaw8wFaXSJCOaOCb1sx/FKx8wPzNH5AVVHi4ByC++vzWZDTjH3XdifsGDfXizGlb6fH9hZxD/+u6PhHZTX0yKgrCsuq+Lxz7aR3DOSyUndbMfxet/PDyxL1/kBpUVA2feP/26nqLRSF4tpRTMnDaRvF50fUFoElGXb8w/z1qpdXDOiB4Ni2tmO4zN0fkAdpUVAWWNM3WRw2+AA7j2/r+04Pqf+/MAbX2fZjqMsacxC86+JSIGIbKrXNlNEckUk1bldVO+5+0QkXUS2icgF9donOm3pIvLHlv9WlKf5bNNevs7Yz73n96FDmC4WY8OUYTGMSuzIKyt3UlmtvQFf1JiewBvAxOO0P22MSXJunwKIyADgamCgs88LIuIvIv7A88CFwABgmrOt8lFllTU88skW+nUN55oRuliMTdPHJrD3UDkL1u+xHUVZ0GARMMasAIoa+XqTgTnGmApjzE4gHRjh3NKNMZnGmEpgjrOt8lEvLs8g92AZMycNJMBfRyVtGtcnin5dw5m9IgNjjO04qpU156fvDhHZ4AwXRTptMUB2vW1ynLYTtSsflF1UyovLM7hkSDQjEzrajuPzRIRbxiWwPb+EZdv0kFFf09QiMAtIBJKAPODJFksEiMh0EUkRkZTCQv1P6W0e/XQLfiLcr4vFuI1LhnSjW7sQXlyeYTuKamVNKgLGmHxjTI0xphZ4mbrhHoBcILbept2dthO1n+j1Zxtjko0xyVFRUU2JqNzUV+n7+M+mvcyYkEi39qG24yhHoL8fvzwrnm93FpGafdB2HNWKmlQERCS63pc/A44eObQAuFpEgkUkHugNrAbWAL1FJF5EgqibPF7Q9NjKE1XV1DJzQRo9OrThV2MSbMdRP3L1iB5EhAQwe4X2BnxJgxdpEZF3gfFAJxHJAR4ExotIEmCALOAWAGNMmojMBTYD1cAMY0yN8zp3AIsAf+A1Y0xai383yq298+1udhSU8PJ1yYQE6mIx7qZtcADXntmTF5ZlkLXvCHF6JVefIO5+NEBycrJJSUmxHUM1U0V1DeMeX0aPjm14b/pIvTyEmyo4XM5Zjy3lyuTu/OVng23HUU0kImuNMcmN2VaPzVOtYt7aXPYeKufOs3tpAXBjncNDuPy0GN5fm8O+kgrbcVQr0CKgXK6qppYXlqUzNLY9Z/XqZDuOasCvxiRQVVPLW3opCZ+gRUC53ILUPeQcKOPOCdoL8ASJUW05r38X3ly1iyMV1bbjKBfTIqBcqqbW8MKydPpHR3BOf1043lPcMi6R4rIq5qZkN7yx8mhaBJRLfbZpLxmFR7hDewEe5bSekZweF8krK3fqZaa9nBYB5TLGGJ5dsoOEqDAmDupqO446RdPHJpJ7sIxPNubZjqJcSIuAcpkvthSwde9hZozvhb+f9gI8zTn9OpMYFcZLyzP1wnJeTIuAcgljDM8uTSe2QyiTdN1gj+TnJ9wyNpHNeYf4Kn2/7TjKRbQIKJf4Kn0/67MPctu4XgTqpaI91uRh3egcHsxLeikJr6U/ncolnl2yg64RdSceKc8VHODPjaPjWbljH5tyi23HUS6gRUC1uNU7i/h2ZxHTxyYQHKDXCPJ015zRg7bBAby8MtN2FOUCWgRUi3tuaTodw4KYpstGeoV2oYFcc0YPFm7II7uo1HYc1cK0CKgWtT77ICu2F/KrMQmEBmkvwFvcODoOAV79cqftKKqFaRFQLer5pem0Cw3kFyO1F+BNotuFMjkphvfWZHPgSKXtOKoFaRFQLWbr3kN8vjmfG0bFER4SaDuOamHTxyZQVlXDv77ZZTuKakFaBFSLeX5pBmFB/tw4Os52FOUCfbuGM6FvFG98nUV5VY3tOKqFaBFQLSKjsISFG/Zw7ZlxtG8TZDuOcpFbxiWy/0glH6zNsR1FtRAtAqpFzFqWQXCAH78aE287inKhM+I7MDS2Pa+szKSmVi8l4Q20CKhmyy4q5d/f5TJtRA86tQ22HUe5kIhwy9gEsvaX8nnaXttxVAvQIqCa7aUVGYjUTRwq73fBwK707NiGF1foheW8QYNFQEReE5ECEdlUr+3vIrJVRDaIyEci0t5pjxORMhFJdW4v1tvnNBHZKCLpIvKM6MXlvUL+oXLmrsnhitNiiW4XajuOagX+fsLNYxJYn32Q1TuLbMdRzdSYnsAbwMQftS0GBhljhgDbgfvqPZdhjElybrfWa58F3Az0dm4/fk3lgWavyKTGGG4bl2g7impFV5zWnY5hQby0Qi8l4ekaLALGmBVA0Y/aPjfGHF189Bug+8leQ0SigQhjzDemrv/4FjClaZGVu9hfUsHb3+5iclI3enRsYzuOakUhgf5cPyqOJVsL2J5/2HYc1QwtMSfwS+A/9b6OF5HvRGS5iIxx2mKA+seU5ThtxyUi00UkRURSCgsLWyCicoXXvtpJRXUtt4/vZTuKsuDakT0JDfRntvYGPFqzioCI/A9QDbztNOUBPYwxw4B7gHdEJOJUX9cYM9sYk2yMSY6KimpOROUixaVVvPn1Li4aFE2vzm1tx1EWRIYFcdXpscxPzSWvuMx2HNVETS4CInIDcAnwc2eIB2NMhTFmv/N4LZAB9AFyOXbIqLvTpjzUm6uyKKmoZsYE7QX4spvOiqfWwOtfZdmOopqoSUVARCYCvwcmGWNK67VHiYi/8ziBugngTGNMHnBIREY6RwVdB8xvdnplRUlFNa99tZNz+3dmQLdT7ugpLxLboQ0XD47mnW93c6i8ynYc1QSNOUT0XWAV0FdEckTkJuA5IBxY/KNDQccCG0QkFfgAuNUYc3RS+XbgFSCduh5C/XkE5UHe/mYXB0urtBeggLrzQ0oqqnnn2922o6gmCGhoA2PMtOM0v3qCbecB807wXAow6JTSKbdTXlXDyyt3MqZ3J4b1iLQdR7mBQTHtOKtXJ177cic3jo7T1eQ8jJ4xrE7Je2uy2VdSob0AdYxbxiVQcLiC+d/tsR1FnSItAqrRKqtreXF5BqfHRXJGfAfbcZQbOatXJwZERzB7ZSa1emE5j6JFQDXah+tyyCsu546ze6NX/VD1iQi3jEsgvaCEJVsLbMdRp0CLgGqU6ppaZi3PYEj3dozt3cl2HOWGLhocTUz7UF5akWE7ijoFWgRUoyzckMeu/aXcMaGX9gLUcQX6160nsSbrAGt3HbAdRzWSFgHVoNpaw3NL0+nbJZxz+3exHUe5sanJsbQLDWS29gY8hhYB1aBFaXtJLyhhxtm98PPTXoA6sbDgAK47syefb84no7DEdhzVCFoE1EkZU9cLiO8UxsWDo23HUR7g+lFxBPr78cpKvbCcJ9AioE5q2bZC0vYc4vbxifhrL0A1Qqe2wVx5Wnfmrcul4HC57TiqAVoE1AkZY3hmyQ5i2ocyZdgJr/yt1E/8akwCVTW1vPl1lu0oqgFaBNQJrcrYz3e7D3Lr+EQC/fW/imq8+E5hTBzYlf9btYuSiuqGd1DW6E+2OqFnl6TTObyua6/UqZo+NoFD5dW8tybbdhR1EloE1HGt3VXEqsz9TB+bQEigXhBMnbphPSIZEd+BV1dmUlVTazuOOgEtAuq4nluSToewIK45o4ftKMqD3TougT3F5SzcoBeWc1daBNRPbMotZum2Qm46K542QQ1ebVypExrfpzN9urTlpeWZOAsQKjejRUD9xHNL0gkPCeDaM3vajqI8nJ+fcPOYBLbuPcyKHftsx1HHoUVAHWN7/mE+S9vLjaPiiAgJtB1HeYHJSTF0iQjmpeV6KQl3pEVAHeP5pem0CfLnxtHxtqMoLxEU4MdNZ8XzdcZ+NuYU246jfkSLgPpe1r4jfLx+D9eO7ElkWJDtOMqLTBvRg/DgAL3MtBtqVBEQkddEpEBENtVr6yAii0Vkh3Mf6bSLiDwjIukiskFEhtfb53pn+x0icn3LfzuqOWYtyyDA34+bxmgvQLWs8JBArhnZg0835rF7f6ntOKqexvYE3gAm/qjtj8AXxpjewBfO1wAXAr2d23RgFtQVDeBB4AxgBPDg0cKh7Ms9WMa8dTlMOz2WzuEhtuMoL/TL0fH4+wmvfKkXlnMnjSoCxpgVQNGPmicDbzqP3wSm1Gt/y9T5BmgvItHABcBiY0yRMeYAsJifFhZlyUvLMxCB6eMSbUdRXqpLRAg/GxbD3JRsio5U2o6jHM2ZE+hijMlzHu8Fjq42EgPUP088x2k7UbuyrOBQOXPWZHP58O7EtA+1HUd5seljEyivquWtVVm2oyhHi0wMm7qzQFrsTBARmS4iKSKSUlhY2FIvq07glS93Ul1Ty23jtRegXKtX53DO7d+ZN7/OoqyyxnYcRfOKQL4zzINzX+C05wKx9bbr7rSdqP0njDGzjTHJxpjkqKioZkRUDSk6Usm/vtnFpKHd6NkxzHYc5QNuGZfIgdIqPlirF5ZzB80pAguAo0f4XA/Mr9d+nXOU0Eig2Bk2WgScLyKRzoTw+U6bsuj1r3ZSWlnDjAm9bEdRPiK5ZyTDerTn5ZV1PVBlV2MPEX0XWAX0FZEcEbkJeAw4T0R2AOc6XwN8CmQC6cDLwO0Axpgi4GFgjXP7s9OmLCkuq+KNr7K4cFBXencJtx1H+QgR4ZaxiewuKuWztL224/i8Rl0dzBgz7QRPnXOcbQ0w4wSv8xrwWqPTKZf6v1VZHK6o1l6AanXnDehCQqcwXlqeycWDoxHRpUtt0TOGfVRpZTWvfrmTs/t1ZlBMO9txlI/x9xN+NSaBjbnFrMrcbzuOT9Mi4KPe+XY3B0qrtBegrLlseAyd2gbx0nI9ecwmLQI+qLyqhpdWZDIqsSOn9dSTtpUdIYF1Fypcvr2QLXmHbMfxWVoEfND7KdkUHq7gjrO1F6Ds+sUZPWkT5M/LK7Q3YIsWAR9TVVPLi8szOa1nJGcmdLQdR/m4dm0Cufr0HixYv4fcg2W24/gkLQI+5qPvcsk9WMYdZ/fSIzKUW7hpTDwGeO3Lnbaj+CQtAj6kptbwwtJ0BsVEML6Pnomt3ENM+1AmDe3GnNW7OViqF5ZrbVoEfMjCDXvI2l/KHRO0F6Dcyy3jEiirquGRT7bYjuJztAj4iNpaw/NL0+nduS3nD+hqO45Sx+jXNYLbx/fig7U5fK5nEbcqLQI+YvGWfLbnl3DH2b3w89NegHI/vz6nNwOiI7j/o43sL6mwHcdnaBHwAcYYnluSTlzHNlw8ONp2HKWOKyjAj6euGsqhsmru/2gjdVegUa6mRcAHfLZpLxtzi7ltfCIB/vpPrtxXv64R3Ht+Hxal5fPRd8e90rxqYfobwcvlHyrn/o82MigmgsuGd7cdR6kG/WpMAqfHRfLg/DT26LkDLqdFwIvV1hp++/56yqpq+MdVwwjUXoDyAP5+whNXDqXGGH73wXpqa3VYyJX0t4IXe+PrLFbu2Mf/XjyAXp3b2o6jVKP17BjG/1zcn6/S9/N/3+yyHceraRHwUlv3HuKxz7Zybv/O/PyMHrbjKHXKrhnRg3F9ovjrf7aQWVhiO47X0iLghcqrarh7TioRIQE8dvkQPTFMeSQR4fErhhAc4M89c9frUpQuokXAC/190Ta27j3M368YSqe2wbbjKNVkXSJCeHjKIFKzD/KSXmnUJbQIeJmVOwp59cudXHdmTyb062w7jlLNNmloNy4ZEs3Ti7ezKbfYdhyv0+QiICJ9RSS13u2QiNwtIjNFJLde+0X19rlPRNJFZJuIXNAy34I66sCRSn77/np6dW7L/Rf1tx1HqRbz8ORBRIYFce/c9VRU19iO41WaXASMMduMMUnGmCTgNKAU+Mh5+umjzxljPgUQkQHA1cBAYCLwgoj4Ny++OsoYw30fbqToSCX/vDqJkED9aJX3iAwL4vHLh7At/zBPLd5uO45XaanhoHOADGPMyY7lmgzMMcZUGGN2AunAiBZ6f5/3fkoOn6Xt5bfn92VgN104XnmfCf06M21ELLNXZLImq8h2HK/RUkXgauDdel/fISIbROQ1ETm6iG0MkF1vmxynTTVT1r4jzPw4jTMTOnLzmATbcZRymf+5eADdI0O5d+56jlRU247jFZpdBEQkCJgEvO80zQISgSQgD3iyCa85XURSRCSlsLCwuRG9WlVNLXe/l0qAn/Dk1KF6hVDl1doGB/DklUlkHyjlL5/q2gMtoSV6AhcC64wx+QDGmHxjTI0xphZ4mR+GfHKB2Hr7dXfafsIYM9sYk2yMSY6K0hWwTubZJemkZh/k0csG0619qO04SrnciPgO3DwmgXe+3c3SbQW243i8ligC06g3FCQi9a9V/DNgk/N4AXC1iASLSDzQG1jdAu/vs9buKuK5JTu4bHgMlwzpZjuOUq3mnvP60KdLW/7wwQZdkrKZmlUERCQMOA/4sF7z4yKyUUQ2ABOA3wAYY9KAucBm4DNghjFGj/VqosPlVdz9XioxkaE8NGmg7ThKtaqQQH+emppE0ZFKHpifZjuORwtozs7GmCNAxx+1XXuS7f8C/KU576nqzFywmdwDZcy95UzCQwJtx1Gq1Q2Kacdd5/TmycXbOX9AFy4dqr3hptAzhj3QJxvymLcuhzsm9CI5roPtOEpZc9v4RIbGtueB+ZvIP1RuO45H0iLgYfKKy7j/o40MjW3Pnef0th1HKasC/P14aupQyipr+MO8DbokZRNoEfAgtbWGe+eup6qmln9elaSLxCgFJEa15b4L+7FsWyFz1mQ3vIM6hv4W8SCvfJnJ1xn7efDSAcR1CrMdRym3cd2ZcYxK7MjDCzeze3+p7TgeRYuAh0jbU8zfF23jgoFdmJoc2/AOSvkQPz/h71cOxV+E376/nhpdkrLRtAh4gPKqGu6ak0pkmyAeu0wXiVHqeGLah/LgpIGsziri1S917YHG0iLgAf766RbSC0p4cupQIsOCbMdRym1dPjyG8wd04YlF29m297DtOB5Bi4CbW7qtgDdX7eKXo+MZ01svoaHUyYgIj142mPCQAO6Zm0pltS5J2RAtAm5sX0kFv3t/A/26hvP7iX1tx1HKI3RqG8yjlw0mbc8hnl2yw3Yct6dFwE0ZY/jjvA0cKq/iH7pIjFKn5IKBXbl8eHdeWJbBd7sP2I7j1rQIuKl3Vu/mv1sK+MPEfvTrGmE7jlIe58FJA+gSHsy9c9dTVqmXKTsRLQJuKKOwhIcXbmZM707cOCrOdhylPFJESCB/v3IomfuO8LfPttqO47a0CLiZyupa7p6TSmigP09cqYvEKNUco3t14oZRcbzxdRZfpe+zHcctaRFwM//473Y25hbz18uG0CUixHYcpTzeHyb2I6FTGL97fz2Hyqtsx3E7WgTcyLeZ+5m1PIOrkmOZOKir7ThKeYXQIH+euiqJ/MMVPLRgs+04bkeLgJsoLqvinrnr6dmhDX+6dIDtOEp5laSdl0ceAAAMw0lEQVTY9tw+PpF563JYlLbXdhy3okXATfxp/ib2Hirn6auSCAtu1lo/SqnjuPPs3gzsFsH9H25kX0mF7ThuQ4uAG5ifmsv81D3cdU5vhvWItB1HKa8UFODHU1OTOFxezX0fbtS1BxxaBCzLOVDK/360idN6RnL7+ETbcZTyan27hvPbC/qweHM+89bl2o7jFrQIWFRTa7jnvfUY4B9XJRGgi8Qo5XI3nZXAiLgOPLQgjdyDZbbjWNfs3zoikiUiG0UkVURSnLYOIrJYRHY495FOu4jIMyKSLiIbRGR4c9/fk724PIPVWUU8NGkgsR3a2I6jlE/w9xOeuHIoNcbwu/fXU+vjaw+01J+eE4wxScaYZOfrPwJfGGN6A184XwNcCPR2btOBWS30/h5nQ85Bnl68nYuHRHPZ8BjbcZTyKT06tuGBSwbwdcZ+3lqVZTuOVa4af5gMvOk8fhOYUq/9LVPnG6C9iES7KIPbKq2s5u45qUSFB/PolMG6SIxSFlx9eizj+0bx1/9sJb2gxHYca1qiCBjgcxFZKyLTnbYuxpg85/FeoIvzOAaovxJ0jtN2DBGZLiIpIpJSWFjYAhHdyyOfbGHn/iM8OXUo7doE2o6jlE8SER6/fAihQf7c+/56qmt8c+2BligCZxljhlM31DNDRMbWf9LUHYd1SoNuxpjZxphkY0xyVJR3LaSyeHM+73y7m+ljEhiV2Ml2HKV8WueIEB6ePIj12QeZtSzDdhwrml0EjDG5zn0B8BEwAsg/Oszj3Bc4m+cC9VdJ7+60+YSCw+X8Yd4GBkRHcM/5fWzHUUoBlw7txqVDu/HPL3awKbfYdpxW16wiICJhIhJ+9DFwPrAJWABc72x2PTDfebwAuM45SmgkUFxv2MirGWP43fsbOFJRzTPTkggO0EVilHIXD08eSIewIO6Zm0p5lW+tPdDcnkAX4EsRWQ+sBj4xxnwGPAacJyI7gHOdrwE+BTKBdOBl4PZmvr/HeGvVLpZvL+R/Lu5Pr87htuMopepp3yaIv10xhO35JTy1eLvtOK2qWRepMcZkAkOP074fOOc47QaY0Zz39EQ78g/z6KdbmNA3imtH9rQdRyl1HBP6duaaM3owe0UmAL+7oC+BPnACp16pzMUqqmv49ZxU2gYH8PgVQ/VwUKXc2IOXDsBPYPaKTFKyinjumuF0ax9qO5ZLeX+Zs+zJz7ezJe8Qj18xhKjwYNtxlFInERzgzyNTBvPMtGFs23uYi55ZyZKt+bZjuZQWARfZV1LBb99fz+wVmfz8jB6c079LwzsppdzCpKHdWPjrMUS3C+WXb6Tw1/9socpLzyPQ4aAWVlNreOfbXfx90TZKK2u4dVwid5/b23YspdQpiu8Uxke3j+LhhZt5aXkmKVkHeHbaMK8bHhJ3v6Z2cnKySUlJsR2jUb7bfYAH5m9iU+4hRiV25M+TB+qRQEp5gQXr93DfvA0EBvjx9NQkJvTrbDvSSYnI2nrXcjsp7Qm0gKIjlTz+2VbmrMmmS0Qwz04bxiVDonUSWCkvMWloNwZ1i2DGO99x4xtruHVcIvee38crjh7SItAMtbWGOWuyeXzRVg6XV3PzmHjuOrcPbXV5SKW8TkJUWz66fRR/XriZF5dnkJJVxDNeMDykw0FNtCHnIA/8exPrc4oZEd+BhycPom9XHfpRyhfMT83l/g83fr9kpbsND+lwkAsdLK3k74u28c7q3XQMC+YfVyUxOambDv0o5UMmJ8UwOKYdt7+9zuOHh7QINFJtreGDtTk89tlWDpZWcsOoOH5zXh8iQvRS0Er5ooSotvx7xmge+viH4aFnrxlGdDvPGh7S4aBGSNtTzAP/3sS63QdJ7hnJnycPYkC3CKuZlFLu45jhoauSmNDX7vCQDge1kOKyKp5evJ23VmUR2SaIJ64cymXDYvDz06EfpdQPJifFMCimHTPeXseNr6/htvGJ3HteHwI8YHhIi8BxGGP4cF0uf/3PFoqOVPKLkT2597y+ugqYUuqEEusND81a9sPRQ+4+PKTDQT+yde8hHvj3JtZkHSAptj2PTBnEoJh2rfb+SinP9+/vcrn/o42EBPrz1NShjG/l4SEdDmqCw+VVPL14B2+uyiIiJIC/XT6YK0+L1aEfpdQpmzIshsHd64aHbnh9DbePT+QeNx0e8vkiYIxhwfo9PPLJFvaVVHD16T34/QV9iQwLsh1NKeXBfhgeSuOFZRmsySri2WnD6douxHa0Y/j0cNCO/MM8MH8T32QWMaR7O/48eRBJse1d8l5KKd/V2sNDOhzUgCMV1TzzxQ5e/XInYcEBPDJlENNG9MBfh36UUi4wZdgPRw/d8PoaZkxI5DfnusfwkE8VAWMMn2zM45GFW9h7qJypyd35w8R+dGyri70opVyrV+cfhoeeX5rBmp0HeGbaMOvDQ00uQyISKyJLRWSziKSJyF1O+0wRyRWRVOd2Ub197hORdBHZJiIXtMQ30FgZhSVc++pq7njnOzqEBTHvtlE8fsVQLQBKqVYTGuTPY5cP4emrhrJpTzEXPbOS5dsLrWZq8pyAiEQD0caYdSISDqwFpgBTgRJjzBM/2n4A8C4wAugG/BfoY4ypOdn7NHdOoLSymueWpPPyykxCAv357fl9+cXInjr0o5SyKr2ghBlvr2Nb/uEWHx5qlTkBY0wekOc8PiwiW4CYk+wyGZhjjKkAdopIOnUFYVVTMzSQj0Vp+Ty8cDO5B8u4bHgM913YX9f5VUq5haPDQzMXOMNDzsplXSJad3ioRcqOiMQBw4BvnaY7RGSDiLwmIpFOWwyQXW+3HE5eNJqsuKyKG15fw63/Wkt4SABzbzmTp6YmaQFQSrmV0CB//nbFEJ6aOpSNOcVc9M+VrGjl4aFmFwERaQvMA+42xhwCZgGJQBJ1PYUnm/Ca00UkRURSCgtP/QMJDw7AAA9cMoCFd57FiPgOp/waSinVWi4b3p2P7xxNp7bBXP/6ap5YtI3qVlrYvlnnCYhIILAQWGSMeeo4z8cBC40xg0TkPgBjzF+d5xYBM40xJx0OauqcgDFGr/GvlPIoZZU1zFyQxnsp2YyI78DrN5xOWBNWKjyVOYHmHB0kwKvAlvoFwJkwPupnwCbn8QLgahEJFpF4oDewuqnv34h8rnpppZRyifrDQ/Edw2gT5O/y92zOeQKjgWuBjSKS6rTdD0wTkSTAAFnALQDGmDQRmQtsBqqBGQ0dGaSUUr7osuHduWx491Z5r+YcHfQlcLw/tz89yT5/Af7S1PdUSinVsuyfs6yUUsoaLQJKKeXDtAgopZQP0yKglFI+TIuAUkr5MC0CSinlw7QIKKWUD3P75SVFpBDYZTtHM3UC9tkO4Sb0sziWfh7H0s/jB835LHoaY6Ias6HbFwFvICIpjb2Oh7fTz+JY+nkcSz+PH7TWZ6HDQUop5cO0CCillA/TItA6ZtsO4Eb0sziWfh7H0s/jB63yWeicgFJK+TDtCSillA/TIuAiIhIrIktFZLOIpInIXbYzuQMR8ReR70Rkoe0sNolIexH5QES2isgWETnTdiabROQ3zs/JJhF5V0Rad7V1y5z12AtEZFO9tg4islhEdjj3kSd7jabSIuA61cC9xpgBwEhghogMsJzJHdwFbLEdwg38E/jMGNMPGIoPfyYiEgP8Gkg2xgwC/IGr7aZqdW8AE3/U9kfgC2NMb+AL5+sWp0XARYwxecaYdc7jw9T9kMfYTWWXiHQHLgZesZ3FJhFpB4ylbnlWjDGVxpiDdlNZFwCEikgA0AbYYzlPqzLGrACKftQ8GXjTefwmMMUV761FoBWISBwwDPjWbhLr/gH8Hqi1HcSyeKAQeN0ZGntFRMJsh7LFGJMLPAHsBvKAYmPM53ZTuYUuxpg85/FeoIsr3kSLgIuJSFtgHnC3MeaQ7Ty2iMglQIExZq3tLG4gABgOzDLGDAOO4KKuvidwxronU1ccuwFhIvILu6nci6k7jNMlh3JqEXAhEQmkrgC8bYz50HYey0YDk0QkC5gDnC0i/7IbyZocIMcYc7Rn+AF1RcFXnQvsNMYUGmOqgA+BUZYzuYN8EYkGcO4LXPEmWgRcRESEujHfLcaYp2znsc0Yc58xprsxJo66Sb8lxhif/GvPGLMXyBaRvk7TOcBmi5Fs2w2MFJE2zs/NOfjwRHk9C4DrncfXA/Nd8SZaBFxnNHAtdX/xpjq3i2yHUm7jTuBtEdkAJAGPWs5jjdMj+gBYB2yk7veST505LCLvAquAviKSIyI3AY8B54nIDup6S4+55L31jGGllPJd2hNQSikfpkVAKaV8mBYBpZTyYVoElFLKh2kRUEopH6ZFQCmlfJgWAaWU8mFaBJRSyof9Py5jVazHzcHtAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "hist = generate_discrete_gaussian_distribution(10000, 5.5, 2, 1, 10)[1]\n",
    "print(hist)\n",
    "plt.plot(hist.keys(), hist.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_discrete_skewed_gaussian_distribution(num_values, a=4, mean=5.5, std=2, low=1, high=10):\n",
    "    \"\"\"\n",
    "    generates a discrete skewed gaussian distribution\n",
    "    :param num_values: int, number of samples to take from discrete distribution\n",
    "    :param a: float, skew of the distribution, positive is right leaning, negative is left learning\n",
    "    :param mean: float, mean of gaussian\n",
    "    :param std: int, standard deviation of gaussian\n",
    "    :param low: int, lowest value the discrete distribution will have\n",
    "    :param high: int, highest value the discrete distribution will have\n",
    "    :return: skewed_gaussian_lst, list, list of values in distributions\n",
    "            bins, dict, map between the values and counts in distribution\n",
    "    >>> np.random.seed(1)\n",
    "    >>> generate_discrete_skewed_gaussian_distribution(num_values=10, a=4, mean=0, std=1, low=-4, high=4)[0]\n",
    "    [2.0, 1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 1.0, 0.0, 0.0]\n",
    "    >>> generate_discrete_skewed_gaussian_distribution(num_values=10, a=4, mean=0, std=1, low=-4, high=4)[1]\n",
    "    {-4: 0.0, -3: 0.0, -2: 0.0, -1: 0.0, 0: 3.0, 1: 7.0, 2: 0.0, 3: 0.0, 4: 0.0}\n",
    "    \"\"\"\n",
    "    skewed_lst = skewnorm(a=a, loc=mean, scale=std).rvs(num_values)\n",
    "    bins = dict(zip(list(range(low, high + 1)), np.zeros(high - low + 1)))\n",
    "    skewed_gaussian_lst = []\n",
    "    for x in skewed_lst:\n",
    "        if x <= low:\n",
    "            skewed_gaussian_lst.append(low)\n",
    "            bins[low] += 1\n",
    "        elif x >= high:\n",
    "            skewed_gaussian_lst.append(high)\n",
    "            bins[high] += 1\n",
    "        else:\n",
    "            skewed_gaussian_lst.append(round(x))\n",
    "            bins[round(x)] += 1\n",
    "    return skewed_gaussian_lst, bins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{1: 5.0, 2: 342.0, 3: 2028.0, 4: 3065.0, 5: 2418.0, 6: 1366.0, 7: 545.0, 8: 181.0, 9: 39.0, 10: 11.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11129d2e8>]"
      ]
     },
     "execution_count": 271,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VNX9//HXJxthyZ4QIAmEZSCEVYwsCqgEEGwralurtWq/37Z0wQp2Vb/ffu1mtf1VrbuiolitS12pUk0KyOYGKFsSICEsSSAbITvZz++P3GhQkJDM5M7M/Twfj3lk5sydmc+MMu97zj1zrhhjUEop5TwBdheglFLKHhoASinlUBoASinlUBoASinlUBoASinlUBoASinlUBoASinlUBoASinlUBoASinlUEF2F/BlYmNjTXJyst1lKKWUT9m2bVu5MSbuTNt5dQAkJyezdetWu8tQSimfIiKHurKdDgEppZRDaQAopZRDaQAopZRDaQAopZRDaQAopZRDaQAopZRDaQAopZRDaQCoXpNXWsO6vaV2l6GUsmgAqF7R2mb44d+38V9PbSEzu8TucpRSdCEARCRURD4SkR0ikiUiv7Pah4vIhyKSJyIvikiI1d7Hup1n3Z/c6blutdr3isglnnpTyvv8a8cR9pfVETugDze/uJ280hq7S1LK8brSA2gE5hhjJgGTgQUiMh34M3CvMWYUcBz4nrX994DjVvu91naISCpwNTAOWAA8LCKB7nwzyju1tLZx/5pcUgaF8caNFxAaHMAPntlG1Ylmu0tTytHOGACmXa11M9i6GGAO8LLVvhK43Lq+yLqNdX+6iIjV/oIxptEYcwDIA6a65V0or7ZqxxHyy+tYNtdFQmRfHvnOuRQer+em5z+htc3YXZ5SjtWlYwAiEigi24FSIBPYD1QaY1qsTQqBBOt6AlAAYN1fBcR0bj/FYzq/1mIR2SoiW8vKys7+HSmv0rH3nzo4nPmpgwA4Lzma3142jvX7yvh/7+y1uUKlnKtLAWCMaTXGTAYSad9rT/FUQcaY5caYNGNMWlzcGVczVV7utU+KOHisnmVzXQQEyKft104bxrenDeXR9ftZteOIjRUq5VxnNQvIGFMJrANmAJEi0rGcdCJQZF0vApIArPsjgGOd20/xGOWHmlvbeGBtHuMTwpmXGv+F+3/7tXGclxzFr17ewe6iKhsqVMrZujILKE5EIq3rfYF5QA7tQfANa7MbgDes66us21j3rzXGGKv9amuW0HDABXzkrjeivM9rHxdxuKKeZemjaT8MdLKQoAAevvZcovqF8MO/b+NYbaMNVSrlXF3pAQwG1onITmALkGmMeRP4NfAzEcmjfYz/SWv7J4EYq/1nwC0Axpgs4CUgG3gbWGKMaXXnm1Heo7m1jfvX5jIxMYL0sQNPu11cWB+WX5dGeW0jP37uY5pb23qxSqWcTdp3zr1TWlqa0TOC+abnPzrMra/u4qnvnsfFKacPgA6vf1LEshe3c/2MYfx+0fheqFAp/yUi24wxaWfazqtPCal8U1NLGw+uzWNyUiQXjenagfzLz0kg60gVj288wLgh4XzrvKEerlIppUtBKLf757YCiipPsGyu65Rj/6fz6wUpzHLF8r+v72bboeMerFApBRoAys0aW1p5aG0e5wyN5MLRZzeNNygwgAeuOYfBEX350bPbKK5q8FCVSinQAFBu9tKWAo5UNfCzeaee+XMmkf1CePz6NOoaW/jhs9toaNZ5Akp5igaAcpuG5lYeWreftGFRzBwV2+3nGTMojHuumsyOgkr+57XdePNEBaV8mQaAcpsXtxRQXN3Azd3c++9swfhBLE138crHhTy1+aB7ClRKnUQDQLlFQ3MrD7+bx9TkaM4fGeOW51ya7mJ+ajx3rM5hc165W55TKfUZDQDlFv/48DAl1Y1u2fvvEBAg3POtyYyI7c+Sf3xMQUW9W55XKdVOA0D1WENzK4+s38/0EdHMcNPef4cBfYJ4/Po02toMP3hmK/VNLWd+kFKqSzQAVI89+8EhymoauXnuaI88f3Jsfx749hT2ldTwy3/u1IPCSrmJBoDqkRNNrTy6Pp/zR8YwbYR79/47u3B0HL9ekMJbu47y8Lv7PfY6SjmJBoDqkWc/OER5bfvYv6ctnj2CRZOH8NeMvazJ0RPLK9VTGgCq2+qbWnh0/X5muWI5Lzna468nIvz56xMZNyScpS9sJ6+09swPUkqdlgaA6rZn3j/Esbomlnlo7P9UQoMDeey6NPoEBbD4ma16YnmlekADQHVLXWMLyzfkM3t0HOcOi+rV1+44sfzhinqWvaAnllequzQAVLesfP8gFXVN3DzXZcvrTx0eze2XjWPd3jLuztATyyvVHXo+AHXWahqaWb4hn4vHxHHO0N7d++/sO9OGkn2kioff3U/qkHC+OnGIbbUo5Yu0B6DO2sr3DlJZ39yrY/+nIiL87rLxnDssil/+cyfZR6ptrUcpX6MBoM5KdUMzj288QHrKQCYlRdpdDiFBATzynSlE9A3mB89spaKuye6SlPIZGgDqrDy9+SBVJ+zf++9sYFgoj113LmW1jSzRE8sr1WUaAKrLqk408/jGfOalxjMhMcLuck4yKSmSu66cwPv5x7jjrRy7y1HKJ+hBYNVlKzYdoKahhWU2zfw5kyunJJJ1pJonNx0gdUg4V6Ul2V2SUl5NewCqS6rqm1mx6QCXjItn3BDv2vvv7NaFKcwcFcv/vrabjw/rieWV+jJnDAARSRKRdSKSLSJZIrLUav+tiBSJyHbrcmmnx9wqInkisldELunUvsBqyxORWzzzlpQnPLkpn5rGFq8a+z+VjhPLx0f04Ud/30ZJtZ5YXqnT6UoPoAX4uTEmFZgOLBGRVOu+e40xk63LagDrvquBccAC4GERCRSRQOAhYCGQClzT6XmUF6usb2LF5oNcOmEQYweH213OGUX1bz+xfG1jCz96dhuNLXpieaVO5YwBYIw5aoz52LpeA+QACV/ykEXAC8aYRmPMASAPmGpd8owx+caYJuAFa1vl5R7fmE9dUwtL071777+zlEHh3P3NSXxyuJLfvK4nllfqVM7qGICIJAPnAB9aTTeKyE4RWSEiHT8JTQAKOj2s0Go7XbvyYhV1TTy9+SCXThjMmEFhdpdzVhZOGMxNc0bx0tZCnnn/kN3lKOV1uhwAIjIAeAVYZoypBh4BRgKTgaPA3e4oSEQWi8hWEdlaVlbmjqdUPfD4xnzqm1tZlu6dM3/OZNnc0cwdG8/v38zmvf16YnmlOutSAIhIMO1f/s8ZY14FMMaUGGNajTFtwOO0D/EAFAGd598lWm2naz+JMWa5MSbNGJMWFxd3tu9HudGx2kZWvneQr04cgivet/b+OwQECPd+axLDY/uz5Dk9sbxSnXVlFpAATwI5xph7OrUP7rTZFcBu6/oq4GoR6SMiwwEX8BGwBXCJyHARCaH9QPEq97wN5QnLN+TT0NzKUh/d++8QFhrM8uvOpaXNsPjv2/TE8kpZutIDuAC4DpjzuSmffxGRXSKyE7gYuBnAGJMFvARkA28DS6yeQgtwI/AO7QeSX7K2VV6ovLaRZ94/xGWThjBq4AC7y+mxEXEDeOCac9hTXM2vXtYTyysFXfglsDFmEyCnuGv1lzzmDuCOU7Sv/rLHKe/x2Pr9NLa0cpOP7/13dtGYgfzqkhT+/PYeJiVG8oPZI+wuSSlb6S+B1ReU1jTw9w8OcfnkBEbE+f7ef2c/unAE6SkDuX9NLpX1unKocjYNAPUFj76bT3Or4ad+tPffQUT4xSVjqGlsYcWmA3aXo5StNADUSUqrG3juw0NccU4Cw2P7212OR4wdHM6CcYN4avNBqur1pPLKuTQA1Ekefnc/LW2Gn84ZZXcpHnVTuqu9F7BZewHKuTQA1KeKqxr4x0eH+fqUBIbF+Ofef4fUIeFcMi6eFZsPUHVCewHKmTQA1KceeTePtjbDT+f439j/qdyU7qKmoYWntBegHEoDQAFwpPIEz39UwDfTEkmK7md3Ob1i3JAI5qXG8+Qm7QUoZ9IAUAA8/G4eBsOSi/177P/zllq9gKc3H7S7FKV6nQaAoqjyBC9uKeCbaUkkRjlj77/D+IQI5o6N58lN+VQ3aC9AOYsGgOKhdXkAjtv777Bsrotq7QUoB9IAcLiCinpe2lLA1ecNJSGyr93l2KK9FzCQJzcd0F6AchQNAId7aF0eASL85OKRdpdiq6Xpo6k60cxK7QUoB9EAcLCCinpe3lbINVOTGBzhzL3/DhMSI0hPGcgTmw5Qo70A5RAaAA72wNpcAgKEnzh07P/zls51UXWiWU8fqRxDA8ChDpbX8crHRVw7bSjx4aF2l+MVJiZGMidlII9vzKe2UU8ao/yfBoBDPbA2j6AA4ccXOnvs//OWpruorG9m5XsH7S5FKY/TAHCgA+V1vPZJId+ZPoyBuvd/kklJkVw0Jk57AcoRNAAc6IE1uYQEBfAj3fs/pY5ewDPvH7S7FKU8SgPAYfaX1fL69iKumz6MuLA+dpfjlc4ZGsWFo+N4fEM+ddoLUH5MA8Bh7l+TS5+gQH6oe/9faulcF8frdUaQ8m8aAA6SV1rDqh1HuP78YcQO0L3/LzNlaBSzR7cfC9BegPJXGgAOct+aPPoGB/LD2br33xVL011U1DXx7AfaC1D+SQPAIfaV1PDmziPccH4y0f1D7C7HJ5w7LIpZrliWb8invkl7Acr/nDEARCRJRNaJSLaIZInIUqs9WkQyRSTX+htltYuI3C8ieSKyU0SmdHquG6ztc0XkBs+9LfV59/0nl37BgSyeNcLuUnzKsrkujmkvQPmprvQAWoCfG2NSgenAEhFJBW4B1hhjXMAa6zbAQsBlXRYDj0B7YAC3A9OAqcDtHaGhPGtPcTVv7TrKf10wnCjd+z8r5w6LZuaoWB5br70A5X/OGADGmKPGmI+t6zVADpAALAJWWputBC63ri8CnjHtPgAiRWQwcAmQaYypMMYcBzKBBW59N+qUXviogJCgAL4/a7jdpfikpVYv4LkPDttdilJudVbHAEQkGTgH+BCIN8Ycte4qBuKt6wlAQaeHFVptp2tXHmSMITO7hFmjYonsp3v/3XFecjQXjIrhsQ37OdHUanc5SrlNlwNARAYArwDLjDHVne8zxhjAuKMgEVksIltFZGtZWZk7ntLRso9WU1R5gvnj4s+8sTqtpemjKa9t4rkP9ViA8h9dCgARCab9y/85Y8yrVnOJNbSD9bfUai8Ckjo9PNFqO137SYwxy40xacaYtLi4uLN5L+oUMrNLEIE5KRoAPTF1eDTnj4zh0fX52gtQfqMrs4AEeBLIMcbc0+muVUDHTJ4bgDc6tV9vzQaaDlRZQ0XvAPNFJMo6+DvfalMelJFVwrlDo3TZBzdYmu6ivLZRewHKb3SlB3ABcB0wR0S2W5dLgbuAeSKSC8y1bgOsBvKBPOBx4CcAxpgK4A/AFuvye6tNeUjh8Xqyj1YzL1X3/t1h2ogYZoyI4bEN+TQ0ay9A+b6gM21gjNkEyGnuTj/F9gZYcprnWgGsOJsCVfdlZpcAMH/cIJsr8R9L57q4evkH/OPDw/z3TJ1VpXyb/hLYj2VmlzBq4ACGx/a3uxS/MX1EDNNHRPPI+v3aC1A+TwPAT1XWN/HhgQrm6/CP2y1NH01ZTSPPf6S/C1C+TQPAT63bW0prm9Hxfw+YMTKGqcOjeeRd7QUo36YB4Kcys0sYGNaHSYmRdpfil5bNdVFa08gL2gtQPkwDwA81NLfy7t4y5qbGExBwuuP3qidmjIhharIeC1C+TQPAD72//xj1Ta06/u9BIsKyuS5Kqht5cUvBmR+glBfSAPBDGdnFDOgTxIyRMXaX4tdmjIzhvOQoHnl3P40t2gtQvkcDwM+0tRkys0u5cEwcfYIC7S7Hr4kIS9NHU1zdwEvaC1A+SAPAz3xSUEl5baMO//SSC0bFkDYsioe1F6B8kAaAn8nMLiEoQLhozEC7S3EEEWHpXBdHqxp4aWuh3eUodVY0APxMRnYx00fEENE32O5SHGPmqFimDI3k4XV52gtQPkUDwI/kldaSX1ana//3svYZQaM5WtXAP7UXoHyIBoAf6Vj8be5YDYDeNssVyzlWL6Cppc3ucpTqEg0AP5KZXcyEhAiGRPa1uxTH6egFHKlq4J/bdEaQ8g0aAH6itKaBTwoqde0fG812xTI5KZKH1+3XXoDyCRoAfmJNTinGoOP/NuqYEVRUeYKXt+mxAOX9NAD8REZWMUnRfRkTH2Z3KY520eg4JiVF8pAeC1A+QAPAD9Q2trB5/zHmpw6i/RTOyi4iwrL09l7Aqx9rL0B5Nw0AP7BhXxlNLW06/u8lLhoTx6TECB5cl0dzq/YClPfSAPADmdklRPULJm1YlN2lKD47FlB4XHsByrtpAPi45tY21uSUMCclnqBA/c/pLS4eM5CJiRE8sFZ7Acp76TeGj9tyoILqhhad/eNl2lcKbe8FvPZxkd3lKHVKGgA+LiO7hD5BAcxyxdpdivqcOSkDmZCgxwKU99IA8GHGGDKzS5jliqNfSJDd5ajP6egFHK6o57VPtBegvM8ZA0BEVohIqYjs7tT2WxEpEpHt1uXSTvfdKiJ5IrJXRC7p1L7AassTkVvc/1acJ+tINUWVJ3Ttfy+WPnYg4xPCeWhdHi3aC1Bepis9gKeBBadov9cYM9m6rAYQkVTgamCc9ZiHRSRQRAKBh4CFQCpwjbWt6oHM7BICpP1LRnknEeGmOS4OHavn9e1H7C5HqZOcMQCMMRuAii4+3yLgBWNMozHmAJAHTLUuecaYfGNME/CCta3qgYzsEs4dFkXMgD52l6K+xLzUeFIHh/PA2lztBSiv0pNjADeKyE5riKhjAnoC0HkpxEKr7XTtXyAii0Vkq4hsLSsr60F5/q2gop6co9XMTx1kdynqDDp+F3DoWD1vaC9AeZHuBsAjwEhgMnAUuNtdBRljlhtj0owxaXFxce56Wr/Tsfa//vrXN8xPjWes9gKUl+lWABhjSowxrcaYNuBx2od4AIqApE6bJlptp2tX3ZSZXcLo+AEkx/a3uxTVBR0zgg4eq2fVDu0FKO/QrQAQkcGdbl4BdMwQWgVcLSJ9RGQ44AI+ArYALhEZLiIhtB8oXtX9sp2tsr6Jjw5W6N6/j5mfGk/KoDAeXKszgpR36Mo00OeB94ExIlIoIt8D/iIiu0RkJ3AxcDOAMSYLeAnIBt4Gllg9hRbgRuAdIAd4ydpWdcPaPaW0thkd//cxAQHCsrku8svr+NdO7QUo+53x10PGmGtO0fzkl2x/B3DHKdpXA6vPqjp1ShlZJcSH92FCQoTdpaizND91ECmDwnhgTR6XTUogMECX71b20V8C+5iG5lY25JYxLzWeAP3y8DkBAcJN6VYvQI8FKJtpAPiYzXnl1De1Mk+Hf3zWgnGDGBMfxv1rc2ltM3aXoxxMA8DHZGaXENYniBkjYuwuRXXTp72AsjpW7dDJcMo+GgA+pLXN8J+cEi4cE0dIkP6n82ULxw9ifEI4d7y1h6r6ZrvLUQ6l3yI+ZHvBccprm5g/Tod/fF1AgHDXlRM5Xt/Enf/Osbsc5VAaAD4kI6uE4EDhojH6C2l/MD4hgu/PHM4LWwp4f/8xu8tRDqQB4COMMWRklzB9RAzhocF2l6PcZNnc0QyN7sdtr+2iobnV7nKUw2gA+Ij9ZbUcKK/Ttf/9TN+QQO64YjwHyut4YG2u3eUoh9EA8BEZ1uJvczUA/M4sVxxXTkngsfX55Byttrsc5SAaAD4iI6uEiYkRDI7oa3cpygN+85VUIvoGc8uru/S3AarXaAD4gNLqBrYXVOrwjx+L6h/C/30tlR0Flax876Dd5SiH0ADwAZk5HWv/6/RPf3bZpCFcODqOv2bspfB4vd3lKAfQAPABmdklDIvpx+j4AXaXojxIRPjj5eMxBn7z+m6M0aEg5VkaAF6utrGF9/KOMW9sPCK6+Ju/S4ruxy8uGcO6vWX8a+dRu8tRfk4DwMut31tGU2ub/vrXQb57fjKTEiP43aosjtc12V2O8mMaAF4uI7uY6P4hnDssyu5SVC8JDBDuvHIilSeauWO1LhOhPEcDwIs1t7axdk8p6SkD9cQhDpM6JJzFs0fw8rZCNueV212O8lMaAF7sw/wKahpa9Ny/DrU03UVyTPsyESeadJkI5X4aAF4sM7uY0OAAZrl08TcnCg0O5E9XTuDQsXr+tmaf3eUoP6QB4KWMMWRmlzDLFUffkEC7y1E2OX9kLFelJfLExgPsLqqyuxzlZzQAvFTWkWqOVDXor38Vt106lqh+wdz66i5aWtvsLkf5EQ0AL5WRVUyAQPpYDQCni+wXwm8vG8euoiqe1mUilBtpAHipjOwS0pKjie4fYncpygt8ZcJg0lMGcnfGPgoqdJkI5R5nDAARWSEipSKyu1NbtIhkikiu9TfKahcRuV9E8kRkp4hM6fSYG6ztc0XkBs+8Hf9QUFHPnuIaHf5RnxIR/nD5eAIEbnttly4TodyiKz2Ap4EFn2u7BVhjjHEBa6zbAAsBl3VZDDwC7YEB3A5MA6YCt3eEhvqijrX/dfqn6mxIZF9+eckYNuaW8/r2IrvLUX7gjAFgjNkAVHyueRGw0rq+Eri8U/szpt0HQKSIDAYuATKNMRXGmONAJl8MFWXJyCpmTHwYw2L6212K8jLXzUjmnKGR/OHNHCp0mQjVQ909BhBvjOlYqaoY6NhVTQAKOm1XaLWdrl19zvG6JrYcrGD+ON37V18UGCDcdeVEahqa+eOb2XaXo3xcjw8Cm/bBSLcNSIrIYhHZKiJby8rK3PW0PmPNnlLajA7/qNMbMyiMH104klc/KWL9Puf9G1Hu090AKLGGdrD+llrtRUBSp+0SrbbTtX+BMWa5MSbNGJMWF+e8X8BmZhczKDyUCQkRdpeivNiSi0cxIq4///PaLuqbWuwuR/mo7gbAKqBjJs8NwBud2q+3ZgNNB6qsoaJ3gPkiEmUd/J1vtalOGppb2bCvnHmpuva/+nKhwYHcecUECo+f4N5MXSZCdU9XpoE+D7wPjBGRQhH5HnAXME9EcoG51m2A1UA+kAc8DvwEwBhTAfwB2GJdfm+1qU425ZZzorlVx/9Vl0wbEcM1U4fy5KYD7CrUZSLU2Qs60wbGmGtOc1f6KbY1wJLTPM8KYMVZVecwGdnFhPUJYtrwGLtLUT7iloUprMkp4dev7OSNGy8gOFB/26m6Tv9v8RKtbYY1OaVcnDKQkCD9z6K6JqJvML+7bBzZR6t5ctMBu8tRPka/abzEJ4ePc6yuSWf/qLO2YPwg5qfGc2/mPg4dq7O7HOVDNAC8REZ2CcGBwkVjnDfzSfWMiPD7ReMJCQzQZSLUWdEA8ALGGDKyipkxMpaw0GC7y1E+aFBEKL9amMLmvGO8vK3Q7nKUj9AA8AJ5pbUcPFavi7+pHrl26lDShkVxx+ocymsb7S5H+QANAC+gi78pdwgIEO68cgL1ja38/l+6TIQ6Mw0AL5CRXcKkpEjiw0PtLkX5OFd8GD+5eCSrdhxh3Z7SMz9AOZoGgM1KqhvYUVCpwz/KbX580UhGDRzA/76+m7pGXSZCnZ4GgM0ydfhHuVmfoEDuunICRZUn+GvGXrvLUV5MA8BmmdklJMf0wzVwgN2lKD+SlhzNddOH8fR7B9leUGl3OcpLaQDYqKahmff26+JvyjN+tWAM8WGh3PLKTppb2+wuR3khDQAbrd9XRnOrYf64QXaXovxQWGgwv180jj3FNSzfkG93OcoLaQDYKCOrhJj+IUwZqqdHVp4xf9wgFo4fxH1rcskvq7W7HOVlNABs0tTSxrq9paSPHUhggA7/KM/53WXj6BOky0SoL9IAsMmHB45R09DCvFQd/lGeNTA8lNsuHcsH+RW8tLXgzA9QjqEBYJPM7BL6BgcyyxVrdynKAb6VlsTU4dHc8VYOpTUNdpejvIQGgA2MMWRmlzDLFUtocKDd5SgH6FgmoqGljd+t0mUiVDsNABvsLqrmaFWDzv5RvWpk3ABumjOKt3Yd5T/WDxCVs2kA2CAju5gAgTkpA+0uRTnM4tkjGRMfxm/e2E1NQ7Pd5SibaQDYIDO7hPOSo4nuH2J3KcphQoICuOvrEyiubuCv7+gyEU6nAdDLDh+rZ09xja79o2xzztAobpiRzDMfHGLboeN2l6NspAHQyzKyiwGYr9M/lY1+cckYBoeHcuurO2lq0WUinEoDoJdlZJeQMiiMoTH97C5FOdiAPkH88Yrx7Cup5dH1++0uR9mkRwEgIgdFZJeIbBeRrVZbtIhkikiu9TfKahcRuV9E8kRkp4hMcccb8CUVdU1sPViha/8rrzAnJZ6vThzMg2vzeG9/ud3lKBu4owdwsTFmsjEmzbp9C7DGGOMC1li3ARYCLuuyGHjEDa/tU9bklNBm0F//Kq9x+9fGkRjVl2uf+JA/rc6hsaXV7pJUL/LEENAiYKV1fSVweaf2Z0y7D4BIERnsgdf3WpnZJQyOCGV8QrjdpSgFQFxYH968aSbfnjqU5RvyWfTgZnKOVttdluolPQ0AA2SIyDYRWWy1xRtjjlrXi4GO8Y4EoPNCJIVWmyOcaGplQ26Zrv2vvE6/kCDuuGICT333PMprm1j04GYeW7+f1jZdOM7f9TQAZhpjptA+vLNERGZ3vtO0Lz14Vv8XichiEdkqIlvLysp6WJ732JRXTkNzm87+UV7r4pSBZNw8m4tT4rjz33u45vEPKKiot7ss5UE9CgBjTJH1txR4DZgKlHQM7Vh/S63Ni4CkTg9PtNo+/5zLjTFpxpi0uLi4npTnVTKyigkLDWLaiGi7S1HqtKL7h/Dod87lr9+cRPaRahbet5GXtxXqMtJ+qtsBICL9RSSs4zowH9gNrAJusDa7AXjDur4KuN6aDTQdqOo0VOTXWtsMa/aUMidlIMGBOvNWeTcR4RvnJvLvpbNIHRLOL/65gx8/+zEVdU12l6bcrCffRvHAJhHZAXwEvGWMeRu4C5gnIrnAXOs2wGogH8gDHgd+0oPX9inbDh2noq5Jf/2rfEpSdD+e/8F0brtEiY5SAAAK9UlEQVQ0hbV7Spl/7wbW7tFF5PxJUHcfaIzJByadov0YkH6KdgMs6e7r+bLM7GJCAgO4cLT/DGkpZwgMEBbPHsksVxw3v7id/356K9dOG8r/fGUs/UK6/fWhvISOR3iYMYaM7BJmjIwhLDTY7nKU6paxg8N5fckFLJ49gn98dJhL79vIJ4d1HSFfpwHgYbmltRw6Vs/8cTr8o3xbaHAgt106lud/MJ3mVsM3Hn2fezL20tyqawn5Kg0AD8vIal/8be5YDQDlH6aPiOHfy2Zx+eQE7l+bx5UPv0deaa3dZalu0ADwsMzsEiYnRRIfHmp3KUq5TXhoMHdfNYlHrp1C4fF6vnL/Rla+d1Cni/oYDQAP2lNczY7CKp39o/zWwgmDeWfZbGaMjOH2VVlcv+IjSqr1pPO+QgPAA5pb23j43Twue3AzEX2DuWzSELtLUspjBoaH8tR3z+MPl49ny8EK5t+7gTd3HrG7LNUFGgButquwikUPbuYvb+8lPWUgmTfPJila1/5X/k1EuG76MFbfNIvkmH7c+I9PWPbCJ1Sd0PMOezOdyOsmJ5pa+dt/9vHEpgPEWD+nXzBe1/1RzjIibgAv//h8HlqXxwNr8/jwQAV3f3MS54+Ktbs0dQraA3CD9/LKWXDfBh7bkM9VaYlk/uxC/fJXjhUcGMCyuaN59cfn0zc4kG8/8SF/eDObhmY914C30R5AD1TVN/On1Tm8uLWA5Jh+/OMH0zh/pO7pKAUwKSmSt26axZ3/zuHJTQfYmFvGPVdNZnxChN2lKYt487SttLQ0s3XrVrvLOKW3dx/lN29kUVHXxPdnDefmuaMJDQ60uyylvNK7e0v55cs7qaxv4uZ5o/nh7JEEBuh5MTxFRLZ1Okvj6bfTADg7pdUN/N8bWbydVUzq4HD+8o2JukejVBccr2vittd28e/dxaQNi+KeqyYzNEYnSHiCBoCbGWN4cUsBd6zOoamljWVzR/P9WcN1eWelzoIxhtc+KeL2N7JoM4bbvzaOb6Yl6lny3KyrAaDHALrgYHkdt766i/fzjzFteDR3fX0iw2P7212WUj5HRLhySiLTRsTw85e286tXdpKZU8KdV04gdkAfu8tzHO0BfImW1jae3HSAezL3ERIYwG1fGcu30pII0LFLpXqsrc2wYvMB/vL2XsL7BvHHyycwLzVejw24gfYAeijrSBW/fmUnu4uqmZ8azx8uH6/r+SjlRgEBwvdnjWCmK5ZlL2znR89uI7JfMBeMjGWWK5aZrlgSo/QYgSdpAHxOQ3Mr963JZfmGfKL6hfDwtVNYOH6QjlEq5SEpg8J548YLeHt3MRtzy9mYW8Zbu9rPFjsitj8zXbHMHBWr59TwAB0C6uSD/GPc+uouDpTXcVVaIrddOpbIfiG99vpKqfYDxXmltWzILWdTbhkf5FdwormVwADhnKRIZrnimOmKZVJiBEE6CeOUdBbQWahuaObO1Xt4/qPDDI3ux51XTuAC/em6Ul6hsaWVjw9VsimvjI255ewqqsIYCAsN4vyRMcxyxTHLFcuwGJ2Y0UEDoIsysor5zRu7Katp5Hszh3PzvNF6rlOlvNjxuiY27y9nU245G3PLKao8AcDQ6H7MdMUy2xXLjJGxRPR17nCRBsAZlNU08ttVWby16ygpg8L489cnMikp0iOvpZTyDGMMB8rrrGMH5by/v5y6plYCpH0pilmjYpk1Oo7JSZGO+s2OBsBpGGP457ZC7ngrhxNNrSyd62Lx7BGO+p9DKX/V3NrG9oJKNu4rY2NeOTsKKmkzMKBPENNHRH96/GBEbH+/ntihAXAKh4/Vc9tru9iUV855yVHceeVERg0c4LbnV0p5l6r6Zt7PL7cOKJdzuKIegITIvswcFcus0bFcMDKWqP7+NdlDA6CT1jbDU5sPcHfGPgIDhF8vTOHaqUP1B11KOcyhY+3DRZtyy9m8v5yahhZEYEJCBDNHtf/2YMrQKJ9f2NFrA0BEFgD3AYHAE8aYu063rTsCIOdoNbe8spMdhVWkpwzkj1eMZ3BE3x49p1LK97W0trGzqIqN+8rZlFfGx4craW1r/z7sExRAWGgQA/oE0b9P+9+w0M+uDwgNIuw093W+3j8kyJYdTa8MABEJBPYB84BCYAtwjTEm+1Tb9yQAGppbeXBtHo+u309E32B+e9k4vjpxsF+P+ymluq+moZkP8ivYc7Sa2sYWahpbqGtsobah03Xrdm1jC40tbV163v4hgQywwqQjPDqCJezT28EM6BP46fX+fQKJHdCH0fFh3Xov3roUxFQgzxiTDyAiLwCLgFMGQHcVVNRzw1MfkV9Wx5VTEvjNV1L9boxPKeVeYaHBzEuNZ15qfJe2b2pp+ywUOl+sgKhrbKGmoVNb02f3Haut/+y+xpZPex6dTUqK5I0lF7j7bZ6ktwMgASjodLsQmNZ5AxFZDCwGGDp0aLdeJD48lOSY/tz+tXFcODqum6UqpdTphQQFEBIU0uOdS2MMjS1t1DR81suoaWghJMjzMxO97hdPxpjlwHJoHwLqznOEBAWw4rvnubUupZTyBBEhNDiQ0OBA4sJ6d0ns3p78XgQkdbqdaLUppZTqZb0dAFsAl4gMF5EQ4GpgVS/XoJRSil4eAjLGtIjIjcA7tE8DXWGMyerNGpRSSrXr9WMAxpjVwOrefl2llFIn0wVwlFLKoTQAlFLKoTQAlFLKoTQAlFLKobx6NVARKQMO2V1HD8UC5XYX4UX08ziZfh6f0c/iZD35PIYZY864DIJXB4A/EJGtXVmUySn08ziZfh6f0c/iZL3xeegQkFJKOZQGgFJKOZQGgOctt7sAL6Ofx8n08/iMfhYn8/jnoccAlFLKobQHoJRSDqUB4CEikiQi60QkW0SyRGSp3TXZTUQCReQTEXnT7lrsJiKRIvKyiOwRkRwRmWF3TXYSkZutfye7ReR5EQm1u6beJCIrRKRURHZ3aosWkUwRybX+Rrn7dTUAPKcF+LkxJhWYDiwRkVSba7LbUiDH7iK8xH3A28aYFGASDv5cRCQBuAlIM8aMp32l4KvtrarXPQ0s+FzbLcAaY4wLWGPddisNAA8xxhw1xnxsXa+h/R94gr1V2UdEEoGvAE/YXYvdRCQCmA08CWCMaTLGVNpble2CgL4iEgT0A47YXE+vMsZsACo+17wIWGldXwlc7u7X1QDoBSKSDJwDfGhvJbb6G/AroM3uQrzAcKAMeMoaEntCRPrbXZRdjDFFwF+Bw8BRoMoYk2FvVV4h3hhz1LpeDHTtbPVnQQPAw0RkAPAKsMwYU213PXYQka8CpcaYbXbX4iWCgCnAI8aYc4A6PNC99xXW2PYi2oNxCNBfRL5jb1XexbRP13T7lE0NAA8SkWDav/yfM8a8anc9NroAuExEDgIvAHNE5Fl7S7JVIVBojOnoEb5MeyA41VzggDGmzBjTDLwKnG9zTd6gREQGA1h/S939AhoAHiIiQvsYb44x5h6767GTMeZWY0yiMSaZ9oN7a40xjt3DM8YUAwUiMsZqSgeybSzJboeB6SLSz/p3k46DD4p3sgq4wbp+A/CGu19AA8BzLgCuo31vd7t1udTuopTX+CnwnIjsBCYDf7K5HttYPaGXgY+BXbR/LznqV8Ei8jzwPjBGRApF5HvAXcA8EcmlvZd0l9tfV38JrJRSzqQ9AKWUcigNAKWUcigNAKWUcigNAKWUcigNAKWUcigNAKWUcigNAKWUcigNAKWUcqj/DwB3FGd4afKAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "hist = generate_discrete_skewed_gaussian_distribution(10000, 3, 3, 2, 1, 10)[1]\n",
    "print(hist)\n",
    "plt.plot(hist.keys(), hist.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_discrete_uniform_distribution(num_values, low=1, high=10):\n",
    "    \"\"\"\n",
    "    generates a discrete uniform distribution\n",
    "    :param num_values: int, number of samples to take from discrete distribution\n",
    "    :param low: int, lowest value the discrete distribution will have\n",
    "    :param high: int, highest value the discrete distribution will have\n",
    "    :return: uniform_lst, list, list of values in distribution\n",
    "            bins, dict, map between the values and counts in distribution\n",
    "    >>> generate_discrete_uniform_distribution(num_values=10, low=1, high=10)[0]\n",
    "    [2.0, 1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 1.0, 0.0, 0.0]\n",
    "    >>> generate_discrete_uniform_distribution(num_values=10, low=1, high=10)[1]\n",
    "    {-4: 0.0, -3: 0.0, -2: 0.0, -1: 0.0, 0: 3.0, 1: 7.0, 2: 0.0, 3: 0.0, 4: 0.0}\n",
    "    \"\"\"\n",
    "    uniform_lst = np.random.choice(range(low, high + 1), num_values, replace=True)\n",
    "    bins = dict(zip(list(range(low, high + 1)), np.zeros(high - low + 1)))\n",
    "    for x in uniform_lst: \n",
    "        bins[x] += 1\n",
    "    return uniform_lst, bins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{1: 1033.0, 2: 1006.0, 3: 988.0, 4: 1015.0, 5: 1008.0, 6: 955.0, 7: 970.0, 8: 1032.0, 9: 984.0, 10: 1009.0}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([<matplotlib.axis.YTick at 0x111a1c438>,\n",
       "  <matplotlib.axis.YTick at 0x111a16d30>,\n",
       "  <matplotlib.axis.YTick at 0x111a0dd68>,\n",
       "  <matplotlib.axis.YTick at 0x111a32c18>,\n",
       "  <matplotlib.axis.YTick at 0x111a3b160>,\n",
       "  <matplotlib.axis.YTick at 0x111a3b630>,\n",
       "  <matplotlib.axis.YTick at 0x111a3bb00>,\n",
       "  <matplotlib.axis.YTick at 0x111aa40b8>,\n",
       "  <matplotlib.axis.YTick at 0x111aa4550>,\n",
       "  <matplotlib.axis.YTick at 0x111aa4a58>],\n",
       " <a list of 10 Text yticklabel objects>)"
      ]
     },
     "execution_count": 303,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGwlJREFUeJzt3X1wXNd93vHvg7cFQQAkSIIURZAibdOSKVu2aERW4iZxLEdv9pj+I/UobWPGVcs2lR2/TW057URTu9NxHE8ceeJqylqMpKlHtiqrMcejRFZlO5pOoxeQciSRMiVUtkjwFRJfAJDE+69/3IPlggQFcBfEQrrPZ2Zn7z337N6zl+B97jl3gaOIwMzM8qem2g0wM7PqcACYmeWUA8DMLKccAGZmOeUAMDPLKQeAmVlOTRsAkrZJOiLp+ZKy90h6QtLPJXVJuiaVS9K3JHVLelbSxpLXbJb0Unpsvjgfx8zMZmomPYB7gBvPKvs68J8i4j3An6Z1gJuA9emxBbgLQNIS4A7gfcA1wB2S2iptvJmZlW/aAIiIx4GjZxcDrWl5EXAgLW8C7ovME8BiSSuBG4BHI+JoRBwDHuXcUDEzszlUV+brPgs8IukbZCHyG6l8FbCvpF5PKjtf+TkkbSHrPbBw4cL3XnHFFWU20cwsn3bs2PFqRLRPV6/cAPgj4HMR8QNJHwfuBj5U5ntNEhFbga0AnZ2d0dXVNRtva2aWG5JemUm9cr8FtBl4KC3/T7JxfYD9wOqSeh2p7HzlZmZWJeUGwAHgt9PyB4GX0vJ24BPp20DXAici4iDwCHC9pLZ08/f6VGZmZlUy7RCQpPuBDwDLJPWQfZvnXwN3SqoDBklj9sDDwM1AN3AK+CRARByV9FXg6VTvKxFx9o1lMzObQ5rPfw7a9wDMzC6cpB0R0TldPf8msJlZTjkAzMxyygFgZpZTDgAzs5xyAJiZ5ZQDwMwspxwAZmY55QAwM8spB4CZWU45AMzMcsoBYGaWUw4AM7OccgCYmeWUA8DMLKccAGZmOTVtAEjaJumIpOfPKv+0pF9I2iXp6yXlX5bULWmPpBtKym9MZd2Sbp/dj2FmZhdqJpPC3wP8FXDfRIGk3wE2Ae+OiCFJy1P5BuAW4ErgUuB/S3p7etm3gd8FeoCnJW2PiN2z9UHMzOzCTBsAEfG4pLVnFf8R8LWIGEp1jqTyTcD3UvkvJXVzZsL47oh4GUDS91JdB4CZWZWUew/g7cBvSnpS0t9L+rVUvgrYV1KvJ5Wdr9zMzKpkJkNA53vdEuBa4NeAByS9ZTYaJGkLaZL5NWvWzMZbmpnZFMrtAfQAD0XmKWAcWAbsB1aX1OtIZecrP0dEbI2IzojobG9vL7N5ZmY2nXID4G+A3wFIN3kbgFeB7cAtkgqS1gHrgaeAp4H1ktZJaiC7Uby90sabmVn5ph0CknQ/8AFgmaQe4A5gG7AtfTV0GNgcEQHskvQA2c3dUeC2iBhL7/Mp4BGgFtgWEbsuwucxM7MZUnbenp86Ozujq6ur2s0wM3tDkbQjIjqnq+ffBDYzyykHgJlZTjkAzMxyygFgZpZTDgAzs5xyAJiZ5ZQDwMwspxwAZmY55QAwM8spB4CZWU45AMzMcsoBYGaWUw4AM7OccgCYmeWUA8DMLKccAGZmOTVtAEjaJulImv3r7G1fkBSSlqV1SfqWpG5Jz0raWFJ3s6SX0mPz7H4MMzO7UDPpAdwD3Hh2oaTVwPXA3pLim8jmAV4PbAHuSnWXkE0l+T7gGuAOSW2VNNzMzCozbQBExOPA0Sk2fRP4IlA6p+Qm4L7IPAEslrQSuAF4NCKORsQx4FGmCBUzM5s7Zd0DkLQJ2B8R/3jWplXAvpL1nlR2vvKp3nuLpC5JXb29veU0z8zMZuCCA0BSE/AnwJ/OfnMgIrZGRGdEdLa3t1+MXZiZGeX1AN4KrAP+UdKvgA5gp6RLgP3A6pK6HansfOVmZlYlFxwAEfFcRCyPiLURsZZsOGdjRBwCtgOfSN8GuhY4EREHgUeA6yW1pZu/16cyMzOrkpl8DfR+4B+AyyX1SLr1dao/DLwMdAP/Hfh3ABFxFPgq8HR6fCWVmZlZlSgipq9VJZ2dndHV1VXtZpiZvaFI2hERndPV828Cm5nllAPAzCynHABmZjnlADAzyykHgJlZTjkAzMxyygFgZpZTDgAzs5xyAJiZ5ZQDwMwspxwAZmY55QAwM8spB4CZWU45AMzMcsoBYGaWUzOZEGabpCOSni8p+3NJv5D0rKT/JWlxybYvS+qWtEfSDSXlN6aybkm3z/5HMTOzCzGTHsA9wI1nlT0KvDMirgJeBL4MIGkDcAtwZXrNf5VUK6kW+DZwE7AB+P1U18zMqmTaAIiIx4GjZ5X9OCJG0+oTZJO8A2wCvhcRQxHxS7KpIa9Jj+6IeDkihoHvpbpmZlYls3EP4F8Cf5uWVwH7Srb1pLLzlZ9D0hZJXZK6ent7Z6F5ZmY2lYoCQNJ/AEaB785OcyAitkZEZ0R0tre3z9bbmpnZWerKfaGkPwQ+AlwXZ2aW3w+sLqnWkcp4nXIzM6uCsnoAkm4Evgh8NCJOlWzaDtwiqSBpHbAeeAp4GlgvaZ2kBrIbxdsra7qZmVVi2h6ApPuBDwDLJPUAd5B966cAPCoJ4ImI+LcRsUvSA8BusqGh2yJiLL3Pp4BHgFpgW0Tsugifx8zMZkhnRm/mn87Ozujq6qp2M8zM3lAk7YiIzunq+TeBzcxyygFgZpZTDgAzs5xyAJiZ5ZQDwMwspxwAZmY55QAwM8spB4CZWU45AMzMcsoBYGaWUw4AM7OccgCYmeWUA8DMLKfKnhBmPjs9PMbmbU/R3lKgvaXA8tYC7c0Flrc2srylwPKWAm1NDdTUqNpNNcu1odExdh3o45m9x3nltZOsWdLE5Ze0cPmKFtpbCqQ/N28XyZsyAE4NjyLBCwf7ePzFIfqHRs+pU1cjljWXhkOB9pbGLDDSYyJACnW1VfgUZm8+B0+cZucrx3lm7zF27j3G8wf6GB4dB2BhQy0nh8eKdRc31fP2FVkYvD2FwttXNLO4qaFazX/TmXY+AEnbyKZ+PBIR70xlS4DvA2uBXwEfj4hjyuL6TuBm4BTwhxGxM71mM/Af09v+54i4d7rGzdZ8AKeGR+ntH6K3f4gj/UMc6Rukd2CII31pPW177eQQUx2OxU31xZBY3tI4KRyWT4RGa4GWQp2vWJLBkTFeHZg43kP09g/S2z/EyeEx6mtraKgVDXU12XLJc0NtaZmmKKuhUDe5TvZ+Ne7RzTODIxNX99nJ/pm9xzl4YhCAQl0N71q1iI2XtbFxzWKuXtPGitZGXh0Y4sXD/bx4qJ89hweKy6UXcctbClx+ScukcFi/vJmFhTfl9WxZZjofwEwC4LeAAeC+kgD4OnA0Ir4m6XagLSK+JOlm4NNkAfA+4M6IeF8KjC6gEwhgB/DeiDj2evue6wlhRsfGee3kcAqKwWJAFNdLQmTiqqVUY31NMRSKIdFcYHFTPa0L0qOxnkUL6mldUMeiBfVvqN5FRNA3OFo8Hr0TJ/eBLFRLj8+J0yPnvL5GsKC+lpHxmPL4Vaqu5kyonAmKKYKmtoYVrY1suLSVKy9t5R0rW1m0oH7W25MnEcGBE4PsfCU70e/ce4zdB/oYHsv+nTvaFrBxTRtXr1nMxjVtvGNlKw11M7sFGREcPDHInsP9vHS4nz2HsmB46Ug/gyNnfo5WL1mQeglnHm9dvnDe/R8bHw+OnRrm1YFhXh0Y4tWB7P/NxPprA9nyZUub+Kt/trGsfcxaAKQ3Wwv8qCQA9gAfiIiDklYCP4uIyyX9t7R8f2m9iUdE/JtUPqne+czXGcEigr7To/QODJb0IgZLehhnTop9g+cOP5Uq1NWkQEjB0Fh31vqZwJgcIPW0FOpm5ap3bDx4bWBy2JV+liP9Z3pMQ1OcuAt1NcXe0Zme0lk9pJYCS5sL1Kb2RgSjKQhGxsYZHhtPy1OVlTyXbk/PQ+l5pOQ9zi07U294dJx9x07T2z9U/Ayrlyxgw8pWNqxcxJWXtrLh0lZWLmp0j+48BkfGeH7/ieKV/c69xzjclx3Pxvoarlq1mKsvW1w86S9vaZz1NoyNB/uOnmJPscfQz4uH+3m59ySj49l5rbZGrF3adE6P4bIlTdTVzt53YEbGxjmaLh6zk/iZk/vEiX3iJH/05BDjU5x262vF0oUFlrU0sHRhgXeuauXf33BFWe2ZaQCU22daEREH0/IhYEVaXgXsK6nXk8rOV34OSVuALQBr1qwps3kXlyQWNdWzqKmety1ved26Q6Nj9J0e5cTpEfoGR7LnicfgaHF9YnvvwBD/r/ckfYNZ+VQ/KGfaAS2FuinDorjelD3X19YUfwgnejMTQfXawNQ/kIsW1Gf3Q1oLvHdNW/EmeunwV7lDX5KKwzfVcqR/kN0H+th9sI9dB/p44UAfP959uDgM2NZUn3oJi9iwMustrFu2cFZPHG8EEUHPsdM8s+94usI/xu6DfYyMZQdqzZImrn3LUjauaWPjmjauWNkyJ/+utTVi7bKFrF22kBuuvKRYPjw6zq9eO8meQ1kg7DnUz+4Dffzt84eK/7YNtTW8dXkzl69oLrm/0MKqxQuKF1WDI2O8dnKYV/uHJp3MpzrJHzt1bo8XsoujZc0FlrUU6GhbwHtWL87WmxtY2lxgWXOB9pYGljUXWLSgfs4vOCoeNIuIkDRrEwtHxFZgK2Q9gNl632op1NXS3lJLe0vhgl87Ph6cHJ4IiZmFyMuvDhTrnh4ZO+c9a2vEsuYGlrc0snJRI1d1LMpO6iUn9+U5ufm9vKWR5Zc38oHLlxfLTg6N8otDWSBMhMM9//dXxSGrQl0NV6xsLQbChktbecclrSxoePMcq8GRMZ7tmbi6P8bOvceLvaUF9bVc1bGIf/Wbb+Hq1dnYfTk/2xdTQ11NcQio1OnhMbqPDBR7CnsO9fPUL4/yNz8/UKyzsKGWZS0Fjg4MT/nlEYDmQh3LmrOT9lvbm7lm3ZLiSb49lU+sL2yonde9yHID4LCklSVDQEdS+X5gdUm9jlS2n2wYqLT8Z2XuOzdqakRLYz0tjfXQduGvHx4dL/YkhsfGWdZcYIm//vq6FhbqeO9lS3jvZUuKZSNj47zce5JdB06w+0AWDg8/d5D7n9oLZPc21i1bmPUU0n2FDStbWdo8v06MZxsZG+fk0ChHTw7z3P4T2dX9vuPsPtBXHEK5bGkT/+Rty4o3aq+4pOUN2wNa0FDLuzoW8a6ORZPKT5weofvImXsLrw4MFa/SJ07mS9Nye0uBxvo3T9iXew/gz4HXSm4CL4mIL0r6MPApztwE/lZEXJNuAu8AJu5o7CS7CXz09fY7X+8BmEUE+4+fLgbC7oNZj2H/8dPFOitaC5OGj668dBGrlyyo6Ipwold4cmiMgaER+gcnLw8MjXJyaJT+oVEGBrPlgaHRSdsm1s++n9PUUMu7Oxaz8bLFXL06G7uf7yFmU5u1ewCS7ie7el8mqQe4A/ga8ICkW4FXgI+n6g+Tnfy7yb4G+kmAiDgq6avA06neV6Y7+ZvNZ5LoaGuio62J60vGn4+fGi6GwUQ4/P2LvYylK+qWQh3vWJkNHW1I34SZ6mRdXB4aZWBwJJ3ks7KZKNTV0NJYx8JCHc3pcUlrI82Nab2xjuaG7LmlsZ53rMzGwd+oV/dWnhn1AKrFPQB7MxgcGePFw/2TegsvHOzj1PDkezS1NSqerIsn6fOtN56nTiE76c/0K5b25nSxvwVkZjPUWF/LVR2LuapjcbFsfDzYd+wU40HxxN1YXzOvbxjam48DwKwKamrEZUsXVrsZlnPuJ5qZ5ZQDwMwspxwAZmY55QAwM8spB4CZWU45AMzMcsoBYGaWUw4AM7OccgCYmeWUA8DMLKccAGZmOeUAMDPLKQeAmVlOVRQAkj4naZek5yXdL6lR0jpJT0rqlvR9SQ2pbiGtd6fta2fjA5iZWXnKDgBJq4A/BjrTVJG1wC3AnwHfjIi3AceAW9NLbgWOpfJvpnpmZlYllQ4B1QELJNUBTcBB4IPAg2n7vcDH0vKmtE7afp08+4WZWdWUHQARsR/4BrCX7MR/gmzi9+MRMTFxaQ+wKi2vAval146m+kvPfl9JWyR1Serq7e0tt3lmZjaNSoaA2siu6tcBlwILgRsrbVBEbI2IzojobG9vr/TtzMzsPCoZAvoQ8MuI6I2IEeAh4P3A4jQkBNAB7E/L+4HVAGn7IuC1CvZvZmYVqCQA9gLXSmpKY/nXAbuBnwK/l+psBn6YlrenddL2n0REVLB/MzOrQCX3AJ4ku5m7E3guvddW4EvA5yV1k43x351ecjewNJV/Hri9gnabmVmFNJ8vwjs7O6Orq6vazTAze0ORtCMiOqer598ENjPLKQeAmVlOOQDMzHLKAWBmllMOADOznHIAmJnllAPAzCynHABmZjnlADAzyykHgJlZTjkAzMxyygFgZpZTDgAzs5xyAJiZ5ZQDwMwspyoKAEmLJT0o6ReSXpD065KWSHpU0kvpuS3VlaRvSeqW9KykjbPzEczMrByV9gDuBP4uIq4A3g28QDbT12MRsR54jDMzf90ErE+PLcBdFe7bzMwqUHYASFoE/BZpyseIGI6I48Am4N5U7V7gY2l5E3BfZJ4gmzx+ZdktNzOzilTSA1gH9AJ/LekZSd+RtBBYEREHU51DwIq0vArYV/L6nlQ2iaQtkrokdfX29lbQPDMzez2VBEAdsBG4KyKuBk5y1kTvkU04fEGTDkfE1ojojIjO9vb2CppnZmavp5IA6AF6IuLJtP4gWSAcnhjaSc9H0vb9wOqS13ekMjMzq4KyAyAiDgH7JF2eiq4DdgPbgc2pbDPww7S8HfhE+jbQtcCJkqEiMzObY3UVvv7TwHclNQAvA58kC5UHJN0KvAJ8PNV9GLgZ6AZOpbpmZlYlFQVARPwc6Jxi03VT1A3gtkr2Z2Zms8e/CWxmllMOADOznHIAmJnllAPAzCynHABmZjnlADAzyykHgJlZTjkAzMxyygFgZpZTDgAzs5xyAJiZ5ZQDwMwspxwAZmY55QAwM8spB4CZWU5VHACSatOk8D9K6+skPSmpW9L302QxSCqk9e60fW2l+zYzs/LNRg/gM8ALJet/BnwzIt4GHANuTeW3AsdS+TdTPTMzq5KKAkBSB/Bh4DtpXcAHySaIB7gX+Fha3pTWSduvS/XNzKwKKu0B/CXwRWA8rS8FjkfEaFrvAVal5VXAPoC0/USqP4mkLZK6JHX19vZW2DwzMzufsgNA0keAIxGxYxbbQ0RsjYjOiOhsb2+fzbc2M7MSlUwK/37go5JuBhqBVuBOYLGkunSV3wHsT/X3A6uBHkl1wCLgtQr2b2ZmFSi7BxARX46IjohYC9wC/CQi/jnwU+D3UrXNwA/T8va0Ttr+k4iIcvdvZmaVuRi/B/Al4POSusnG+O9O5XcDS1P554HbL8K+zcxshioZAiqKiJ8BP0vLLwPXTFFnEPins7E/MzOrnH8T2MwspxwAZmY55QAwM8spB4CZWU45AMzMcsoBYGaWUw4AM7OccgCYmeWUA8DMLKccAGZmOeUAMDPLKQeAmVlOOQDMzHLKAWBmllMOADOznKpkTuDVkn4qabekXZI+k8qXSHpU0kvpuS2VS9K3JHVLelbSxtn6EGZmduEq6QGMAl+IiA3AtcBtkjaQzfT1WESsBx7jzMxfNwHr02MLcFcF+zYzswpVMifwwYjYmZb7gReAVcAm4N5U7V7gY2l5E3BfZJ4gmzx+ZdktNzOziszKPQBJa4GrgSeBFRFxMG06BKxIy6uAfSUv60llZmZWBRUHgKRm4AfAZyOir3RbRAQQF/h+WyR1Serq7e2ttHlmZnYeFQWApHqyk/93I+KhVHx4YmgnPR9J5fuB1SUv70hlk0TE1ojojIjO9vb2SppnZmavo5JvAQm4G3ghIv6iZNN2YHNa3gz8sKT8E+nbQNcCJ0qGiszMbI7VVfDa9wN/ADwn6eep7E+ArwEPSLoVeAX4eNr2MHAz0A2cAj5Zwb7NzKxCZQdARPwfQOfZfN0U9QO4rdz9mZnZ7PJvApuZ5ZQDwMwspxwAZmY55QAwM8spB4CZWU45AMzMcsoBYGaWUw4AM7OccgCYmeWUA8DMLKccAGZmOeUAMDPLKQeAmVlOOQDMzHLKAWBmllMOADOznJrzAJB0o6Q9krol3T7X+zczs8ycBoCkWuDbwE3ABuD3JW2YyzaYmVlmrnsA1wDdEfFyRAwD3wM2zXEbzMyMyiaFL8cqYF/Jeg/wvtIKkrYAW9LqgKQ9c9S2i2UZ8Gq1GzGP+HhM5uNxho/FZJUcj8tmUmmuA2BaEbEV2FrtdswWSV0R0VntdswXPh6T+Xic4WMx2Vwcj7keAtoPrC5Z70hlZmY2x+Y6AJ4G1ktaJ6kBuAXYPsdtMDMz5ngIKCJGJX0KeASoBbZFxK65bEMVvGmGs2aJj8dkPh5n+FhMdtGPhyLiYu/DzMzmIf8msJlZTjkAzMxyygFwkUhaLemnknZL2iXpM9VuU7VJqpX0jKQfVbst1SZpsaQHJf1C0guSfr3abaomSZ9L/0+el3S/pMZqt2kuSdom6Yik50vKlkh6VNJL6blttvfrALh4RoEvRMQG4FrgNv/ZCz4DvFDtRswTdwJ/FxFXAO8mx8dF0irgj4HOiHgn2RdEbqluq+bcPcCNZ5XdDjwWEeuBx9L6rHIAXCQRcTAidqblfrL/4Kuq26rqkdQBfBj4TrXbUm2SFgG/BdwNEBHDEXG8uq2qujpggaQ6oAk4UOX2zKmIeBw4elbxJuDetHwv8LHZ3q8DYA5IWgtcDTxZ3ZZU1V8CXwTGq92QeWAd0Av8dRoS+46khdVuVLVExH7gG8Be4CBwIiJ+XN1WzQsrIuJgWj4ErJjtHTgALjJJzcAPgM9GRF+121MNkj4CHImIHdVuyzxRB2wE7oqIq4GTXITu/RtFGtveRBaMlwILJf2L6rZqfons+/qz/p19B8BFJKme7OT/3Yh4qNrtqaL3Ax+V9CuyvwD7QUn/o7pNqqoeoCciJnqED5IFQl59CPhlRPRGxAjwEPAbVW7TfHBY0kqA9HxktnfgALhIJIlsjPeFiPiLarenmiLiyxHRERFryW7u/SQicnuFFxGHgH2SLk9F1wG7q9ikatsLXCupKf2/uY4c3xQvsR3YnJY3Az+c7R04AC6e9wN/QHa1+/P0uLnajbJ549PAdyU9C7wH+C9Vbk/VpJ7Qg8BO4Dmy81Ku/iyEpPuBfwAul9Qj6Vbga8DvSnqJrJf0tVnfr/8UhJlZPrkHYGaWUw4AM7OccgCYmeWUA8DMLKccAGZmOeUAMDPLKQeAmVlO/X/HG8Ynjh7R5gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "hist = generate_discrete_uniform_distribution(10000, 1, 10)[1]\n",
    "print(hist)\n",
    "plt.plot(hist.keys(), hist.values())\n",
    "plt.yticks(list(range(0, 2000, 200)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 2, 2, 1, 2, 3, 4, 5])"
      ]
     },
     "execution_count": 286,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.digitize([1, 2, 2, 2, 1, 2, 3, 4, 5], bins = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Script Commands"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# hmm_default_medium\n",
      "python3 run_model.py --model_type hmm --base_log logs/gfp/hmm/ --name hmm_default_medium --input 4998 --hidden_size 50 --latent_dim -1 --seq_length 238 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 100 --batch_size 10 --layers 1 --dataset gfp --num_data 1000 --early_stopping True --patience 10 \n",
      "# rnn_default_medium\n",
      "python3 run_model.py --model_type rnn --base_log logs/gfp/rnn/ --name rnn_default_medium --input 4998 --hidden_size 200 --latent_dim -1 --seq_length 238 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 100 --batch_size 10 --layers 1 --dataset gfp --num_data 1000 --early_stopping True --patience 10 \n",
      "# vae_default_medium\n",
      "python3 run_model.py --model_type vae --base_log logs/gfp/vae/ --name vae_default_medium --input 4998 --hidden_size 200 --latent_dim 20 --seq_length 238 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 100 --batch_size 10 --layers 1 --dataset gfp --num_data 1000 --early_stopping True --patience 10 \n"
     ]
    }
   ],
   "source": [
    "for mp in [hmm_default_args(), rnn_default_args(), vae_default_args()]:\n",
    "    command = \"python3 run_model.py \"\n",
    "    for key, value in mp.items():\n",
    "        command += \"--{0} {1} \".format(key, value)\n",
    "    print(\"#\", mp[\"name\"])\n",
    "    print(command)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# hmm_default_small\n",
      "python3 run_model.py --model_type hmm --base_log logs/gfp/hmm/ --name hmm_default_small --input 4998 --hidden_size 20 --latent_dim -1 --seq_length 238 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset gfp --num_data 100 --early_stopping True --patience 10 \n",
      "# rnn_default_small\n",
      "python3 run_model.py --model_type rnn --base_log logs/gfp/rnn/ --name rnn_default_small --input 4998 --hidden_size 50 --latent_dim -1 --seq_length 238 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset gfp --num_data 100 --early_stopping True --patience 10 \n",
      "# vae_default_small\n",
      "python3 run_model.py --model_type vae --base_log logs/gfp/vae/ --name vae_default_small --input 4998 --hidden_size 50 --latent_dim 20 --seq_length 238 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset gfp --num_data 100 --early_stopping True --patience 10 \n"
     ]
    }
   ],
   "source": [
    "for mp in [hmm_default_small_args(), rnn_default_small_args(), vae_default_small_args()]:\n",
    "    command = \"python3 run_model.py \"\n",
    "    for key, value in mp.items():\n",
    "        command += \"--{0} {1} \".format(key, value)\n",
    "    print(\"#\", mp[\"name\"])\n",
    "    print(command)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger=open(\"./scripts/synthetic_unimodal_small_script.sh\", \"w\")\n",
    "num_commands = 0\n",
    "for mp in [vae_default_small_args(), rnn_default_small_args(), hmm_default_small_args()]:\n",
    "    for i, data_set_description in pd.read_csv(\"./data/dataset_descriptions.csv\").iterrows():\n",
    "        if \"unimodal\" not in data_set_description[\"dataset\"]:\n",
    "            continue\n",
    "        command = \"python3 run_model.py \"\n",
    "        mp[\"dataset\"] = data_set_description[\"dataset\"]\n",
    "        mp[\"base_log\"] = os.path.join(\"logs\", mp[\"dataset\"], mp[\"model_type\"])\n",
    "        mp[\"seq_length\"] = data_set_description[\"sequence_length\"]\n",
    "        mp[\"input\"] = len(get_all_amino_acids(gap=False)) * mp[\"seq_length\"]\n",
    "        for key, value in mp.items():\n",
    "            command += \"--{0} {1} \".format(key, value)\n",
    "        print(command, file=logger)\n",
    "        num_commands += 1\n",
    "assert(num_commands == 27)\n",
    "logger.close()\n",
    "# python3 run_model.py --model_type vae --base_log logs/synthetic_unimodal_data_length_20_uniform/vae/ --name vae_default_small --input 400 --hidden_size 50 --latent_dim 20 --seq_length 20 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 100 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_20_uniform --num_data 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger=open(\"./scripts/synthetic_unimodal_medium_script.sh\", \"w\")\n",
    "num_commands = 0\n",
    "for mp in [vae_default_args(), rnn_default_args(), hmm_default_args()]:\n",
    "    for i, data_set_description in pd.read_csv(\"./data/dataset_descriptions.csv\").iterrows():\n",
    "        if \"unimodal\" not in data_set_description[\"dataset\"]:\n",
    "            continue\n",
    "        command = \"python3 run_model.py \"\n",
    "        mp[\"dataset\"] = data_set_description[\"dataset\"]\n",
    "        mp[\"base_log\"] = os.path.join(\"logs\", mp[\"dataset\"], mp[\"model_type\"])\n",
    "        mp[\"seq_length\"] = data_set_description[\"sequence_length\"]\n",
    "        mp[\"input\"] = len(get_all_amino_acids(gap=False)) * mp[\"seq_length\"]\n",
    "        for key, value in mp.items():\n",
    "            command += \"--{0} {1} \".format(key, value)\n",
    "        print(command, file=logger)\n",
    "        num_commands += 1\n",
    "assert(num_commands == 27)\n",
    "logger.close()\n",
    "# python3 run_model.py --model_type vae --base_log logs/synthetic_unimodal_data_length_20_uniform/vae/ --name vae_default_small --input 400 --hidden_size 50 --latent_dim 20 --seq_length 20 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 100 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_20_uniform --num_data 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_commands = 0\n",
    "for mp in [vae_default_small_args(), rnn_default_small_args(), hmm_default_small_args()]:\n",
    "    logger=open(\"./scripts/{0}_unimodal_small_script.sh\".format(mp[\"model_type\"]), \"w+\")\n",
    "    for i, data_set_description in pd.read_csv(\"./data/dataset_descriptions.csv\").iterrows():\n",
    "        if \"unimodal\" not in data_set_description[\"dataset\"]:\n",
    "            continue\n",
    "        command = \"python3 run_model.py \"\n",
    "        mp[\"dataset\"] = data_set_description[\"dataset\"]\n",
    "        mp[\"base_log\"] = os.path.join(\"logs\", mp[\"dataset\"], mp[\"model_type\"])\n",
    "        mp[\"seq_length\"] = data_set_description[\"sequence_length\"]\n",
    "        mp[\"input\"] = len(get_all_amino_acids(gap=False)) * mp[\"seq_length\"]\n",
    "        for key, value in mp.items():\n",
    "            command += \"--{0} {1} \".format(key, value)\n",
    "        print(command, file=logger)\n",
    "        num_commands += 1\n",
    "    logger.close()\n",
    "assert(num_commands == 27)\n",
    "# python3 run_model.py --model_type vae --base_log logs/synthetic_unimodal_data_length_20_uniform/vae/ --name vae_default_small --input 400 --hidden_size 50 --latent_dim 20 --seq_length 20 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 100 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_20_uniform --num_data 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger = open(\"./scripts/synthetic_multimodal_medium_script.sh\", \"w\")\n",
    "num_commands = 0\n",
    "for mp in [vae_default_args(), rnn_default_args(), hmm_default_args()]:\n",
    "    for i, data_set_description in pd.read_csv(\"./data/dataset_descriptions.csv\").iterrows():\n",
    "        if \"multimodal\" not in data_set_description[\"dataset\"]:\n",
    "            continue\n",
    "        command = \"python3 run_model.py \"\n",
    "        mp[\"dataset\"] = data_set_description[\"dataset\"]\n",
    "        mp[\"base_log\"] = os.path.join(\"logs\", mp[\"dataset\"], mp[\"model_type\"])\n",
    "        mp[\"seq_length\"] = data_set_description[\"sequence_length\"]\n",
    "        mp[\"input\"] = len(get_all_amino_acids(gap=False)) * mp[\"seq_length\"]\n",
    "        for key, value in mp.items():\n",
    "            command += \"--{0} {1} \".format(key, value)\n",
    "        print(command, file=logger)\n",
    "        num_commands += 1\n",
    "assert(num_commands == 9)\n",
    "logger.close()\n",
    "# python3 run_model.py --model_type vae --base_log logs/synthetic_unimodal_data_length_20_uniform/vae/ --name vae_default_small --input 400 --hidden_size 50 --latent_dim 20 --seq_length 20 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 100 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_20_uniform --num_data 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger = open(\"./scripts/synthetic_multimodal_small_script.sh\", \"w\")\n",
    "num_commands = 0\n",
    "for i, data_set_description in pd.read_csv(\"./data/dataset_descriptions.csv\").iterrows():\n",
    "    for mp in [vae_default_small_args(), rnn_default_small_args(), hmm_default_small_args()]:\n",
    "        if \"multimodal\" not in data_set_description[\"dataset\"]:\n",
    "            continue\n",
    "        command = \"python3 run_model.py \"\n",
    "        mp[\"dataset\"] = data_set_description[\"dataset\"]\n",
    "        mp[\"base_log\"] = os.path.join(\"logs\", mp[\"dataset\"], mp[\"model_type\"])\n",
    "        mp[\"seq_length\"] = data_set_description[\"sequence_length\"]\n",
    "        mp[\"input\"] = len(get_all_amino_acids(gap=False)) * mp[\"seq_length\"]\n",
    "        for key, value in mp.items():\n",
    "            command += \"--{0} {1} \".format(key, value)\n",
    "        print(command, file=logger)\n",
    "        num_commands += 1\n",
    "assert(num_commands == 9)\n",
    "logger.close()\n",
    "# python3 run_model.py --model_type vae --base_log logs/synthetic_unimodal_data_length_20_uniform/vae/ --name vae_default_small --input 400 --hidden_size 50 --latent_dim 20 --seq_length 20 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 100 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_20_uniform --num_data 1000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Model Tests and Examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = hmm_default_small_args()\n",
    "train, test, valid = get_dataloader(args)\n",
    "hmm = GenerativeHMM(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "hmm.load_model(\"./logs/gfp/hmm/hmm_default_small/hmm_default_small_saved_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average neg log prob: 643.8108\n",
      "Average neg log prob: 644.5363\n",
      "Average neg log prob: 644.3261\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "644.3261382530319"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hmm.evaluate(dataloader=train, verbose=True)\n",
    "hmm.evaluate(dataloader=valid, verbose=True)\n",
    "hmm.evaluate(dataloader=test, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total loss: 141.1837\n",
      "total loss: 147.8163\n",
      "total loss: 146.6423\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "146.64228515625"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "args = rnn_default_small_args()\n",
    "train, test, valid = get_dataloader(args)\n",
    "rnn = GenerativeRNN(args)\n",
    "rnn.load_model(\"./logs/gfp/rnn/rnn_default_small/rnn_default_small_saved_model\", initial_probs=True)\n",
    "rnn.evaluate(dataloader=train, verbose=True)\n",
    "rnn.evaluate(dataloader=valid, verbose=True)\n",
    "rnn.evaluate(dataloader=test, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total loss: 45.6191 cross entropy loss: 30.0420. kld loss: 15.5770\n",
      "total loss: 48.6362 cross entropy loss: 33.0811. kld loss: 15.5551\n",
      "total loss: 48.0014 cross entropy loss: 32.4644. kld loss: 15.5370\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(48.001426696777344, 32.464420318603516, 15.537006378173828)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "args = vae_default_small_args()\n",
    "train, test, valid = get_dataloader(args)\n",
    "vae = GenerativeVAE(args)\n",
    "vae.load_model(\"./logs/gfp/vae/vae_default_small/vae_default_small_saved_model\", initial_probs=True)\n",
    "vae.evaluate(dataloader=train, verbose=True)\n",
    "vae.evaluate(dataloader=valid, verbose=True)\n",
    "vae.evaluate(dataloader=test, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total loss: 32.8738 cross entropy loss: 26.6793. kld loss: 6.1945\n",
      "total loss: 33.3337 cross entropy loss: 27.1171. kld loss: 6.2166\n",
      "total loss: 32.7038 cross entropy loss: 26.5114. kld loss: 6.1924\n",
      "total loss: 28.3956 cross entropy loss: 25.3037. kld loss: 3.0919\n",
      "total loss: 28.4135 cross entropy loss: 25.2355. kld loss: 3.1780\n",
      "total loss: 28.0267 cross entropy loss: 24.8950. kld loss: 3.1317\n",
      "total loss: 32.7605 cross entropy loss: 26.5660. kld loss: 6.1945\n",
      "total loss: 32.9785 cross entropy loss: 26.7619. kld loss: 6.2166\n",
      "total loss: 32.5586 cross entropy loss: 26.3662. kld loss: 6.1924\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(32.55861282348633, 26.366209030151367, 6.1924028396606445)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "args = vae_default_args()\n",
    "train, test, valid = get_dataloader(args)\n",
    "vae = GenerativeVAE(args)\n",
    "vae.load_model(\"./logs/gfp/vae/vae_default_medium/vae_default_medium_saved_model\")\n",
    "vae.evaluate(dataloader=train, verbose=True)\n",
    "vae.evaluate(dataloader=valid, verbose=True)\n",
    "vae.evaluate(dataloader=test, verbose=True)\n",
    "vae.load_model(\"./logs/gfp/vae/vae_default_medium/vae_checkpoint_50.pt\")\n",
    "vae.evaluate(dataloader=train, verbose=True)\n",
    "vae.evaluate(dataloader=valid, verbose=True)\n",
    "vae.evaluate(dataloader=test, verbose=True)\n",
    "vae.load_model(\"./logs/gfp/vae/vae_default_medium/vae_checkpoint_100.pt\")\n",
    "vae.evaluate(dataloader=train, verbose=True)\n",
    "vae.evaluate(dataloader=valid, verbose=True)\n",
    "vae.evaluate(dataloader=test, verbose=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GFP Data checks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"./data/gfp_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "wild_type = df[\"nucSequence\"].values[0]\n",
    "aa_wild_type = translate(df[\"nucSequence\"].values[0])\n",
    "assert(all([count_substring_mismatch(x1, wild_type) == x2 for x1, x2 in zip(df[\"nucSequence\"].values, df[\"numNucMutations\"])]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(['Unnamed: 0', 'Unnamed: 0.1', 'Unnamed: 0.1.1'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['nucSequence', 'numNucMutations', 'numAAMutations', 'uniqueBarcodes',\n",
       "       'medianBrightness', 'std', 'aaSequence'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"aaSequence\"] = df[\"nucSequence\"].apply(lambda x : translate(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "714\n",
      "238\n",
      "238\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(wild_type)), print(len(aa_wild_type)), print(len(df[\"aaSequence\"][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for x1, x2 in zip(df[\"aaSequence\"].values, df[\"numAAMutations\"].values):\n",
    "    assert(count_substring_mismatch(x1, aa_wild_type) == x2)\n",
    "    assert(\"*\" in x1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(46733, 7) (11684, 7)\n"
     ]
    }
   ],
   "source": [
    "df = df.sample(frac=1).reset_index(drop=True)\n",
    "df[\"aaSequence\"] = df[\"aaSequence\"].apply(lambda x : x.upper())\n",
    "train_size = int(df.shape[0] * 0.8)\n",
    "train_df = df[:train_size].reset_index(drop=True)\n",
    "assert(wild_type in train_df[\"nucSequence\"].values)\n",
    "test_df = df[train_size: ].reset_index(drop=True)\n",
    "print(train_df.shape, test_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(                                         nucSequence  numNucMutations  \\\n",
       " 0  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGACG...                6   \n",
       " 1  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                2   \n",
       " 2  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                5   \n",
       " 3  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                4   \n",
       " 4  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                3   \n",
       " 5  AGCAAGGGCGAGGAGCTGTTCTCCGGGGTGGTGCCCATCCTGGTCG...                5   \n",
       " 6  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                2   \n",
       " 7  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                5   \n",
       " 8  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTTCCCATCCTGGTCG...                3   \n",
       " 9  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                2   \n",
       " \n",
       "    numAAMutations  uniqueBarcodes  medianBrightness  std  \\\n",
       " 0               6               1          1.303274  0.0   \n",
       " 1               1               1          3.807237  0.0   \n",
       " 2               5               1          3.618254  0.0   \n",
       " 3               3               1          1.301357  0.0   \n",
       " 4               3               1          3.144210  0.0   \n",
       " 5               5               1          3.524743  0.0   \n",
       " 6               2               1          3.259355  0.0   \n",
       " 7               5               1          3.360476  0.0   \n",
       " 8               1               1          3.581428  0.0   \n",
       " 9               0               1          3.653555  0.0   \n",
       " \n",
       "                                           aaSequence  \n",
       " 0  SKGEELFTGVVPILDELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 1  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 2  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 3  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 4  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 5  SKGEELFSGVVPILVELDGVVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 6  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 7  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLALKFI...  \n",
       " 8  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 9  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  ,\n",
       " (58417, 7),\n",
       "                                          nucSequence  numNucMutations  \\\n",
       " 0  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGACG...                6   \n",
       " 1  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                2   \n",
       " 2  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                5   \n",
       " 3  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                4   \n",
       " 4  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                3   \n",
       " 5  AGCAAGGGCGAGGAGCTGTTCTCCGGGGTGGTGCCCATCCTGGTCG...                5   \n",
       " 6  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                2   \n",
       " 7  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                5   \n",
       " 8  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTTCCCATCCTGGTCG...                3   \n",
       " 9  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                2   \n",
       " \n",
       "    numAAMutations  uniqueBarcodes  medianBrightness  std  \\\n",
       " 0               6               1          1.303274  0.0   \n",
       " 1               1               1          3.807237  0.0   \n",
       " 2               5               1          3.618254  0.0   \n",
       " 3               3               1          1.301357  0.0   \n",
       " 4               3               1          3.144210  0.0   \n",
       " 5               5               1          3.524743  0.0   \n",
       " 6               2               1          3.259355  0.0   \n",
       " 7               5               1          3.360476  0.0   \n",
       " 8               1               1          3.581428  0.0   \n",
       " 9               0               1          3.653555  0.0   \n",
       " \n",
       "                                           aaSequence  \n",
       " 0  SKGEELFTGVVPILDELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 1  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 2  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 3  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 4  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 5  SKGEELFSGVVPILVELDGVVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 6  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 7  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLALKFI...  \n",
       " 8  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 9  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  ,\n",
       " (46733, 7),\n",
       "                                          nucSequence  numNucMutations  \\\n",
       " 0  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                3   \n",
       " 1  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                3   \n",
       " 2  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                3   \n",
       " 3  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                6   \n",
       " 4  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                5   \n",
       " 5  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                4   \n",
       " 6  AGCAAGGGCGAGGAGCCGTTCACCGGGGTGGTGCCCATCCTGGTCG...                2   \n",
       " 7  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                5   \n",
       " 8  AGCAAGGGCGAGGAGCTGTTCATCGGGGTGGTGCCCATCCTGGTCG...                5   \n",
       " 9  AGCAAGGGCGAGGAGCTGTTCACCGGGGTGGTGCCCATCCTGGTCG...                4   \n",
       " \n",
       "    numAAMutations  uniqueBarcodes  medianBrightness  std  \\\n",
       " 0               2               1          3.695210  0.0   \n",
       " 1               2               1          3.645163  0.0   \n",
       " 2               2               1          3.739286  0.0   \n",
       " 3               6               1          1.301030  0.0   \n",
       " 4               4               1          3.675671  0.0   \n",
       " 5               4               1          3.316222  0.0   \n",
       " 6               2               1          3.679801  0.0   \n",
       " 7               4               1          1.545725  0.0   \n",
       " 8               5               1          1.507369  0.0   \n",
       " 9               3               1          1.469539  0.0   \n",
       " \n",
       "                                           aaSequence  \n",
       " 0  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 1  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 2  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 3  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATNGKQTLKFI...  \n",
       " 4  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 5  SKGEELFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKLI...  \n",
       " 6  SKGEEPFTGVVPILVELDGDVNGHKFSVSGEGEGDATYGKLTLKFI...  \n",
       " 7  SKGEELFTGVVPILVEQDGDVNGHKLSVSGEGEGDATYGKLTLEFI...  \n",
       " 8  SKGEELFIGVVPILVELDGGVNGHKFSVSGEGEGDAAYGKLTLKFI...  \n",
       " 9  SKGEELFTGVVPILVELDGDVNGHKSSVSGEGEGDATYGKLTLKSI...  ,\n",
       " (11684, 7))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10), df.shape, train_df.head(10), train_df.shape, test_df.head(10), test_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.to_csv(\"./data/gfp_train.csv\", index=False)\n",
    "test_df.to_csv(\"./data/gfp_test.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"./data/gfp_x_train.npy\", train_df[\"aaSequence\"].values)\n",
    "np.save(\"./data/gfp_x_test.npy\", test_df[\"aaSequence\"].values)\n",
    "np.save(\"./data/gfp_y_train.npy\", train_df[\"medianBrightness\"].values)\n",
    "np.save(\"./data/gfp_y_test.npy\", test_df[\"medianBrightness\"].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_test, y_train, y_test = load_data(\"./data/gfp_\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[46733, 11684, 46733, 11684]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(map(len, [x_train, x_test, y_train, y_test]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([4.0140e+03, 1.0766e+04, 1.8015e+04, 5.7710e+03, 5.9220e+03,\n",
       "        1.2210e+03, 8.4700e+02, 1.1400e+02, 5.7000e+01, 6.0000e+00]),\n",
       " array([ 0. ,  1.5,  3. ,  4.5,  6. ,  7.5,  9. , 10.5, 12. , 13.5, 15. ]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFDhJREFUeJzt3X+s3fV93/Hna6bJkjQZUFyP2DC7mZMJUOsEi7BlibLSgCFRTKYpM+qCk6I4UWBLpkidaaURpWOia9J0aBkRSTyMSiEMQrEap8RlUdGkkmAI5WcoFwLlega7kIauqUhN3vvjfO568Ode+3LPtc9x/HxIR+d73t/P9/t9H/963e/38z3HqSokSRr298bdgCRp8hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hwz7gYW6oQTTqiVK1eOuw1JOqLcfffdf1FVSw827ogNh5UrV7Jz585xtyFJR5QkT85nnJeVJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEkdw0GS1DEcJEmdI/YT0np5Vm7+2liO+8QV7x7LcSWNxjMHSVLnoOGQZEuSPUkeGKp9Jcm97fFEkntbfWWSvxla94WhbU5Pcn+SqSRXJkmrH59kR5JH2/Nxh+KNSpLmbz5nDtcA64YLVfWvq2pNVa0Bbga+OrT6sZl1VfXRofpVwIeB1e0xs8/NwO1VtRq4vb2WJI3RQcOhqu4AnpttXfvp//3A9QfaR5ITgddV1Z1VVcC1wPlt9Xpga1veOlSXJI3JqHMObweeqapHh2qrknwnyR8neXurLQemh8ZMtxrAsqra3ZafBpaN2JMkaUSj3q10AS89a9gNnFxVzyY5Hfj9JKfOd2dVVUlqrvVJNgGbAE4++eQFtixJOpgFnzkkOQb4l8BXZmpV9UJVPduW7wYeA94I7AJWDG2+otUAnmmXnWYuP+2Z65hVdXVVra2qtUuXHvQ/MpIkLdAol5V+CfhuVf3/y0VJliZZ0pZ/jsHE8+PtstHzSc5s8xQXAre2zbYBG9vyxqG6JGlM5nMr6/XAnwBvSjKd5KK2agP9RPQ7gPvara03AR+tqpnJ7I8BXwKmGJxRfL3VrwDeleRRBoFzxQjvR5K0CA4651BVF8xR/+AstZsZ3No62/idwGmz1J8FzjpYH5Kkw8dPSEuSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKlz0HBIsiXJniQPDNU+lWRXknvb47yhdZcmmUrySJJzhurrWm0qyeah+qok32r1ryR5xWK+QUnSyzefM4drgHWz1D9XVWvaYztAklOADcCpbZv/nmRJkiXA54FzgVOAC9pYgN9s+/rHwPeBi0Z5Q5Kk0R00HKrqDuC5ee5vPXBDVb1QVd8DpoAz2mOqqh6vqh8BNwDrkwT4ReCmtv1W4PyX+R4kSYtslDmHS5Lc1y47Hddqy4GnhsZMt9pc9Z8B/rKq9u1Xn1WSTUl2Jtm5d+/eEVqXJB3IQsPhKuANwBpgN/DZRevoAKrq6qpaW1Vrly5dejgOKUlHpWMWslFVPTOznOSLwB+0l7uAk4aGrmg15qg/Cxyb5Jh29jA8XpI0Jgs6c0hy4tDL9wEzdzJtAzYkeWWSVcBq4NvAXcDqdmfSKxhMWm+rqgK+Cfyrtv1G4NaF9CRJWjwHPXNIcj3wTuCEJNPAZcA7k6wBCngC+AhAVT2Y5EbgIWAfcHFVvdj2cwlwG7AE2FJVD7ZD/AfghiT/CfgO8OVFe3eSpAU5aDhU1QWzlOf8B7yqLgcun6W+Hdg+S/1xBnczSZImhJ+QliR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DAdJUsdwkCR1DhoOSbYk2ZPkgaHabyX5bpL7ktyS5NhWX5nkb5Lc2x5fGNrm9CT3J5lKcmWStPrxSXYkebQ9H3co3qgkaf7mc+ZwDbBuv9oO4LSq+nngz4BLh9Y9VlVr2uOjQ/WrgA8Dq9tjZp+bgdurajVwe3stSRqjg4ZDVd0BPLdf7RtVta+9vBNYcaB9JDkReF1V3VlVBVwLnN9Wrwe2tuWtQ3VJ0pgsxpzDrwBfH3q9Ksl3kvxxkre32nJgemjMdKsBLKuq3W35aWDZIvQkSRrBMaNsnOTXgX3Ada20Gzi5qp5Ncjrw+0lOne/+qqqS1AGOtwnYBHDyyScvvHFJ0gEt+MwhyQeB9wC/3C4VUVUvVNWzbflu4DHgjcAuXnrpaUWrATzTLjvNXH7aM9cxq+rqqlpbVWuXLl260NYlSQexoHBIsg74VeC9VfXDofrSJEva8s8xmHh+vF02ej7Jme0upQuBW9tm24CNbXnjUF2SNCYHvayU5HrgncAJSaaByxjcnfRKYEe7I/XOdmfSO4BPJ/lb4MfAR6tqZjL7YwzufHoVgzmKmXmKK4Abk1wEPAm8f1HemSRpwQ4aDlV1wSzlL88x9mbg5jnW7QROm6X+LHDWwfqQJB0+fkJaktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQZ6f+Q1suzcvPXxt2CJM2LZw6SpI7hIEnqGA6SpI7hIEnqzCsckmxJsifJA0O145PsSPJoez6u1ZPkyiRTSe5L8pahbTa28Y8m2ThUPz3J/W2bK5NkMd+kJOnlme+ZwzXAuv1qm4Hbq2o1cHt7DXAusLo9NgFXwSBMgMuAtwJnAJfNBEob8+Gh7fY/liTpMJpXOFTVHcBz+5XXA1vb8lbg/KH6tTVwJ3BskhOBc4AdVfVcVX0f2AGsa+teV1V3VlUB1w7tS5I0BqPMOSyrqt1t+WlgWVteDjw1NG661Q5Un56lLkkak0WZkG4/8ddi7OtAkmxKsjPJzr179x7qw0nSUWuUcHimXRKiPe9p9V3ASUPjVrTageorZql3qurqqlpbVWuXLl06QuuSpAMZJRy2ATN3HG0Ebh2qX9juWjoT+EG7/HQbcHaS49pE9NnAbW3d80nObHcpXTi0L0nSGMzru5WSXA+8EzghyTSDu46uAG5MchHwJPD+Nnw7cB4wBfwQ+BBAVT2X5DeAu9q4T1fVzCT3xxjcEfUq4OvtIUkak3mFQ1VdMMeqs2YZW8DFc+xnC7BllvpO4LT59CJJOvT8hLQkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6hoMkqWM4SJI6Cw6HJG9Kcu/Q4/kkn0jyqSS7hurnDW1zaZKpJI8kOWeovq7VppJsHvVNSZJGc8xCN6yqR4A1AEmWALuAW4APAZ+rqs8Mj09yCrABOBV4PfBHSd7YVn8eeBcwDdyVZFtVPbTQ3iRJo1lwOOznLOCxqnoyyVxj1gM3VNULwPeSTAFntHVTVfU4QJIb2ljDQZLGZLHmHDYA1w+9viTJfUm2JDmu1ZYDTw2NmW61ueqSpDEZORySvAJ4L/A/W+kq4A0MLjntBj476jGGjrUpyc4kO/fu3btYu5Uk7WcxzhzOBe6pqmcAquqZqnqxqn4MfJG/u3S0CzhpaLsVrTZXvVNVV1fV2qpau3Tp0kVoXZI0m8UIhwsYuqSU5MShde8DHmjL24ANSV6ZZBWwGvg2cBewOsmqdhayoY2VJI3JSBPSSV7D4C6jjwyV/0uSNUABT8ysq6oHk9zIYKJ5H3BxVb3Y9nMJcBuwBNhSVQ+O0pckaTQjhUNV/TXwM/vVPnCA8ZcDl89S3w5sH6UXSdLi8RPSkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqSO4SBJ6hgOkqTOYv0f0tLEWbn5a2M57hNXvHssx5UWk+GgQ2pc/0BLGo2XlSRJHcNBktQxHCRJHcNBktQxHCRJnZHDIckTSe5Pcm+Sna12fJIdSR5tz8e1epJcmWQqyX1J3jK0n41t/KNJNo7alyRp4RbrzOFfVNWaqlrbXm8Gbq+q1cDt7TXAucDq9tgEXAWDMAEuA94KnAFcNhMokqTD71BdVloPbG3LW4Hzh+rX1sCdwLFJTgTOAXZU1XNV9X1gB7DuEPUmSTqIxQiHAr6R5O4km1ptWVXtbstPA8va8nLgqaFtp1ttrvpLJNmUZGeSnXv37l2E1iVJs1mMT0j/86raleRngR1Jvju8sqoqSS3Ccaiqq4GrAdauXbso+5Qk9UY+c6iqXe15D3ALgzmDZ9rlItrznjZ8F3DS0OYrWm2uuiRpDEYKhySvSfLamWXgbOABYBswc8fRRuDWtrwNuLDdtXQm8IN2+ek24Owkx7WJ6LNbTZI0BqNeVloG3JJkZl+/V1V/mOQu4MYkFwFPAu9v47cD5wFTwA+BDwFU1XNJfgO4q437dFU9N2JvkqQFGikcqupx4BdmqT8LnDVLvYCL59jXFmDLKP1IkhaHn5CWJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSZzG+PuOI4396L0kH5pmDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOgsOhyQnJflmkoeSPJjk463+qSS7ktzbHucNbXNpkqkkjyQ5Z6i+rtWmkmwe7S1JkkY1yrey7gM+WVX3JHktcHeSHW3d56rqM8ODk5wCbABOBV4P/FGSN7bVnwfeBUwDdyXZVlUPjdCbJGkECw6HqtoN7G7Lf5XkYWD5ATZZD9xQVS8A30syBZzR1k1V1eMASW5oYw0HSRqTRZlzSLISeDPwrVa6JMl9SbYkOa7VlgNPDW023Wpz1SVJYzJyOCT5aeBm4BNV9TxwFfAGYA2DM4vPjnqMoWNtSrIzyc69e/cu1m4lSfsZKRyS/BSDYLiuqr4KUFXPVNWLVfVj4Iv83aWjXcBJQ5uvaLW56p2qurqq1lbV2qVLl47SuiTpAEa5WynAl4GHq+q3h+onDg17H/BAW94GbEjyyiSrgNXAt4G7gNVJViV5BYNJ620L7UuSNLpR7lZ6G/AB4P4k97barwEXJFkDFPAE8BGAqnowyY0MJpr3ARdX1YsASS4BbgOWAFuq6sER+pIkjWiUu5X+N5BZVm0/wDaXA5fPUt9+oO0kSYeXn5CWJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSZ5TvVpI0i5Wbvza2Yz9xxbvHdmz9ZPHMQZLUMRwkSR3DQZLUcc5B+gkyrvkO5zp+8njmIEnqGA6SpI7hIEnqGA6SpI7hIEnqTEw4JFmX5JEkU0k2j7sfSTqaTUQ4JFkCfB44FzgFuCDJKePtSpKOXpPyOYczgKmqehwgyQ3AeuChsXYlaV78PqmfPJMSDsuBp4ZeTwNvHVMvko4gfvDv0JiUcJiXJJuATe3l/03yyAJ3dQLwF4vT1SEz6T1Oen8w+T1Oen9gj3PKb8576KT9Gv6j+QyalHDYBZw09HpFq71EVV0NXD3qwZLsrKq1o+7nUJr0Hie9P5j8Hie9P7DHxTDp/c1lIiakgbuA1UlWJXkFsAHYNuaeJOmoNRFnDlW1L8klwG3AEmBLVT045rYk6ag1EeEAUFXbge2H6XAjX5o6DCa9x0nvDya/x0nvD+xxMUx6f7NKVY27B0nShJmUOQdJ0gQ56sJhkr+mI8lJSb6Z5KEkDyb5+Lh7mkuSJUm+k+QPxt3L/pIcm+SmJN9N8nCSfzrunvaX5N+33+MHklyf5O9PQE9bkuxJ8sBQ7fgkO5I82p6Pm7D+fqv9Pt+X5JYkx46rv7l6HFr3ySSV5IRx9PZyHVXhcAR8Tcc+4JNVdQpwJnDxhPU37OPAw+NuYg7/FfjDqvonwC8wYX0mWQ78O2BtVZ3G4CaMDePtCoBrgHX71TYDt1fVauD29npcrqHvbwdwWlX9PPBnwKWHu6n9XEPfI0lOAs4G/vxwN7RQR1U4MPQ1HVX1I2DmazomQlXtrqp72vJfMfhHbfl4u+olWQG8G/jSuHvZX5J/ALwD+DJAVf2oqv5yvF3N6hjgVUmOAV4N/J8x90NV3QE8t195PbC1LW8Fzj+sTQ2Zrb+q+kZV7Wsv72TwGamxmePXEOBzwK8CR8wk79EWDrN9TcfE/eMLkGQl8GbgW+PtZFa/w+AP+o/H3cgsVgF7gf/RLnt9Kclrxt3UsKraBXyGwU+Ru4EfVNU3xtvVnJZV1e62/DSwbJzNHMSvAF8fdxP7S7Ie2FVVfzruXl6Ooy0cjghJfhq4GfhEVT0/7n6GJXkPsKeq7h53L3M4BngLcFVVvRn4a8Z7KaTTrtuvZxBkrwdek+TfjLerg6vBrY0T+ZNvkl9ncFn2unH3MizJq4FfA/7juHt5uY62cJjX13SMU5KfYhAM11XVV8fdzyzeBrw3yRMMLsv9YpLfHW9LLzENTFfVzBnXTQzCYpL8EvC9qtpbVX8LfBX4Z2PuaS7PJDkRoD3vGXM/nSQfBN4D/HJN3r35b2DwQ8Cftr8zK4B7kvzDsXY1D0dbOEz013QkCYNr5Q9X1W+Pu5/ZVNWlVbWiqlYy+PX7X1U1MT/1VtXTwFNJ3tRKZzF5X/3+58CZSV7dfs/PYsImzYdsAza25Y3ArWPspZNkHYNLnO+tqh+Ou5/9VdX9VfWzVbWy/Z2ZBt7S/pxOtKMqHNrE1czXdDwM3DhhX9PxNuADDH4av7c9zht3U0egfwtcl+Q+YA3wn8fcz0u0s5qbgHuA+xn8PRz7p2iTXA/8CfCmJNNJLgKuAN6V5FEGZzxXTFh//w14LbCj/X35wrj6O0CPRyQ/IS1J6hxVZw6SpPkxHCRJHcNBktQxHCRJHcNBktQxHCRJHcNBktQxHCRJnf8HvNvihJbEGDkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist([count_substring_mismatch(x1, aa_wild_type) for x1 in x_train])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([9.880e+02, 2.745e+03, 4.431e+03, 1.444e+03, 1.504e+03, 3.050e+02,\n",
       "        2.190e+02, 2.800e+01, 1.800e+01, 2.000e+00]),\n",
       " array([ 0. ,  1.5,  3. ,  4.5,  6. ,  7.5,  9. , 10.5, 12. , 13.5, 15. ]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADzpJREFUeJzt3X+s3XV9x/Hny9bfLhalY6xtdslsNNVMIQ3iSJYFJlQxlj/UYJx2rkn/YRsuJq64ZGQqS80WUbPp0khHVSISdIEImzaAMUsmWkBR6BidorQr9moBdUZd9b0/zqfkrNzrvbe9vd/Tfp6P5OZ8v+/v55zv+3tv732d74/zbaoKSVJ/njZ0A5KkYRgAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4tH7qBX+X000+vqampoduQpJPK3Xff/f2qWjnXuIkOgKmpKXbv3j10G5J0UknynfmM8xCQJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1aqI/CayFmdp662DrfnjbJYOtW9KxcQ9AkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkTs07AJIsS3Jvks+1+bOS3JVkb5JPJ3lGqz+zze9ty6fGXuPKVn8wycWLvTGSpPlbyB7AFcCesfn3A9dU1YuAx4DNrb4ZeKzVr2njSLIOuAx4KbAB+EiSZcfXviTpWM0rAJKsBi4BPtbmA1wA3NSG7AQubdMb2zxt+YVt/Ebghqr6WVV9G9gLnLsYGyFJWrj57gF8EHgX8Ms2/0Lg8ao63Ob3Aava9CrgEYC2/Ik2/sn6DM+RJC2xOQMgyeuAg1V19xL0Q5ItSXYn2T09Pb0Uq5SkLs1nD+B84PVJHgZuYHTo50PAiiRH/lP51cD+Nr0fWAPQlj8f+MF4fYbnPKmqtlfV+qpav3LlygVvkCRpfuYMgKq6sqpWV9UUo5O4d1TVW4A7gTe0YZuAm9v0LW2etvyOqqpWv6xdJXQWsBb4yqJtiSRpQZbPPWRWfwHckOR9wL3Ata1+LfCJJHuBQ4xCg6q6P8mNwAPAYeDyqvrFcaxfknQcFhQAVfVF4Itt+lvMcBVPVf0UeOMsz78auHqhTUqSFp+fBJakThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnZozAJI8K8lXknw9yf1J/rrVz0pyV5K9ST6d5Bmt/sw2v7ctnxp7rStb/cEkF5+ojZIkzW0+ewA/Ay6oqpcDrwA2JDkPeD9wTVW9CHgM2NzGbwYea/Vr2jiSrAMuA14KbAA+kmTZYm6MJGn+5gyAGvlxm316+yrgAuCmVt8JXNqmN7Z52vILk6TVb6iqn1XVt4G9wLmLshWSpAWb1zmAJMuSfA04COwC/gt4vKoOtyH7gFVtehXwCEBb/gTwwvH6DM+RJC2xeQVAVf2iql4BrGb0rv0lJ6qhJFuS7E6ye3p6+kStRpK6t6CrgKrqceBO4FXAiiTL26LVwP42vR9YA9CWPx/4wXh9hueMr2N7Va2vqvUrV65cSHuSpAWYz1VAK5OsaNPPBl4N7GEUBG9owzYBN7fpW9o8bfkdVVWtflm7SugsYC3wlcXaEEnSwiyfewhnAjvbFTtPA26sqs8leQC4Icn7gHuBa9v4a4FPJNkLHGJ05Q9VdX+SG4EHgMPA5VX1i8XdHEnSfM0ZAFV1H3D2DPVvMcNVPFX1U+CNs7zW1cDVC29TkrTY/CSwJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdms8HwbRAU1tvHboFSZqTewCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpU3MGQJI1Se5M8kCS+5Nc0eovSLIryUPt8bRWT5IPJ9mb5L4k54y91qY2/qEkm07cZkmS5jKfPYDDwDurah1wHnB5knXAVuD2qloL3N7mAV4DrG1fW4CPwigwgKuAVwLnAlcdCQ1J0tKbMwCq6kBV3dOmfwTsAVYBG4GdbdhO4NI2vRH4eI18GViR5EzgYmBXVR2qqseAXcCGRd0aSdK8LegcQJIp4GzgLuCMqjrQFj0KnNGmVwGPjD1tX6vNVj96HVuS7E6ye3p6eiHtSZIWYN4BkOR5wGeAd1TVD8eXVVUBtRgNVdX2qlpfVetXrly5GC8pSZrBvAIgydMZ/fG/vqo+28rfa4d2aI8HW30/sGbs6atbbba6JGkA87kKKMC1wJ6q+sDYoluAI1fybAJuHqu/rV0NdB7wRDtU9HngoiSntZO/F7WaJGkAy+cx5nzgrcA3knyt1d4NbANuTLIZ+A7wprbsNuC1wF7gJ8DbAarqUJL3Al9t495TVYcWZSskSQs2ZwBU1b8BmWXxhTOML+DyWV5rB7BjIQ1Kkk4MPwksSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlT8/k/gaWJNbX11sHW/fC2SwZbt7QY3AOQpE65B6BFMeQ7cUnHxj0ASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKn5gyAJDuSHEzyzbHaC5LsSvJQezyt1ZPkw0n2JrkvyTljz9nUxj+UZNOJ2RxJ0nzNZw/gOmDDUbWtwO1VtRa4vc0DvAZY2762AB+FUWAAVwGvBM4FrjoSGpKkYcwZAFX1JeDQUeWNwM42vRO4dKz+8Rr5MrAiyZnAxcCuqjpUVY8Bu3hqqEiSltCxngM4o6oOtOlHgTPa9CrgkbFx+1pttrokaSDHfRK4qgqoRegFgCRbkuxOsnt6enqxXlaSdJTlx/i87yU5s6oOtEM8B1t9P7BmbNzqVtsP/P5R9S/O9MJVtR3YDrB+/frjCpaprbcez9Ml6ZR2rHsAtwBHruTZBNw8Vn9buxroPOCJdqjo88BFSU5rJ38vajVJ0kDm3ANI8ilG795PT7KP0dU824Abk2wGvgO8qQ2/DXgtsBf4CfB2gKo6lOS9wFfbuPdU1dEnliVJS2jOAKiqN8+y6MIZxhZw+SyvswPYsaDuJEknjJ8ElqROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpUwaAJHXKAJCkThkAktQpA0CSOmUASFKnDABJ6pQBIEmdMgAkqVMGgCR1ygCQpE4ZAJLUKQNAkjplAEhSpwwASeqUASBJnTIAJKlTBoAkdcoAkKROGQCS1CkDQJI6ZQBIUqeWD92AdLKa2nrrIOt9eNslg6xXpx73ACSpUwaAJHXKAJCkTnkOQDrJDHXuATz/cKpxD0CSOmUASFKnDABJ6pQBIEmdMgAkqVNLHgBJNiR5MMneJFuXev2SpJElDYAky4B/AF4DrAPenGTdUvYgSRpZ6s8BnAvsrapvASS5AdgIPLDEfUg6Bt7/6NSy1AGwCnhkbH4f8Mol7kHSScbgOTEm7pPASbYAW9rsj5M8eBwvdzrw/ePv6oSZ9P5g8nuc9P5g8nuc9P5goB7z/gUNn6Tv42/NZ9BSB8B+YM3Y/OpWe1JVbQe2L8bKkuyuqvWL8VonwqT3B5Pf46T3B5Pf46T3B/Z4oiz1VUBfBdYmOSvJM4DLgFuWuAdJEku8B1BVh5P8CfB5YBmwo6ruX8oeJEkjS34OoKpuA25botUtyqGkE2jS+4PJ73HS+4PJ73HS+wN7PCFSVUP3IEkagLeCkKROnZIBMOm3m0iyJsmdSR5Icn+SK4buaSZJliW5N8nnhu5lJklWJLkpyX8k2ZPkVUP3NC7Jn7ef7zeTfCrJsyagpx1JDib55ljtBUl2JXmoPZ42gT3+bfs535fkn5OsmKT+xpa9M0klOX2I3hbqlAuAk+R2E4eBd1bVOuA84PIJ7BHgCmDP0E38Ch8C/rWqXgK8nAnqNckq4M+A9VX1MkYXPVw2bFcAXAdsOKq2Fbi9qtYCt7f5IV3HU3vcBbysqn4H+E/gyqVuasx1PLU/kqwBLgK+u9QNHatTLgAYu91EVf0cOHK7iYlRVQeq6p42/SNGf7hWDdvV/5dkNXAJ8LGhe5lJkucDvwdcC1BVP6+qx4ft6imWA89Oshx4DvDfA/dDVX0JOHRUeSOws03vBC5d0qaOMlOPVfWFqjrcZr/M6DNEg5jlewhwDfAu4KQ5sXoqBsBMt5uYqD+u45JMAWcDdw3byVN8kNE/5l8O3cgszgKmgX9qh6k+luS5Qzd1RFXtB/6O0bvBA8ATVfWFYbua1RlVdaBNPwqcMWQz8/DHwL8M3cS4JBuB/VX19aF7WYhTMQBOGkmeB3wGeEdV/XDofo5I8jrgYFXdPXQvv8Jy4Bzgo1V1NvA/DH/o4kntOPpGRkH1m8Bzk/zhsF3NrUaXBU7sO9gkf8noEOr1Q/dyRJLnAO8G/mroXhbqVAyAOW83MQmSPJ3RH//rq+qzQ/dzlPOB1yd5mNEhtAuSfHLYlp5iH7Cvqo7sOd3EKBAmxR8A366q6ar6X+CzwO8O3NNsvpfkTID2eHDgfmaU5I+A1wFvqcm6fv23GQX919vvzGrgniS/MWhX83AqBsDE324iSRgdu95TVR8Yup+jVdWVVbW6qqYYff/uqKqJevdaVY8CjyR5cStdyGTdVvy7wHlJntN+3hcyQSepj3ILsKlNbwJuHrCXGSXZwOiQ5Our6idD9zOuqr5RVb9eVVPtd2YfcE77NzrRTrkAaCeKjtxuYg9w4wTebuJ84K2M3ll/rX29duimTkJ/Clyf5D7gFcDfDNzPk9qeyU3APcA3GP2uDf5J0SSfAv4deHGSfUk2A9uAVyd5iNGey7YJ7PHvgV8DdrXfl3+csP5OSn4SWJI6dcrtAUiS5scAkKROGQCS1CkDQJI6ZQBIUqcMAEnqlAEgSZ0yACSpU/8HkEngqsluevMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist([count_substring_mismatch(x1, aa_wild_type) for x1 in x_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(46733,)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "238"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x_train[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Refresh python syntax in kwargs and class inheritance\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "def greet_me(**kwargs):\n",
    "    if kwargs is not None:\n",
    "        for key, value in kwargs.items():\n",
    "            print(\"%s == %s\" % (key,value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pass_kwargs(x = 3, **kwargs):\n",
    "    greet_me(**kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'tmp': 2, 'math': 4}\n",
      "tmp == 2\n",
      "math == 4\n"
     ]
    }
   ],
   "source": [
    "d = {\"tmp\":2, \"math\":4}\n",
    "pass_kwargs(x = 3, **d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "class S: \n",
    "    def print_hello(self):\n",
    "        print(\"hello\")\n",
    "        \n",
    "class B(S): \n",
    "    def print_goodbye(self): \n",
    "        super().print_hello()\n",
    "        print('goodbye')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello\n",
      "goodbye\n"
     ]
    }
   ],
   "source": [
    "B().print_goodbye()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Print Out Previous Experiment Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_bash(cmd, queue=None):\n",
    "    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, executable='/bin/bash')\n",
    "    std_out = p.stdout.read().strip()\n",
    "    if p.stderr:\n",
    "        std_err = p.stderr.read().strip()\n",
    "    else:\n",
    "        std_err = None\n",
    "    if queue is not None:\n",
    "        queue.put(std_out)\n",
    "    return std_out, std_err  # This is the stdout from the shell command\n",
    "\n",
    "std_out_lst, std_err_lst = [], []\n",
    "for i in range(0, 27):\n",
    "    std_out, std_err = run_bash(\"ts -i {0}\".format(i))\n",
    "    std_out = std_out.decode('utf-8') if std_out else None\n",
    "    std_err = std_err.decode('utf-8') if std_err else None\n",
    "    std_out_lst.append(std_out)\n",
    "    std_err_lst.append(std_err)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial 0 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type hmm --base_log logs/synthetic_unimodal_data_length_100_gaussian/hmm --name hmm_default_small --input 2000 --hidden_size 30 --latent_dim -1 --seq_length 100 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_100_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:34:09 2019\n",
      "End time: Thu May  9 03:35:59 2019\n",
      "Time run: 110.697662s\n",
      "--------------------------------------------------\n",
      "Trial 0 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 1 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type rnn --base_log logs/synthetic_unimodal_data_length_100_gaussian/rnn --name rnn_default_small --input 2000 --hidden_size 100 --latent_dim -1 --seq_length 100 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_100_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:34:09 2019\n",
      "End time: Thu May  9 03:35:34 2019\n",
      "Time run: 85.472137s\n",
      "--------------------------------------------------\n",
      "Trial 1 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 2 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type vae --base_log logs/synthetic_unimodal_data_length_100_gaussian/vae --name vae_default_small --input 2000 --hidden_size 50 --latent_dim 20 --seq_length 100 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_100_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:34:09 2019\n",
      "End time: Thu May  9 03:34:28 2019\n",
      "Time run: 19.009546s\n",
      "--------------------------------------------------\n",
      "Trial 2 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 3 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type hmm --base_log logs/synthetic_unimodal_data_length_100_skewed_gaussian/hmm --name hmm_default_small --input 2000 --hidden_size 30 --latent_dim -1 --seq_length 100 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_100_skewed_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:34:28 2019\n",
      "End time: Thu May  9 03:36:15 2019\n",
      "Time run: 107.837997s\n",
      "--------------------------------------------------\n",
      "Trial 3 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 4 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type rnn --base_log logs/synthetic_unimodal_data_length_100_skewed_gaussian/rnn --name rnn_default_small --input 2000 --hidden_size 100 --latent_dim -1 --seq_length 100 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_100_skewed_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:35:34 2019\n",
      "End time: Thu May  9 03:36:46 2019\n",
      "Time run: 72.306396s\n",
      "--------------------------------------------------\n",
      "Trial 4 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 5 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type vae --base_log logs/synthetic_unimodal_data_length_100_skewed_gaussian/vae --name vae_default_small --input 2000 --hidden_size 50 --latent_dim 20 --seq_length 100 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_100_skewed_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:35:59 2019\n",
      "End time: Thu May  9 03:36:15 2019\n",
      "Time run: 15.577449s\n",
      "--------------------------------------------------\n",
      "Trial 5 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 6 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type hmm --base_log logs/synthetic_unimodal_data_length_100_uniform/hmm --name hmm_default_small --input 2000 --hidden_size 30 --latent_dim -1 --seq_length 100 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_100_uniform --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:36:15 2019\n",
      "End time: Thu May  9 03:37:40 2019\n",
      "Time run: 85.302139s\n",
      "--------------------------------------------------\n",
      "Trial 6 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 7 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type rnn --base_log logs/synthetic_unimodal_data_length_100_uniform/rnn --name rnn_default_small --input 2000 --hidden_size 100 --latent_dim -1 --seq_length 100 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_100_uniform --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:36:15 2019\n",
      "End time: Thu May  9 03:37:27 2019\n",
      "Time run: 71.316887s\n",
      "--------------------------------------------------\n",
      "Trial 7 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 8 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type vae --base_log logs/synthetic_unimodal_data_length_100_uniform/vae --name vae_default_small --input 2000 --hidden_size 50 --latent_dim 20 --seq_length 100 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_100_uniform --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:36:46 2019\n",
      "End time: Thu May  9 03:37:04 2019\n",
      "Time run: 17.567320s\n",
      "--------------------------------------------------\n",
      "Trial 8 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 9 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type hmm --base_log logs/synthetic_unimodal_data_length_20_gaussian/hmm --name hmm_default_small --input 400 --hidden_size 30 --latent_dim -1 --seq_length 20 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_20_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:37:04 2019\n",
      "End time: Thu May  9 03:37:31 2019\n",
      "Time run: 26.821106s\n",
      "--------------------------------------------------\n",
      "Trial 9 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 10 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type rnn --base_log logs/synthetic_unimodal_data_length_20_gaussian/rnn --name rnn_default_small --input 400 --hidden_size 100 --latent_dim -1 --seq_length 20 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_20_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:37:27 2019\n",
      "End time: Thu May  9 03:37:45 2019\n",
      "Time run: 18.565990s\n",
      "--------------------------------------------------\n",
      "Trial 10 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 11 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type vae --base_log logs/synthetic_unimodal_data_length_20_gaussian/vae --name vae_default_small --input 400 --hidden_size 50 --latent_dim 20 --seq_length 20 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_20_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:37:31 2019\n",
      "End time: Thu May  9 03:37:41 2019\n",
      "Time run: 10.690344s\n",
      "--------------------------------------------------\n",
      "Trial 11 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 12 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type hmm --base_log logs/synthetic_unimodal_data_length_20_skewed_gaussian/hmm --name hmm_default_small --input 400 --hidden_size 30 --latent_dim -1 --seq_length 20 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_20_skewed_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:37:40 2019\n",
      "End time: Thu May  9 03:38:07 2019\n",
      "Time run: 26.557686s\n",
      "--------------------------------------------------\n",
      "Trial 12 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 13 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type rnn --base_log logs/synthetic_unimodal_data_length_20_skewed_gaussian/rnn --name rnn_default_small --input 400 --hidden_size 100 --latent_dim -1 --seq_length 20 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_20_skewed_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:37:41 2019\n",
      "End time: Thu May  9 03:38:01 2019\n",
      "Time run: 19.611984s\n",
      "--------------------------------------------------\n",
      "Trial 13 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 14 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type vae --base_log logs/synthetic_unimodal_data_length_20_skewed_gaussian/vae --name vae_default_small --input 400 --hidden_size 50 --latent_dim 20 --seq_length 20 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_20_skewed_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:37:45 2019\n",
      "End time: Thu May  9 03:37:56 2019\n",
      "Time run: 10.380044s\n",
      "--------------------------------------------------\n",
      "Trial 14 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 15 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type hmm --base_log logs/synthetic_unimodal_data_length_20_uniform/hmm --name hmm_default_small --input 400 --hidden_size 30 --latent_dim -1 --seq_length 20 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_20_uniform --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:37:56 2019\n",
      "End time: Thu May  9 03:38:29 2019\n",
      "Time run: 33.571880s\n",
      "--------------------------------------------------\n",
      "Trial 15 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 16 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type rnn --base_log logs/synthetic_unimodal_data_length_20_uniform/rnn --name rnn_default_small --input 400 --hidden_size 100 --latent_dim -1 --seq_length 20 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_20_uniform --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:38:01 2019\n",
      "End time: Thu May  9 03:38:27 2019\n",
      "Time run: 25.696617s\n",
      "--------------------------------------------------\n",
      "Trial 16 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 17 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type vae --base_log logs/synthetic_unimodal_data_length_20_uniform/vae --name vae_default_small --input 400 --hidden_size 50 --latent_dim 20 --seq_length 20 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_20_uniform --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:38:07 2019\n",
      "End time: Thu May  9 03:38:22 2019\n",
      "Time run: 15.446612s\n",
      "--------------------------------------------------\n",
      "Trial 17 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 18 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type hmm --base_log logs/synthetic_unimodal_data_length_50_gaussian/hmm --name hmm_default_small --input 1000 --hidden_size 30 --latent_dim -1 --seq_length 50 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_50_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:38:22 2019\n",
      "End time: Thu May  9 03:39:07 2019\n",
      "Time run: 45.318989s\n",
      "--------------------------------------------------\n",
      "Trial 18 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 19 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type rnn --base_log logs/synthetic_unimodal_data_length_50_gaussian/rnn --name rnn_default_small --input 1000 --hidden_size 100 --latent_dim -1 --seq_length 50 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_50_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:38:27 2019\n",
      "End time: Thu May  9 03:38:59 2019\n",
      "Time run: 31.918270s\n",
      "--------------------------------------------------\n",
      "Trial 19 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 20 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type vae --base_log logs/synthetic_unimodal_data_length_50_gaussian/vae --name vae_default_small --input 1000 --hidden_size 50 --latent_dim 20 --seq_length 50 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_50_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:38:29 2019\n",
      "End time: Thu May  9 03:38:40 2019\n",
      "Time run: 10.535923s\n",
      "--------------------------------------------------\n",
      "Trial 20 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 21 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type hmm --base_log logs/synthetic_unimodal_data_length_50_skewed_gaussian/hmm --name hmm_default_small --input 1000 --hidden_size 30 --latent_dim -1 --seq_length 50 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_50_skewed_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:38:40 2019\n",
      "End time: Thu May  9 03:39:30 2019\n",
      "Time run: 50.260647s\n",
      "--------------------------------------------------\n",
      "Trial 21 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 22 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type rnn --base_log logs/synthetic_unimodal_data_length_50_skewed_gaussian/rnn --name rnn_default_small --input 1000 --hidden_size 100 --latent_dim -1 --seq_length 50 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_50_skewed_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:38:59 2019\n",
      "End time: Thu May  9 03:39:47 2019\n",
      "Time run: 48.647903s\n",
      "--------------------------------------------------\n",
      "Trial 22 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 23 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type vae --base_log logs/synthetic_unimodal_data_length_50_skewed_gaussian/vae --name vae_default_small --input 1000 --hidden_size 50 --latent_dim 20 --seq_length 50 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_50_skewed_gaussian --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:39:07 2019\n",
      "End time: Thu May  9 03:39:24 2019\n",
      "Time run: 16.212975s\n",
      "--------------------------------------------------\n",
      "Trial 23 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 24 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type hmm --base_log logs/synthetic_unimodal_data_length_50_uniform/hmm --name hmm_default_small --input 1000 --hidden_size 30 --latent_dim -1 --seq_length 50 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_50_uniform --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:39:24 2019\n",
      "End time: Thu May  9 03:40:24 2019\n",
      "Time run: 60.193420s\n",
      "--------------------------------------------------\n",
      "Trial 24 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 25 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type rnn --base_log logs/synthetic_unimodal_data_length_50_uniform/rnn --name rnn_default_small --input 1000 --hidden_size 100 --latent_dim -1 --seq_length 50 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_50_uniform --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:39:30 2019\n",
      "End time: Thu May  9 03:40:15 2019\n",
      "Time run: 44.894474s\n",
      "--------------------------------------------------\n",
      "Trial 25 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n",
      "Trial 26 Standard out:\n",
      "--------------------------------------------------\n",
      "Exit status: died with exit code 0\n",
      "Command: bash -c python3 run_model.py --model_type vae --base_log logs/synthetic_unimodal_data_length_50_uniform/vae --name vae_default_small --input 1000 --hidden_size 50 --latent_dim 20 --seq_length 50 --pseudo_count 1 --n_jobs 1 --device cpu --learning_rate 0.001 --epochs 10 --batch_size 10 --layers 1 --dataset synthetic_unimodal_data_length_50_uniform --num_data 100\n",
      "Slots required: 1\n",
      "Enqueue time: Thu May  9 03:34:09 2019\n",
      "Start time: Thu May  9 03:39:47 2019\n",
      "End time: Thu May  9 03:40:03 2019\n",
      "Time run: 15.398121s\n",
      "--------------------------------------------------\n",
      "Trial 26 Standard err:\n",
      "--------------------------------------------------\n",
      "None\n",
      "==================================================\n"
     ]
    }
   ],
   "source": [
    "for i, (std_out, std_err) in enumerate(zip(std_out_lst, std_err_lst)):\n",
    "    print(\"Trial {0} Standard out:\".format(i))\n",
    "    print(\"-\" * 50)\n",
    "    print(std_out)\n",
    "    print(\"-\" * 50)\n",
    "    print(\"Trial {0} Standard err:\".format(i))\n",
    "    print(\"-\" * 50)\n",
    "    print(std_err)\n",
    "    print(\"=\" * 50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## mutation counts of datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset: synthetic_multimodal_data_modes_2_length_51_uniform.csv, average mutation count: 5.5\n",
      "dataset: synthetic_multimodal_data_modes_3_length_51_uniform.csv, average mutation count: 5.5\n",
      "dataset: synthetic_multimodal_data_modes_5_length_51_uniform.csv, average mutation count: 5.5\n",
      "dataset: synthetic_mutations_21_unimodal_data_length_100_uniform.csv, average mutation count: 11.0089\n",
      "dataset: synthetic_mutations_21_unimodal_data_length_50_uniform.csv, average mutation count: 11.0089\n",
      "dataset: synthetic_unimodal_data_length_100_gaussian.csv, average mutation count: 5.5079\n",
      "dataset: synthetic_unimodal_data_length_100_skewed_gaussian.csv, average mutation count: 5.05046\n",
      "dataset: synthetic_unimodal_data_length_100_uniform.csv, average mutation count: 5.51174\n",
      "dataset: synthetic_unimodal_data_length_20_gaussian.csv, average mutation count: 5.5079\n",
      "dataset: synthetic_unimodal_data_length_20_skewed_gaussian.csv, average mutation count: 5.05046\n",
      "dataset: synthetic_unimodal_data_length_20_uniform.csv, average mutation count: 5.51174\n",
      "dataset: synthetic_unimodal_data_length_50_gaussian.csv, average mutation count: 5.5079\n",
      "dataset: synthetic_unimodal_data_length_50_skewed_gaussian.csv, average mutation count: 5.05046\n",
      "dataset: synthetic_unimodal_data_length_50_uniform.csv, average mutation count: 5.51174\n"
     ]
    }
   ],
   "source": [
    "for dataset in os.listdir(\"./data/\"): \n",
    "    if \".csv\" in dataset: \n",
    "        df = pd.read_csv(\"./data/\" + dataset)\n",
    "        if \"mutation_count\" not in df.columns:\n",
    "            continue\n",
    "        print(\"dataset: {0}, average mutation count: {1}\".format(dataset, np.mean(df[\"mutation_count\"].values)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset: gfp, average mutation count: 3.7105123508567712\n"
     ]
    }
   ],
   "source": [
    "print('dataset: gfp, average mutation count: {0}'.format(pd.read_csv(\"./data/gfp_data.csv\")[\"numAAMutations\"].mean()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check all 27 files\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./logs/\n",
      "./logs/gfp\n",
      "./logs/gfp/hmm\n",
      "./logs/gfp/hmm/hmm_default_medium\n",
      "Press enter to continue\n",
      "./logs/gfp/hmm/hmm_default_small\n",
      "Press enter to continue\n",
      "./logs/gfp/rnn\n",
      "./logs/gfp/rnn/rnn_default_medium\n",
      "Press enter to continue\n",
      "./logs/gfp/rnn/rnn_default_small\n",
      "Press enter to continue\n",
      "./logs/gfp/vae\n",
      "./logs/gfp/vae/vae_default_large\n",
      "Press enter to continue\n",
      "./logs/gfp/vae/vae_default_medium\n",
      "Press enter to continue\n",
      "./logs/gfp/vae/vae_default_small\n",
      "Press enter to continue\n",
      "./logs/synthetic_multimodal_data_modes_2_length_51_uniform\n",
      "./logs/synthetic_multimodal_data_modes_2_length_51_uniform/hmm\n",
      "./logs/synthetic_multimodal_data_modes_2_length_51_uniform/hmm/hmm_default_medium\n",
      "Press enter to continue\n",
      "./logs/synthetic_multimodal_data_modes_2_length_51_uniform/hmm/hmm_default_small\n",
      "Press enter to continue\n",
      "./logs/synthetic_multimodal_data_modes_2_length_51_uniform/rnn\n",
      "./logs/synthetic_multimodal_data_modes_2_length_51_uniform/rnn/rnn_default_medium\n",
      "Press enter to continue\n",
      "./logs/synthetic_multimodal_data_modes_2_length_51_uniform/rnn/rnn_default_small\n",
      "Press enter to continue\n",
      "./logs/synthetic_multimodal_data_modes_2_length_51_uniform/vae\n",
      "./logs/synthetic_multimodal_data_modes_2_length_51_uniform/vae/vae_default_medium\n",
      "Press enter to continue\n",
      "./logs/synthetic_multimodal_data_modes_2_length_51_uniform/vae/vae_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_multimodal_data_modes_3_length_51_uniform\n",
      "./logs/synthetic_multimodal_data_modes_3_length_51_uniform/hmm\n",
      "./logs/synthetic_multimodal_data_modes_3_length_51_uniform/hmm/hmm_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_multimodal_data_modes_3_length_51_uniform/hmm/hmm_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_multimodal_data_modes_3_length_51_uniform/rnn\n",
      "./logs/synthetic_multimodal_data_modes_3_length_51_uniform/rnn/rnn_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_multimodal_data_modes_3_length_51_uniform/rnn/rnn_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_multimodal_data_modes_3_length_51_uniform/vae\n",
      "./logs/synthetic_multimodal_data_modes_3_length_51_uniform/vae/vae_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_multimodal_data_modes_3_length_51_uniform/vae/vae_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_multimodal_data_modes_5_length_51_uniform\n",
      "./logs/synthetic_multimodal_data_modes_5_length_51_uniform/hmm\n",
      "./logs/synthetic_multimodal_data_modes_5_length_51_uniform/hmm/hmm_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_multimodal_data_modes_5_length_51_uniform/hmm/hmm_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_multimodal_data_modes_5_length_51_uniform/rnn\n",
      "./logs/synthetic_multimodal_data_modes_5_length_51_uniform/rnn/rnn_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_multimodal_data_modes_5_length_51_uniform/rnn/rnn_default_small\n",
      "Press enter to continue\n",
      "./logs/synthetic_multimodal_data_modes_5_length_51_uniform/vae\n",
      "./logs/synthetic_multimodal_data_modes_5_length_51_uniform/vae/vae_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_multimodal_data_modes_5_length_51_uniform/vae/vae_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_100_gaussian\n",
      "./logs/synthetic_unimodal_data_length_100_gaussian/hmm\n",
      "./logs/synthetic_unimodal_data_length_100_gaussian/hmm/hmm_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_100_gaussian/rnn\n",
      "./logs/synthetic_unimodal_data_length_100_gaussian/rnn/rnn_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_100_gaussian/rnn/rnn_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_100_gaussian/vae\n",
      "./logs/synthetic_unimodal_data_length_100_gaussian/vae/vae_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_100_gaussian/vae/vae_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_100_skewed_gaussian\n",
      "./logs/synthetic_unimodal_data_length_100_skewed_gaussian/hmm\n",
      "./logs/synthetic_unimodal_data_length_100_skewed_gaussian/hmm/hmm_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_100_skewed_gaussian/rnn\n",
      "./logs/synthetic_unimodal_data_length_100_skewed_gaussian/rnn/rnn_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_100_skewed_gaussian/rnn/rnn_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_100_skewed_gaussian/vae\n",
      "./logs/synthetic_unimodal_data_length_100_skewed_gaussian/vae/vae_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_100_skewed_gaussian/vae/vae_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_100_uniform\n",
      "./logs/synthetic_unimodal_data_length_100_uniform/hmm\n",
      "./logs/synthetic_unimodal_data_length_100_uniform/hmm/hmm_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_100_uniform/rnn\n",
      "./logs/synthetic_unimodal_data_length_100_uniform/rnn/rnn_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_100_uniform/rnn/rnn_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_100_uniform/vae\n",
      "./logs/synthetic_unimodal_data_length_100_uniform/vae/vae_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_100_uniform/vae/vae_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_20_gaussian\n",
      "./logs/synthetic_unimodal_data_length_20_gaussian/hmm\n",
      "./logs/synthetic_unimodal_data_length_20_gaussian/hmm/hmm_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_20_gaussian/hmm/hmm_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_20_gaussian/rnn\n",
      "./logs/synthetic_unimodal_data_length_20_gaussian/rnn/rnn_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_20_gaussian/rnn/rnn_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_20_gaussian/vae\n",
      "./logs/synthetic_unimodal_data_length_20_gaussian/vae/vae_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_20_gaussian/vae/vae_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_20_skewed_gaussian\n",
      "./logs/synthetic_unimodal_data_length_20_skewed_gaussian/hmm\n",
      "./logs/synthetic_unimodal_data_length_20_skewed_gaussian/hmm/hmm_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_20_skewed_gaussian/rnn\n",
      "./logs/synthetic_unimodal_data_length_20_skewed_gaussian/rnn/rnn_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_20_skewed_gaussian/rnn/rnn_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_20_skewed_gaussian/vae\n",
      "./logs/synthetic_unimodal_data_length_20_skewed_gaussian/vae/vae_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_20_skewed_gaussian/vae/vae_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_20_uniform\n",
      "./logs/synthetic_unimodal_data_length_20_uniform/hmm\n",
      "./logs/synthetic_unimodal_data_length_20_uniform/hmm/hmm_default_small\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_20_uniform/rnn\n",
      "./logs/synthetic_unimodal_data_length_20_uniform/rnn/rnn_default_medium\n",
      "Press enter to continuec\n",
      "./logs/synthetic_unimodal_data_length_20_uniform/rnn/rnn_default_small\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m~/Library/Python/3.6/lib/python/site-packages/ipykernel/kernelbase.py\u001b[0m in \u001b[0;36m_input_request\u001b[0;34m(self, prompt, ident, parent, password)\u001b[0m\n\u001b[1;32m    728\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 729\u001b[0;31m                 \u001b[0mident\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreply\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstdin_socket\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    730\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Library/Python/3.6/lib/python/site-packages/jupyter_client/session.py\u001b[0m in \u001b[0;36mrecv\u001b[0;34m(self, socket, mode, content, copy)\u001b[0m\n\u001b[1;32m    802\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 803\u001b[0;31m             \u001b[0mmsg_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msocket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_multipart\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    804\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mzmq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mZMQError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Library/Python/3.6/lib/python/site-packages/zmq/sugar/socket.py\u001b[0m in \u001b[0;36mrecv_multipart\u001b[0;34m(self, flags, copy, track)\u001b[0m\n\u001b[1;32m    465\u001b[0m         \"\"\"\n\u001b[0;32m--> 466\u001b[0;31m         \u001b[0mparts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mflags\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrack\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrack\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    467\u001b[0m         \u001b[0;31m# have first part already, only loop while more to receive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket.Socket.recv\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket.Socket.recv\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket._recv_copy\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m~/Library/Python/3.6/lib/python/site-packages/zmq/backend/cython/checkrc.pxd\u001b[0m in \u001b[0;36mzmq.backend.cython.checkrc._check_rc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-406417866bf1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m                 \u001b[0mrun_bash\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"open {0}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m             \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Press enter to continue\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mSyntaxError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m             \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Library/Python/3.6/lib/python/site-packages/ipykernel/kernelbase.py\u001b[0m in \u001b[0;36mraw_input\u001b[0;34m(self, prompt)\u001b[0m\n\u001b[1;32m    702\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent_ident\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    703\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent_header\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 704\u001b[0;31m             \u001b[0mpassword\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    705\u001b[0m         )\n\u001b[1;32m    706\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Library/Python/3.6/lib/python/site-packages/ipykernel/kernelbase.py\u001b[0m in \u001b[0;36m_input_request\u001b[0;34m(self, prompt, ident, parent, password)\u001b[0m\n\u001b[1;32m    732\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    733\u001b[0m                 \u001b[0;31m# re-raise KeyboardInterrupt, to truncate traceback\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 734\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    735\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    736\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for dirpath, dirnames, filenames in os.walk(\"./logs/\"):\n",
    "    print(dirpath)\n",
    "    if not dirnames:\n",
    "        for file in filenames: \n",
    "            file_name = os.path.join(dirpath, file)\n",
    "            if \"log.txt\" in file_name: \n",
    "                run_bash(\"open {0}\".format(file_name))\n",
    "        try:\n",
    "            x = input(\"Press enter to continue\")\n",
    "        except SyntaxError:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## try catch block"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "File b'./a' does not exist",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-135-342e8fcbdf7f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     15\u001b[0m         \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mexc_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-135-342e8fcbdf7f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"./a\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mlogger\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"./tmp.txt\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"w+\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Library/Python/3.6/lib/python/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, doublequote, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    676\u001b[0m                     skip_blank_lines=skip_blank_lines)\n\u001b[1;32m    677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 678\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    679\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    680\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Library/Python/3.6/lib/python/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    438\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    439\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 440\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    441\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    442\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Library/Python/3.6/lib/python/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    785\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    786\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 787\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    788\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    789\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Library/Python/3.6/lib/python/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1012\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1013\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1014\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1015\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'python'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Library/Python/3.6/lib/python/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   1706\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'usecols'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1707\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1708\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1709\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1710\u001b[0m         \u001b[0mpassed_names\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnames\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: File b'./a' does not exist"
     ]
    }
   ],
   "source": [
    "x = False\n",
    "try: \n",
    "    pd.read_csv(\"./a\")\n",
    "except Exception as e:\n",
    "    logger = open(\"./tmp.txt\", \"w+\")\n",
    "    exc_type, exc_value, exc_traceback = sys.exc_info()\n",
    "    print(\"=\" * 50, file=logger)\n",
    "    print(\"Error in running experiment: {0}\\nTraceback log: \".format(\"1\"), file=logger)\n",
    "    print(\"=\" * 50, file=logger)\n",
    "    traceback.print_tb(exc_traceback, file=logger)\n",
    "    x = True\n",
    "finally:\n",
    "    print(1)\n",
    "    if logger:\n",
    "        logger.close()\n",
    "    if x: \n",
    "        raise exc_value\n",
    "    else: \n",
    "        print(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "!open tmp.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm tmp.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "10000000\n",
      "20000000\n",
      "30000000\n",
      "python\n"
     ]
    }
   ],
   "source": [
    "try: \n",
    "    for i in range(10000000000): \n",
    "        if i % 10000000 == 0: \n",
    "            print(i)\n",
    "except: \n",
    "    print(\"python\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "10000000\n",
      "20000000\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-145-b03458565a3e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10000000000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m10000000\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# these two are different \n",
    "try: \n",
    "    for i in range(10000000000): \n",
    "        if i % 10000000 == 0: \n",
    "            print(i)\n",
    "except Exception as e: \n",
    "    print(\"python\") "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyzing results\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "hmm_df = pd.read_csv(\"./logs/optimized_results/hmm_grid_search_datasets_sample_std_out.csv\")\n",
    "rnn_df = pd.read_csv('./logs/optimized_results/rnn_grid_search_datasets_sample_std_out.csv')\n",
    "vae_df = pd.read_csv(\"./logs/optimized_results/vae_grid_search_datasets_sample_std_out.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "156\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_type</th>\n",
       "      <th>test_score</th>\n",
       "      <th>total_time</th>\n",
       "      <th>dataset</th>\n",
       "      <th>average_mismatches</th>\n",
       "      <th>valid_score</th>\n",
       "      <th>train_score</th>\n",
       "      <th>num_params</th>\n",
       "      <th>exit_code</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>vae</td>\n",
       "      <td>(39.45875930786133, 39.16001510620117, 0.29873...</td>\n",
       "      <td>15.426934</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.670</td>\n",
       "      <td>(39.78868865966797, 39.482662200927734, 0.3060...</td>\n",
       "      <td>(36.52116394042969, 35.80773162841797, 0.71344...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>vae</td>\n",
       "      <td>(39.16112518310547, 39.01034927368164, 0.15078...</td>\n",
       "      <td>19.651253</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.782</td>\n",
       "      <td>(39.471614837646484, 39.31938171386719, 0.1522...</td>\n",
       "      <td>(36.4890251159668, 36.09037780761719, 0.398657...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>vae</td>\n",
       "      <td>(39.361846923828125, 39.257869720458984, 0.103...</td>\n",
       "      <td>36.790556</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.507</td>\n",
       "      <td>(39.65810012817383, 39.54773712158203, 0.11037...</td>\n",
       "      <td>(36.50764846801758, 36.250755310058594, 0.2568...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>vae</td>\n",
       "      <td>(39.323123931884766, 39.019893646240234, 0.303...</td>\n",
       "      <td>12.772545</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.661</td>\n",
       "      <td>(39.59053421020508, 39.301246643066406, 0.2892...</td>\n",
       "      <td>(36.4285774230957, 35.69258499145508, 0.735994...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>vae</td>\n",
       "      <td>(39.155860900878906, 38.941837310791016, 0.214...</td>\n",
       "      <td>17.382276</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.627</td>\n",
       "      <td>(39.49126052856445, 39.2797966003418, 0.211458...</td>\n",
       "      <td>(36.368534088134766, 35.817726135253906, 0.550...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>vae</td>\n",
       "      <td>(39.330284118652344, 39.17107391357422, 0.1592...</td>\n",
       "      <td>33.281378</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.622</td>\n",
       "      <td>(39.599281311035156, 39.44047164916992, 0.1588...</td>\n",
       "      <td>(36.400875091552734, 35.98995590209961, 0.4109...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>vae</td>\n",
       "      <td>(39.997737884521484, 39.63236618041992, 0.3653...</td>\n",
       "      <td>14.326173</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>6.099</td>\n",
       "      <td>(40.4110221862793, 40.04662322998047, 0.364383...</td>\n",
       "      <td>(37.09550094604492, 36.323673248291016, 0.7718...</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>vae</td>\n",
       "      <td>(39.261268615722656, 39.04073715209961, 0.2205...</td>\n",
       "      <td>17.181729</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.647</td>\n",
       "      <td>(39.46714782714844, 39.24443435668945, 0.22271...</td>\n",
       "      <td>(36.46193313598633, 35.90756607055664, 0.55436...</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>vae</td>\n",
       "      <td>(39.362857818603516, 39.06566619873047, 0.2971...</td>\n",
       "      <td>29.666096</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.501</td>\n",
       "      <td>(39.70541763305664, 39.4078254699707, 0.297588...</td>\n",
       "      <td>(36.08356857299805, 35.264488220214844, 0.8190...</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>vae</td>\n",
       "      <td>(39.86193084716797, 39.19924545288086, 0.66268...</td>\n",
       "      <td>14.487885</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.503</td>\n",
       "      <td>(40.134185791015625, 39.46178436279297, 0.6724...</td>\n",
       "      <td>(35.921974182128906, 34.30979919433594, 1.6121...</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>vae</td>\n",
       "      <td>(39.84731674194336, 38.90536117553711, 0.94195...</td>\n",
       "      <td>17.265143</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.544</td>\n",
       "      <td>(40.28424072265625, 39.3470458984375, 0.937189...</td>\n",
       "      <td>(34.53658676147461, 32.040687561035156, 2.4958...</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>vae</td>\n",
       "      <td>(39.295494079589844, 38.98809051513672, 0.3074...</td>\n",
       "      <td>27.626355</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.439</td>\n",
       "      <td>(39.516143798828125, 39.21133804321289, 0.3047...</td>\n",
       "      <td>(35.81599807739258, 34.90946578979492, 0.90652...</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>vae</td>\n",
       "      <td>(36.9610481262207, 36.645816802978516, 0.31523...</td>\n",
       "      <td>14.884972</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.122</td>\n",
       "      <td>(36.68040466308594, 36.363861083984375, 0.3165...</td>\n",
       "      <td>(33.846797943115234, 33.086181640625, 0.760626...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>vae</td>\n",
       "      <td>(36.600669860839844, 36.4581184387207, 0.14254...</td>\n",
       "      <td>20.822176</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.053</td>\n",
       "      <td>(36.35088348388672, 36.20940399169922, 0.14148...</td>\n",
       "      <td>(33.91124725341797, 33.56279373168945, 0.34845...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>vae</td>\n",
       "      <td>(36.782432556152344, 36.698638916015625, 0.083...</td>\n",
       "      <td>38.685860</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.205</td>\n",
       "      <td>(36.515647888183594, 36.4343147277832, 0.08132...</td>\n",
       "      <td>(33.96339797973633, 33.768226623535156, 0.1951...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>vae</td>\n",
       "      <td>(36.60530090332031, 36.46498489379883, 0.14031...</td>\n",
       "      <td>12.527444</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.314</td>\n",
       "      <td>(36.36986541748047, 36.233642578125, 0.1362221...</td>\n",
       "      <td>(34.19662094116211, 33.915653228759766, 0.2809...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>vae</td>\n",
       "      <td>(36.47758102416992, 36.37152862548828, 0.10605...</td>\n",
       "      <td>16.454788</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.153</td>\n",
       "      <td>(36.25579071044922, 36.14877700805664, 0.10701...</td>\n",
       "      <td>(33.836151123046875, 33.56602096557617, 0.2701...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>vae</td>\n",
       "      <td>(36.74794006347656, 36.62401580810547, 0.12391...</td>\n",
       "      <td>31.721673</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.157</td>\n",
       "      <td>(36.49137878417969, 36.36961364746094, 0.12176...</td>\n",
       "      <td>(33.808631896972656, 33.48420715332031, 0.3244...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>vae</td>\n",
       "      <td>(37.333290100097656, 36.990840911865234, 0.342...</td>\n",
       "      <td>14.093270</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.424</td>\n",
       "      <td>(37.04022979736328, 36.6895751953125, 0.350664...</td>\n",
       "      <td>(34.442848205566406, 33.8046989440918, 0.63814...</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>vae</td>\n",
       "      <td>(36.786739349365234, 36.455448150634766, 0.331...</td>\n",
       "      <td>19.671381</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.201</td>\n",
       "      <td>(36.52296829223633, 36.19729995727539, 0.32566...</td>\n",
       "      <td>(33.50249481201172, 32.650569915771484, 0.8519...</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>vae</td>\n",
       "      <td>(36.61471176147461, 36.50148391723633, 0.11323...</td>\n",
       "      <td>26.937424</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.354</td>\n",
       "      <td>(36.38849639892578, 36.27131271362305, 0.11719...</td>\n",
       "      <td>(33.94681167602539, 33.67296600341797, 0.27384...</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>vae</td>\n",
       "      <td>(36.92631530761719, 36.619850158691406, 0.3064...</td>\n",
       "      <td>12.170369</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.110</td>\n",
       "      <td>(36.71547317504883, 36.41338348388672, 0.30208...</td>\n",
       "      <td>(34.00155258178711, 33.302734375, 0.6988178491...</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>vae</td>\n",
       "      <td>(36.26698303222656, 36.16910934448242, 0.09786...</td>\n",
       "      <td>14.382359</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.072</td>\n",
       "      <td>(36.10603332519531, 36.00755310058594, 0.09848...</td>\n",
       "      <td>(34.024688720703125, 33.7932014465332, 0.23148...</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>vae</td>\n",
       "      <td>(36.54599380493164, 36.37313461303711, 0.17286...</td>\n",
       "      <td>26.472795</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>4.958</td>\n",
       "      <td>(36.35851287841797, 36.18794250488281, 0.17056...</td>\n",
       "      <td>(33.659854888916016, 33.16192626953125, 0.4979...</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>vae</td>\n",
       "      <td>(39.0084228515625, 38.738487243652344, 0.26992...</td>\n",
       "      <td>14.467145</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>5.966</td>\n",
       "      <td>(38.852317810058594, 38.581783294677734, 0.270...</td>\n",
       "      <td>(36.6823844909668, 36.15475082397461, 0.527631...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>vae</td>\n",
       "      <td>(38.66717529296875, 38.46552658081055, 0.20166...</td>\n",
       "      <td>19.129324</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>5.719</td>\n",
       "      <td>(38.68019104003906, 38.46910095214844, 0.21108...</td>\n",
       "      <td>(36.35286331176758, 35.90190124511719, 0.45096...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>vae</td>\n",
       "      <td>(38.97932434082031, 38.60063552856445, 0.37869...</td>\n",
       "      <td>34.872817</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>5.723</td>\n",
       "      <td>(38.84820556640625, 38.45430374145508, 0.39389...</td>\n",
       "      <td>(36.26937484741211, 35.542999267578125, 0.7263...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>vae</td>\n",
       "      <td>(38.695133209228516, 38.374122619628906, 0.321...</td>\n",
       "      <td>12.630501</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>5.551</td>\n",
       "      <td>(38.64427947998047, 38.3138542175293, 0.330418...</td>\n",
       "      <td>(36.34822463989258, 35.64902114868164, 0.69920...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>vae</td>\n",
       "      <td>(38.74881362915039, 38.32174301147461, 0.42706...</td>\n",
       "      <td>17.442826</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>5.947</td>\n",
       "      <td>(38.688697814941406, 38.24883270263672, 0.4398...</td>\n",
       "      <td>(36.16000747680664, 35.38113784790039, 0.77887...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>vae</td>\n",
       "      <td>(38.889156341552734, 38.52256774902344, 0.3665...</td>\n",
       "      <td>30.673074</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>6.142</td>\n",
       "      <td>(38.818058013916016, 38.44440460205078, 0.3736...</td>\n",
       "      <td>(36.124515533447266, 35.4119758605957, 0.71254...</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>vae</td>\n",
       "      <td>(37.54356002807617, 35.19465637207031, 2.34890...</td>\n",
       "      <td>12.749238</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>13.410</td>\n",
       "      <td>(38.214378356933594, 35.87680435180664, 2.3375...</td>\n",
       "      <td>(35.16950607299805, 32.3515510559082, 2.817945...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>vae</td>\n",
       "      <td>(36.81386184692383, 34.234954833984375, 2.5789...</td>\n",
       "      <td>16.094194</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>14.114</td>\n",
       "      <td>(37.531429290771484, 34.949005126953125, 2.582...</td>\n",
       "      <td>(34.111122131347656, 30.876379013061523, 3.234...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>vae</td>\n",
       "      <td>(36.87081527709961, 34.32442855834961, 2.54637...</td>\n",
       "      <td>27.120536</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>15.232</td>\n",
       "      <td>(37.52959060668945, 34.9891357421875, 2.540454...</td>\n",
       "      <td>(34.49650192260742, 31.58693504333496, 2.90956...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>vae</td>\n",
       "      <td>(36.82023620605469, 34.814605712890625, 2.0056...</td>\n",
       "      <td>9.214412</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>14.121</td>\n",
       "      <td>(37.63935089111328, 35.63624572753906, 2.00311...</td>\n",
       "      <td>(35.00693893432617, 32.729732513427734, 2.2772...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>vae</td>\n",
       "      <td>(36.88414764404297, 34.2717399597168, 2.612413...</td>\n",
       "      <td>14.414025</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>14.466</td>\n",
       "      <td>(37.727359771728516, 35.11228561401367, 2.6150...</td>\n",
       "      <td>(33.92018127441406, 30.58126449584961, 3.33892...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>vae</td>\n",
       "      <td>(36.27071762084961, 34.136505126953125, 2.1342...</td>\n",
       "      <td>22.198293</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>13.910</td>\n",
       "      <td>(37.2382926940918, 35.12124252319336, 2.117049...</td>\n",
       "      <td>(34.16447830200195, 31.696767807006836, 2.4677...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>vae</td>\n",
       "      <td>(38.37299728393555, 35.2120361328125, 3.160964...</td>\n",
       "      <td>14.021271</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>15.860</td>\n",
       "      <td>(39.710731506347656, 36.542633056640625, 3.168...</td>\n",
       "      <td>(34.345455169677734, 30.851818084716797, 3.493...</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>vae</td>\n",
       "      <td>(38.49557876586914, 35.287193298339844, 3.2083...</td>\n",
       "      <td>23.470499</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>15.944</td>\n",
       "      <td>(39.7661018371582, 36.548614501953125, 3.21747...</td>\n",
       "      <td>(33.39211654663086, 29.52691650390625, 3.86519...</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>vae</td>\n",
       "      <td>(38.92380905151367, 35.37278747558594, 3.55101...</td>\n",
       "      <td>32.083009</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>19.259</td>\n",
       "      <td>(40.10024642944336, 36.54537582397461, 3.55486...</td>\n",
       "      <td>(35.20914077758789, 31.552507400512695, 3.6566...</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>vae</td>\n",
       "      <td>(38.071441650390625, 34.98810958862305, 3.0833...</td>\n",
       "      <td>10.996424</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>15.960</td>\n",
       "      <td>(39.333412170410156, 36.23714828491211, 3.0962...</td>\n",
       "      <td>(34.08641815185547, 30.73343849182129, 3.35297...</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>vae</td>\n",
       "      <td>(38.318023681640625, 35.01900863647461, 3.2990...</td>\n",
       "      <td>13.629519</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>18.670</td>\n",
       "      <td>(39.77329635620117, 36.47252655029297, 3.30077...</td>\n",
       "      <td>(34.826351165771484, 31.383567810058594, 3.442...</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>vae</td>\n",
       "      <td>(38.64324951171875, 35.13362121582031, 3.50962...</td>\n",
       "      <td>27.899915</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>19.781</td>\n",
       "      <td>(39.97652053833008, 36.48212814331055, 3.49438...</td>\n",
       "      <td>(34.89718246459961, 31.24835777282715, 3.64882...</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>vae</td>\n",
       "      <td>(38.95459747314453, 35.794891357421875, 3.1597...</td>\n",
       "      <td>11.858646</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>17.206</td>\n",
       "      <td>(40.122779846191406, 36.96379852294922, 3.1589...</td>\n",
       "      <td>(35.683021545410156, 32.29621505737305, 3.3868...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>vae</td>\n",
       "      <td>(38.86046600341797, 35.20796203613281, 3.65250...</td>\n",
       "      <td>17.161840</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>18.663</td>\n",
       "      <td>(40.38964080810547, 36.73396301269531, 3.65569...</td>\n",
       "      <td>(34.77299118041992, 30.72983741760254, 4.04314...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>vae</td>\n",
       "      <td>(38.73073196411133, 35.06247329711914, 3.66826...</td>\n",
       "      <td>35.562833</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>16.547</td>\n",
       "      <td>(39.933509826660156, 36.25691604614258, 3.6766...</td>\n",
       "      <td>(32.96151351928711, 28.339155197143555, 4.6223...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>vae</td>\n",
       "      <td>(38.60601043701172, 35.318572998046875, 3.2874...</td>\n",
       "      <td>10.635996</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>16.549</td>\n",
       "      <td>(40.054542541503906, 36.76142120361328, 3.2931...</td>\n",
       "      <td>(34.491851806640625, 30.83196258544922, 3.6598...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>vae</td>\n",
       "      <td>(38.703224182128906, 35.330467224121094, 3.372...</td>\n",
       "      <td>12.939351</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>17.616</td>\n",
       "      <td>(40.053443908691406, 36.68704605102539, 3.3663...</td>\n",
       "      <td>(34.51634979248047, 30.821033477783203, 3.6953...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>vae</td>\n",
       "      <td>(39.25286865234375, 35.56135559082031, 3.69152...</td>\n",
       "      <td>20.584708</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>19.475</td>\n",
       "      <td>(40.43674850463867, 36.760623931884766, 3.6761...</td>\n",
       "      <td>(35.26069641113281, 31.403690338134766, 3.8570...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>vae</td>\n",
       "      <td>(41.72443389892578, 37.00864791870117, 4.71578...</td>\n",
       "      <td>13.395561</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>21.477</td>\n",
       "      <td>(41.960052490234375, 37.2425422668457, 4.71750...</td>\n",
       "      <td>(36.95732498168945, 32.12738037109375, 4.82994...</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>vae</td>\n",
       "      <td>(42.24528503417969, 37.349334716796875, 4.8959...</td>\n",
       "      <td>21.199946</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>21.077</td>\n",
       "      <td>(42.311222076416016, 37.40298843383789, 4.9082...</td>\n",
       "      <td>(36.085323333740234, 30.96672248840332, 5.1185...</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>vae</td>\n",
       "      <td>(43.671199798583984, 37.71725845336914, 5.9539...</td>\n",
       "      <td>22.959535</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>26.819</td>\n",
       "      <td>(43.86665725708008, 37.92597579956055, 5.94067...</td>\n",
       "      <td>(39.507102966308594, 33.4919319152832, 6.01517...</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>vae</td>\n",
       "      <td>(41.72861862182617, 36.6434440612793, 5.085182...</td>\n",
       "      <td>11.741329</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>20.089</td>\n",
       "      <td>(41.80409622192383, 36.697940826416016, 5.1061...</td>\n",
       "      <td>(35.673545837402344, 30.25191879272461, 5.4216...</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>vae</td>\n",
       "      <td>(41.66337966918945, 36.724632263183594, 4.9387...</td>\n",
       "      <td>20.827550</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>20.758</td>\n",
       "      <td>(41.89988327026367, 36.95887756347656, 4.94100...</td>\n",
       "      <td>(34.971832275390625, 29.67393684387207, 5.2978...</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>vae</td>\n",
       "      <td>(43.05619812011719, 37.54515838623047, 5.51104...</td>\n",
       "      <td>21.003112</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>25.624</td>\n",
       "      <td>(43.238807678222656, 37.71279525756836, 5.5260...</td>\n",
       "      <td>(38.18496322631836, 32.524742126464844, 5.6602...</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>vae</td>\n",
       "      <td>(42.84497833251953, 37.28434371948242, 5.56062...</td>\n",
       "      <td>14.326285</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>21.229</td>\n",
       "      <td>(42.99226760864258, 37.368202209472656, 5.6240...</td>\n",
       "      <td>(36.51313018798828, 30.22275161743164, 6.29038...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>vae</td>\n",
       "      <td>(41.66609573364258, 36.9946403503418, 4.671453...</td>\n",
       "      <td>16.889705</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>20.933</td>\n",
       "      <td>(41.681541442871094, 37.01041030883789, 4.6711...</td>\n",
       "      <td>(35.93172836303711, 30.980684280395508, 4.9510...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>vae</td>\n",
       "      <td>(43.97494888305664, 37.761497497558594, 6.2134...</td>\n",
       "      <td>15.408413</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>30.089</td>\n",
       "      <td>(44.25593948364258, 38.052085876464844, 6.2038...</td>\n",
       "      <td>(40.57714080810547, 34.31729507446289, 6.25985...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>vae</td>\n",
       "      <td>(42.48219680786133, 37.41885757446289, 5.06333...</td>\n",
       "      <td>10.209677</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>20.663</td>\n",
       "      <td>(42.49210739135742, 37.41920852661133, 5.07290...</td>\n",
       "      <td>(36.51702117919922, 30.986858367919922, 5.5301...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>vae</td>\n",
       "      <td>(42.24217224121094, 36.863929748535156, 5.3782...</td>\n",
       "      <td>15.643077</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>21.805</td>\n",
       "      <td>(42.45795822143555, 37.087486267089844, 5.3704...</td>\n",
       "      <td>(35.406272888183594, 29.39113998413086, 6.0151...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>vae</td>\n",
       "      <td>(41.8814811706543, 36.90108108520508, 4.980400...</td>\n",
       "      <td>21.757634</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>22.776</td>\n",
       "      <td>(42.303611755371094, 37.3139762878418, 4.98963...</td>\n",
       "      <td>(36.32438659667969, 31.05082893371582, 5.27355...</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>156 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    model_type                                         test_score  total_time  \\\n",
       "0          vae  (39.45875930786133, 39.16001510620117, 0.29873...   15.426934   \n",
       "1          vae  (39.16112518310547, 39.01034927368164, 0.15078...   19.651253   \n",
       "2          vae  (39.361846923828125, 39.257869720458984, 0.103...   36.790556   \n",
       "3          vae  (39.323123931884766, 39.019893646240234, 0.303...   12.772545   \n",
       "4          vae  (39.155860900878906, 38.941837310791016, 0.214...   17.382276   \n",
       "5          vae  (39.330284118652344, 39.17107391357422, 0.1592...   33.281378   \n",
       "6          vae  (39.997737884521484, 39.63236618041992, 0.3653...   14.326173   \n",
       "7          vae  (39.261268615722656, 39.04073715209961, 0.2205...   17.181729   \n",
       "8          vae  (39.362857818603516, 39.06566619873047, 0.2971...   29.666096   \n",
       "9          vae  (39.86193084716797, 39.19924545288086, 0.66268...   14.487885   \n",
       "10         vae  (39.84731674194336, 38.90536117553711, 0.94195...   17.265143   \n",
       "11         vae  (39.295494079589844, 38.98809051513672, 0.3074...   27.626355   \n",
       "12         vae  (36.9610481262207, 36.645816802978516, 0.31523...   14.884972   \n",
       "13         vae  (36.600669860839844, 36.4581184387207, 0.14254...   20.822176   \n",
       "14         vae  (36.782432556152344, 36.698638916015625, 0.083...   38.685860   \n",
       "15         vae  (36.60530090332031, 36.46498489379883, 0.14031...   12.527444   \n",
       "16         vae  (36.47758102416992, 36.37152862548828, 0.10605...   16.454788   \n",
       "17         vae  (36.74794006347656, 36.62401580810547, 0.12391...   31.721673   \n",
       "18         vae  (37.333290100097656, 36.990840911865234, 0.342...   14.093270   \n",
       "19         vae  (36.786739349365234, 36.455448150634766, 0.331...   19.671381   \n",
       "20         vae  (36.61471176147461, 36.50148391723633, 0.11323...   26.937424   \n",
       "21         vae  (36.92631530761719, 36.619850158691406, 0.3064...   12.170369   \n",
       "22         vae  (36.26698303222656, 36.16910934448242, 0.09786...   14.382359   \n",
       "23         vae  (36.54599380493164, 36.37313461303711, 0.17286...   26.472795   \n",
       "24         vae  (39.0084228515625, 38.738487243652344, 0.26992...   14.467145   \n",
       "25         vae  (38.66717529296875, 38.46552658081055, 0.20166...   19.129324   \n",
       "26         vae  (38.97932434082031, 38.60063552856445, 0.37869...   34.872817   \n",
       "27         vae  (38.695133209228516, 38.374122619628906, 0.321...   12.630501   \n",
       "28         vae  (38.74881362915039, 38.32174301147461, 0.42706...   17.442826   \n",
       "29         vae  (38.889156341552734, 38.52256774902344, 0.3665...   30.673074   \n",
       "..         ...                                                ...         ...   \n",
       "126        vae  (37.54356002807617, 35.19465637207031, 2.34890...   12.749238   \n",
       "127        vae  (36.81386184692383, 34.234954833984375, 2.5789...   16.094194   \n",
       "128        vae  (36.87081527709961, 34.32442855834961, 2.54637...   27.120536   \n",
       "129        vae  (36.82023620605469, 34.814605712890625, 2.0056...    9.214412   \n",
       "130        vae  (36.88414764404297, 34.2717399597168, 2.612413...   14.414025   \n",
       "131        vae  (36.27071762084961, 34.136505126953125, 2.1342...   22.198293   \n",
       "132        vae  (38.37299728393555, 35.2120361328125, 3.160964...   14.021271   \n",
       "133        vae  (38.49557876586914, 35.287193298339844, 3.2083...   23.470499   \n",
       "134        vae  (38.92380905151367, 35.37278747558594, 3.55101...   32.083009   \n",
       "135        vae  (38.071441650390625, 34.98810958862305, 3.0833...   10.996424   \n",
       "136        vae  (38.318023681640625, 35.01900863647461, 3.2990...   13.629519   \n",
       "137        vae  (38.64324951171875, 35.13362121582031, 3.50962...   27.899915   \n",
       "138        vae  (38.95459747314453, 35.794891357421875, 3.1597...   11.858646   \n",
       "139        vae  (38.86046600341797, 35.20796203613281, 3.65250...   17.161840   \n",
       "140        vae  (38.73073196411133, 35.06247329711914, 3.66826...   35.562833   \n",
       "141        vae  (38.60601043701172, 35.318572998046875, 3.2874...   10.635996   \n",
       "142        vae  (38.703224182128906, 35.330467224121094, 3.372...   12.939351   \n",
       "143        vae  (39.25286865234375, 35.56135559082031, 3.69152...   20.584708   \n",
       "144        vae  (41.72443389892578, 37.00864791870117, 4.71578...   13.395561   \n",
       "145        vae  (42.24528503417969, 37.349334716796875, 4.8959...   21.199946   \n",
       "146        vae  (43.671199798583984, 37.71725845336914, 5.9539...   22.959535   \n",
       "147        vae  (41.72861862182617, 36.6434440612793, 5.085182...   11.741329   \n",
       "148        vae  (41.66337966918945, 36.724632263183594, 4.9387...   20.827550   \n",
       "149        vae  (43.05619812011719, 37.54515838623047, 5.51104...   21.003112   \n",
       "150        vae  (42.84497833251953, 37.28434371948242, 5.56062...   14.326285   \n",
       "151        vae  (41.66609573364258, 36.9946403503418, 4.671453...   16.889705   \n",
       "152        vae  (43.97494888305664, 37.761497497558594, 6.2134...   15.408413   \n",
       "153        vae  (42.48219680786133, 37.41885757446289, 5.06333...   10.209677   \n",
       "154        vae  (42.24217224121094, 36.863929748535156, 5.3782...   15.643077   \n",
       "155        vae  (41.8814811706543, 36.90108108520508, 4.980400...   21.757634   \n",
       "\n",
       "                                               dataset  average_mismatches  \\\n",
       "0          synthetic_unimodal_data_length_100_gaussian               5.670   \n",
       "1          synthetic_unimodal_data_length_100_gaussian               5.782   \n",
       "2          synthetic_unimodal_data_length_100_gaussian               5.507   \n",
       "3          synthetic_unimodal_data_length_100_gaussian               5.661   \n",
       "4          synthetic_unimodal_data_length_100_gaussian               5.627   \n",
       "5          synthetic_unimodal_data_length_100_gaussian               5.622   \n",
       "6          synthetic_unimodal_data_length_100_gaussian               6.099   \n",
       "7          synthetic_unimodal_data_length_100_gaussian               5.647   \n",
       "8          synthetic_unimodal_data_length_100_gaussian               5.501   \n",
       "9          synthetic_unimodal_data_length_100_gaussian               5.503   \n",
       "10         synthetic_unimodal_data_length_100_gaussian               5.544   \n",
       "11         synthetic_unimodal_data_length_100_gaussian               5.439   \n",
       "12   synthetic_unimodal_data_length_100_skewed_gaus...               5.122   \n",
       "13   synthetic_unimodal_data_length_100_skewed_gaus...               5.053   \n",
       "14   synthetic_unimodal_data_length_100_skewed_gaus...               5.205   \n",
       "15   synthetic_unimodal_data_length_100_skewed_gaus...               5.314   \n",
       "16   synthetic_unimodal_data_length_100_skewed_gaus...               5.153   \n",
       "17   synthetic_unimodal_data_length_100_skewed_gaus...               5.157   \n",
       "18   synthetic_unimodal_data_length_100_skewed_gaus...               5.424   \n",
       "19   synthetic_unimodal_data_length_100_skewed_gaus...               5.201   \n",
       "20   synthetic_unimodal_data_length_100_skewed_gaus...               5.354   \n",
       "21   synthetic_unimodal_data_length_100_skewed_gaus...               5.110   \n",
       "22   synthetic_unimodal_data_length_100_skewed_gaus...               5.072   \n",
       "23   synthetic_unimodal_data_length_100_skewed_gaus...               4.958   \n",
       "24          synthetic_unimodal_data_length_100_uniform               5.966   \n",
       "25          synthetic_unimodal_data_length_100_uniform               5.719   \n",
       "26          synthetic_unimodal_data_length_100_uniform               5.723   \n",
       "27          synthetic_unimodal_data_length_100_uniform               5.551   \n",
       "28          synthetic_unimodal_data_length_100_uniform               5.947   \n",
       "29          synthetic_unimodal_data_length_100_uniform               6.142   \n",
       "..                                                 ...                 ...   \n",
       "126  synthetic_multimodal_data_modes_2_length_51_un...              13.410   \n",
       "127  synthetic_multimodal_data_modes_2_length_51_un...              14.114   \n",
       "128  synthetic_multimodal_data_modes_2_length_51_un...              15.232   \n",
       "129  synthetic_multimodal_data_modes_2_length_51_un...              14.121   \n",
       "130  synthetic_multimodal_data_modes_2_length_51_un...              14.466   \n",
       "131  synthetic_multimodal_data_modes_2_length_51_un...              13.910   \n",
       "132  synthetic_multimodal_data_modes_3_length_51_un...              15.860   \n",
       "133  synthetic_multimodal_data_modes_3_length_51_un...              15.944   \n",
       "134  synthetic_multimodal_data_modes_3_length_51_un...              19.259   \n",
       "135  synthetic_multimodal_data_modes_3_length_51_un...              15.960   \n",
       "136  synthetic_multimodal_data_modes_3_length_51_un...              18.670   \n",
       "137  synthetic_multimodal_data_modes_3_length_51_un...              19.781   \n",
       "138  synthetic_multimodal_data_modes_3_length_51_un...              17.206   \n",
       "139  synthetic_multimodal_data_modes_3_length_51_un...              18.663   \n",
       "140  synthetic_multimodal_data_modes_3_length_51_un...              16.547   \n",
       "141  synthetic_multimodal_data_modes_3_length_51_un...              16.549   \n",
       "142  synthetic_multimodal_data_modes_3_length_51_un...              17.616   \n",
       "143  synthetic_multimodal_data_modes_3_length_51_un...              19.475   \n",
       "144  synthetic_multimodal_data_modes_5_length_51_un...              21.477   \n",
       "145  synthetic_multimodal_data_modes_5_length_51_un...              21.077   \n",
       "146  synthetic_multimodal_data_modes_5_length_51_un...              26.819   \n",
       "147  synthetic_multimodal_data_modes_5_length_51_un...              20.089   \n",
       "148  synthetic_multimodal_data_modes_5_length_51_un...              20.758   \n",
       "149  synthetic_multimodal_data_modes_5_length_51_un...              25.624   \n",
       "150  synthetic_multimodal_data_modes_5_length_51_un...              21.229   \n",
       "151  synthetic_multimodal_data_modes_5_length_51_un...              20.933   \n",
       "152  synthetic_multimodal_data_modes_5_length_51_un...              30.089   \n",
       "153  synthetic_multimodal_data_modes_5_length_51_un...              20.663   \n",
       "154  synthetic_multimodal_data_modes_5_length_51_un...              21.805   \n",
       "155  synthetic_multimodal_data_modes_5_length_51_un...              22.776   \n",
       "\n",
       "                                           valid_score  \\\n",
       "0    (39.78868865966797, 39.482662200927734, 0.3060...   \n",
       "1    (39.471614837646484, 39.31938171386719, 0.1522...   \n",
       "2    (39.65810012817383, 39.54773712158203, 0.11037...   \n",
       "3    (39.59053421020508, 39.301246643066406, 0.2892...   \n",
       "4    (39.49126052856445, 39.2797966003418, 0.211458...   \n",
       "5    (39.599281311035156, 39.44047164916992, 0.1588...   \n",
       "6    (40.4110221862793, 40.04662322998047, 0.364383...   \n",
       "7    (39.46714782714844, 39.24443435668945, 0.22271...   \n",
       "8    (39.70541763305664, 39.4078254699707, 0.297588...   \n",
       "9    (40.134185791015625, 39.46178436279297, 0.6724...   \n",
       "10   (40.28424072265625, 39.3470458984375, 0.937189...   \n",
       "11   (39.516143798828125, 39.21133804321289, 0.3047...   \n",
       "12   (36.68040466308594, 36.363861083984375, 0.3165...   \n",
       "13   (36.35088348388672, 36.20940399169922, 0.14148...   \n",
       "14   (36.515647888183594, 36.4343147277832, 0.08132...   \n",
       "15   (36.36986541748047, 36.233642578125, 0.1362221...   \n",
       "16   (36.25579071044922, 36.14877700805664, 0.10701...   \n",
       "17   (36.49137878417969, 36.36961364746094, 0.12176...   \n",
       "18   (37.04022979736328, 36.6895751953125, 0.350664...   \n",
       "19   (36.52296829223633, 36.19729995727539, 0.32566...   \n",
       "20   (36.38849639892578, 36.27131271362305, 0.11719...   \n",
       "21   (36.71547317504883, 36.41338348388672, 0.30208...   \n",
       "22   (36.10603332519531, 36.00755310058594, 0.09848...   \n",
       "23   (36.35851287841797, 36.18794250488281, 0.17056...   \n",
       "24   (38.852317810058594, 38.581783294677734, 0.270...   \n",
       "25   (38.68019104003906, 38.46910095214844, 0.21108...   \n",
       "26   (38.84820556640625, 38.45430374145508, 0.39389...   \n",
       "27   (38.64427947998047, 38.3138542175293, 0.330418...   \n",
       "28   (38.688697814941406, 38.24883270263672, 0.4398...   \n",
       "29   (38.818058013916016, 38.44440460205078, 0.3736...   \n",
       "..                                                 ...   \n",
       "126  (38.214378356933594, 35.87680435180664, 2.3375...   \n",
       "127  (37.531429290771484, 34.949005126953125, 2.582...   \n",
       "128  (37.52959060668945, 34.9891357421875, 2.540454...   \n",
       "129  (37.63935089111328, 35.63624572753906, 2.00311...   \n",
       "130  (37.727359771728516, 35.11228561401367, 2.6150...   \n",
       "131  (37.2382926940918, 35.12124252319336, 2.117049...   \n",
       "132  (39.710731506347656, 36.542633056640625, 3.168...   \n",
       "133  (39.7661018371582, 36.548614501953125, 3.21747...   \n",
       "134  (40.10024642944336, 36.54537582397461, 3.55486...   \n",
       "135  (39.333412170410156, 36.23714828491211, 3.0962...   \n",
       "136  (39.77329635620117, 36.47252655029297, 3.30077...   \n",
       "137  (39.97652053833008, 36.48212814331055, 3.49438...   \n",
       "138  (40.122779846191406, 36.96379852294922, 3.1589...   \n",
       "139  (40.38964080810547, 36.73396301269531, 3.65569...   \n",
       "140  (39.933509826660156, 36.25691604614258, 3.6766...   \n",
       "141  (40.054542541503906, 36.76142120361328, 3.2931...   \n",
       "142  (40.053443908691406, 36.68704605102539, 3.3663...   \n",
       "143  (40.43674850463867, 36.760623931884766, 3.6761...   \n",
       "144  (41.960052490234375, 37.2425422668457, 4.71750...   \n",
       "145  (42.311222076416016, 37.40298843383789, 4.9082...   \n",
       "146  (43.86665725708008, 37.92597579956055, 5.94067...   \n",
       "147  (41.80409622192383, 36.697940826416016, 5.1061...   \n",
       "148  (41.89988327026367, 36.95887756347656, 4.94100...   \n",
       "149  (43.238807678222656, 37.71279525756836, 5.5260...   \n",
       "150  (42.99226760864258, 37.368202209472656, 5.6240...   \n",
       "151  (41.681541442871094, 37.01041030883789, 4.6711...   \n",
       "152  (44.25593948364258, 38.052085876464844, 6.2038...   \n",
       "153  (42.49210739135742, 37.41920852661133, 5.07290...   \n",
       "154  (42.45795822143555, 37.087486267089844, 5.3704...   \n",
       "155  (42.303611755371094, 37.3139762878418, 4.98963...   \n",
       "\n",
       "                                           train_score  num_params  exit_code  \\\n",
       "0    (36.52116394042969, 35.80773162841797, 0.71344...    408240.0        0.0   \n",
       "1    (36.4890251159668, 36.09037780761719, 0.398657...    408240.0        0.0   \n",
       "2    (36.50764846801758, 36.250755310058594, 0.2568...    408240.0        0.0   \n",
       "3    (36.4285774230957, 35.69258499145508, 0.735994...    408240.0        0.0   \n",
       "4    (36.368534088134766, 35.817726135253906, 0.550...    408240.0        0.0   \n",
       "5    (36.400875091552734, 35.98995590209961, 0.4109...    408240.0        0.0   \n",
       "6    (37.09550094604492, 36.323673248291016, 0.7718...    814440.0        0.0   \n",
       "7    (36.46193313598633, 35.90756607055664, 0.55436...    814440.0        0.0   \n",
       "8    (36.08356857299805, 35.264488220214844, 0.8190...    814440.0        0.0   \n",
       "9    (35.921974182128906, 34.30979919433594, 1.6121...    814440.0        0.0   \n",
       "10   (34.53658676147461, 32.040687561035156, 2.4958...    814440.0        0.0   \n",
       "11   (35.81599807739258, 34.90946578979492, 0.90652...    814440.0        0.0   \n",
       "12   (33.846797943115234, 33.086181640625, 0.760626...    408240.0        0.0   \n",
       "13   (33.91124725341797, 33.56279373168945, 0.34845...    408240.0        0.0   \n",
       "14   (33.96339797973633, 33.768226623535156, 0.1951...    408240.0        0.0   \n",
       "15   (34.19662094116211, 33.915653228759766, 0.2809...    408240.0        0.0   \n",
       "16   (33.836151123046875, 33.56602096557617, 0.2701...    408240.0        0.0   \n",
       "17   (33.808631896972656, 33.48420715332031, 0.3244...    408240.0        0.0   \n",
       "18   (34.442848205566406, 33.8046989440918, 0.63814...    814440.0        0.0   \n",
       "19   (33.50249481201172, 32.650569915771484, 0.8519...    814440.0        0.0   \n",
       "20   (33.94681167602539, 33.67296600341797, 0.27384...    814440.0        0.0   \n",
       "21   (34.00155258178711, 33.302734375, 0.6988178491...    814440.0        0.0   \n",
       "22   (34.024688720703125, 33.7932014465332, 0.23148...    814440.0        0.0   \n",
       "23   (33.659854888916016, 33.16192626953125, 0.4979...    814440.0        0.0   \n",
       "24   (36.6823844909668, 36.15475082397461, 0.527631...    408240.0        0.0   \n",
       "25   (36.35286331176758, 35.90190124511719, 0.45096...    408240.0        0.0   \n",
       "26   (36.26937484741211, 35.542999267578125, 0.7263...    408240.0        0.0   \n",
       "27   (36.34822463989258, 35.64902114868164, 0.69920...    408240.0        0.0   \n",
       "28   (36.16000747680664, 35.38113784790039, 0.77887...    408240.0        0.0   \n",
       "29   (36.124515533447266, 35.4119758605957, 0.71254...    408240.0        0.0   \n",
       "..                                                 ...         ...        ...   \n",
       "126  (35.16950607299805, 32.3515510559082, 2.817945...    421460.0        0.0   \n",
       "127  (34.111122131347656, 30.876379013061523, 3.234...    421460.0        0.0   \n",
       "128  (34.49650192260742, 31.58693504333496, 2.90956...    421460.0        0.0   \n",
       "129  (35.00693893432617, 32.729732513427734, 2.2772...    421460.0        0.0   \n",
       "130  (33.92018127441406, 30.58126449584961, 3.33892...    421460.0        0.0   \n",
       "131  (34.16447830200195, 31.696767807006836, 2.4677...    421460.0        0.0   \n",
       "132  (34.345455169677734, 30.851818084716797, 3.493...    211260.0        0.0   \n",
       "133  (33.39211654663086, 29.52691650390625, 3.86519...    211260.0        0.0   \n",
       "134  (35.20914077758789, 31.552507400512695, 3.6566...    211260.0        0.0   \n",
       "135  (34.08641815185547, 30.73343849182129, 3.35297...    211260.0        0.0   \n",
       "136  (34.826351165771484, 31.383567810058594, 3.442...    211260.0        0.0   \n",
       "137  (34.89718246459961, 31.24835777282715, 3.64882...    211260.0        0.0   \n",
       "138  (35.683021545410156, 32.29621505737305, 3.3868...    421460.0        0.0   \n",
       "139  (34.77299118041992, 30.72983741760254, 4.04314...    421460.0        0.0   \n",
       "140  (32.96151351928711, 28.339155197143555, 4.6223...    421460.0        0.0   \n",
       "141  (34.491851806640625, 30.83196258544922, 3.6598...    421460.0        0.0   \n",
       "142  (34.51634979248047, 30.821033477783203, 3.6953...    421460.0        0.0   \n",
       "143  (35.26069641113281, 31.403690338134766, 3.8570...    421460.0        0.0   \n",
       "144  (36.95732498168945, 32.12738037109375, 4.82994...    211260.0        0.0   \n",
       "145  (36.085323333740234, 30.96672248840332, 5.1185...    211260.0        0.0   \n",
       "146  (39.507102966308594, 33.4919319152832, 6.01517...    211260.0        0.0   \n",
       "147  (35.673545837402344, 30.25191879272461, 5.4216...    211260.0        0.0   \n",
       "148  (34.971832275390625, 29.67393684387207, 5.2978...    211260.0        0.0   \n",
       "149  (38.18496322631836, 32.524742126464844, 5.6602...    211260.0        0.0   \n",
       "150  (36.51313018798828, 30.22275161743164, 6.29038...    421460.0        0.0   \n",
       "151  (35.93172836303711, 30.980684280395508, 4.9510...    421460.0        0.0   \n",
       "152  (40.57714080810547, 34.31729507446289, 6.25985...    421460.0        0.0   \n",
       "153  (36.51702117919922, 30.986858367919922, 5.5301...    421460.0        0.0   \n",
       "154  (35.406272888183594, 29.39113998413086, 6.0151...    421460.0        0.0   \n",
       "155  (36.32438659667969, 31.05082893371582, 5.27355...    421460.0        0.0   \n",
       "\n",
       "                                            name  \n",
       "0     vae_hidden_size_100_batch_size_10_lr_0.003  \n",
       "1     vae_hidden_size_100_batch_size_10_lr_0.001  \n",
       "2    vae_hidden_size_100_batch_size_10_lr_0.0003  \n",
       "3     vae_hidden_size_100_batch_size_20_lr_0.003  \n",
       "4     vae_hidden_size_100_batch_size_20_lr_0.001  \n",
       "5    vae_hidden_size_100_batch_size_20_lr_0.0003  \n",
       "6     vae_hidden_size_200_batch_size_10_lr_0.003  \n",
       "7     vae_hidden_size_200_batch_size_10_lr_0.001  \n",
       "8    vae_hidden_size_200_batch_size_10_lr_0.0003  \n",
       "9     vae_hidden_size_200_batch_size_20_lr_0.003  \n",
       "10    vae_hidden_size_200_batch_size_20_lr_0.001  \n",
       "11   vae_hidden_size_200_batch_size_20_lr_0.0003  \n",
       "12    vae_hidden_size_100_batch_size_10_lr_0.003  \n",
       "13    vae_hidden_size_100_batch_size_10_lr_0.001  \n",
       "14   vae_hidden_size_100_batch_size_10_lr_0.0003  \n",
       "15    vae_hidden_size_100_batch_size_20_lr_0.003  \n",
       "16    vae_hidden_size_100_batch_size_20_lr_0.001  \n",
       "17   vae_hidden_size_100_batch_size_20_lr_0.0003  \n",
       "18    vae_hidden_size_200_batch_size_10_lr_0.003  \n",
       "19    vae_hidden_size_200_batch_size_10_lr_0.001  \n",
       "20   vae_hidden_size_200_batch_size_10_lr_0.0003  \n",
       "21    vae_hidden_size_200_batch_size_20_lr_0.003  \n",
       "22    vae_hidden_size_200_batch_size_20_lr_0.001  \n",
       "23   vae_hidden_size_200_batch_size_20_lr_0.0003  \n",
       "24    vae_hidden_size_100_batch_size_10_lr_0.003  \n",
       "25    vae_hidden_size_100_batch_size_10_lr_0.001  \n",
       "26   vae_hidden_size_100_batch_size_10_lr_0.0003  \n",
       "27    vae_hidden_size_100_batch_size_20_lr_0.003  \n",
       "28    vae_hidden_size_100_batch_size_20_lr_0.001  \n",
       "29   vae_hidden_size_100_batch_size_20_lr_0.0003  \n",
       "..                                           ...  \n",
       "126   vae_hidden_size_200_batch_size_10_lr_0.003  \n",
       "127   vae_hidden_size_200_batch_size_10_lr_0.001  \n",
       "128  vae_hidden_size_200_batch_size_10_lr_0.0003  \n",
       "129   vae_hidden_size_200_batch_size_20_lr_0.003  \n",
       "130   vae_hidden_size_200_batch_size_20_lr_0.001  \n",
       "131  vae_hidden_size_200_batch_size_20_lr_0.0003  \n",
       "132   vae_hidden_size_100_batch_size_10_lr_0.003  \n",
       "133   vae_hidden_size_100_batch_size_10_lr_0.001  \n",
       "134  vae_hidden_size_100_batch_size_10_lr_0.0003  \n",
       "135   vae_hidden_size_100_batch_size_20_lr_0.003  \n",
       "136   vae_hidden_size_100_batch_size_20_lr_0.001  \n",
       "137  vae_hidden_size_100_batch_size_20_lr_0.0003  \n",
       "138   vae_hidden_size_200_batch_size_10_lr_0.003  \n",
       "139   vae_hidden_size_200_batch_size_10_lr_0.001  \n",
       "140  vae_hidden_size_200_batch_size_10_lr_0.0003  \n",
       "141   vae_hidden_size_200_batch_size_20_lr_0.003  \n",
       "142   vae_hidden_size_200_batch_size_20_lr_0.001  \n",
       "143  vae_hidden_size_200_batch_size_20_lr_0.0003  \n",
       "144   vae_hidden_size_100_batch_size_10_lr_0.003  \n",
       "145   vae_hidden_size_100_batch_size_10_lr_0.001  \n",
       "146  vae_hidden_size_100_batch_size_10_lr_0.0003  \n",
       "147   vae_hidden_size_100_batch_size_20_lr_0.003  \n",
       "148   vae_hidden_size_100_batch_size_20_lr_0.001  \n",
       "149  vae_hidden_size_100_batch_size_20_lr_0.0003  \n",
       "150   vae_hidden_size_200_batch_size_10_lr_0.003  \n",
       "151   vae_hidden_size_200_batch_size_10_lr_0.001  \n",
       "152  vae_hidden_size_200_batch_size_10_lr_0.0003  \n",
       "153   vae_hidden_size_200_batch_size_20_lr_0.003  \n",
       "154   vae_hidden_size_200_batch_size_20_lr_0.001  \n",
       "155  vae_hidden_size_200_batch_size_20_lr_0.0003  \n",
       "\n",
       "[156 rows x 10 columns]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vae_df = vae_df.iloc[0:156, :]\n",
    "print(len(vae_df))\n",
    "vae_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_type</th>\n",
       "      <th>test_score</th>\n",
       "      <th>total_time</th>\n",
       "      <th>dataset</th>\n",
       "      <th>average_mismatches</th>\n",
       "      <th>valid_score</th>\n",
       "      <th>train_score</th>\n",
       "      <th>num_params</th>\n",
       "      <th>exit_code</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>vae</td>\n",
       "      <td>39.458759</td>\n",
       "      <td>15.426934</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.670</td>\n",
       "      <td>39.788689</td>\n",
       "      <td>36.521164</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>vae</td>\n",
       "      <td>39.161125</td>\n",
       "      <td>19.651253</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.782</td>\n",
       "      <td>39.471615</td>\n",
       "      <td>36.489025</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>vae</td>\n",
       "      <td>39.361847</td>\n",
       "      <td>36.790556</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.507</td>\n",
       "      <td>39.658100</td>\n",
       "      <td>36.507648</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>vae</td>\n",
       "      <td>39.323124</td>\n",
       "      <td>12.772545</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.661</td>\n",
       "      <td>39.590534</td>\n",
       "      <td>36.428577</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>vae</td>\n",
       "      <td>39.155861</td>\n",
       "      <td>17.382276</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.627</td>\n",
       "      <td>39.491261</td>\n",
       "      <td>36.368534</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>vae</td>\n",
       "      <td>39.330284</td>\n",
       "      <td>33.281378</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.622</td>\n",
       "      <td>39.599281</td>\n",
       "      <td>36.400875</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>vae</td>\n",
       "      <td>39.997738</td>\n",
       "      <td>14.326173</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>6.099</td>\n",
       "      <td>40.411022</td>\n",
       "      <td>37.095501</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>vae</td>\n",
       "      <td>39.261269</td>\n",
       "      <td>17.181729</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.647</td>\n",
       "      <td>39.467148</td>\n",
       "      <td>36.461933</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>vae</td>\n",
       "      <td>39.362858</td>\n",
       "      <td>29.666096</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.501</td>\n",
       "      <td>39.705418</td>\n",
       "      <td>36.083569</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>vae</td>\n",
       "      <td>39.861931</td>\n",
       "      <td>14.487885</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.503</td>\n",
       "      <td>40.134186</td>\n",
       "      <td>35.921974</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>vae</td>\n",
       "      <td>39.847317</td>\n",
       "      <td>17.265143</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.544</td>\n",
       "      <td>40.284241</td>\n",
       "      <td>34.536587</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>vae</td>\n",
       "      <td>39.295494</td>\n",
       "      <td>27.626355</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>5.439</td>\n",
       "      <td>39.516144</td>\n",
       "      <td>35.815998</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.961048</td>\n",
       "      <td>14.884972</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.122</td>\n",
       "      <td>36.680405</td>\n",
       "      <td>33.846798</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.600670</td>\n",
       "      <td>20.822176</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.053</td>\n",
       "      <td>36.350883</td>\n",
       "      <td>33.911247</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.782433</td>\n",
       "      <td>38.685860</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.205</td>\n",
       "      <td>36.515648</td>\n",
       "      <td>33.963398</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.605301</td>\n",
       "      <td>12.527444</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.314</td>\n",
       "      <td>36.369865</td>\n",
       "      <td>34.196621</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.477581</td>\n",
       "      <td>16.454788</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.153</td>\n",
       "      <td>36.255791</td>\n",
       "      <td>33.836151</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.747940</td>\n",
       "      <td>31.721673</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.157</td>\n",
       "      <td>36.491379</td>\n",
       "      <td>33.808632</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>vae</td>\n",
       "      <td>37.333290</td>\n",
       "      <td>14.093270</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.424</td>\n",
       "      <td>37.040230</td>\n",
       "      <td>34.442848</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.786739</td>\n",
       "      <td>19.671381</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.201</td>\n",
       "      <td>36.522968</td>\n",
       "      <td>33.502495</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.614712</td>\n",
       "      <td>26.937424</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.354</td>\n",
       "      <td>36.388496</td>\n",
       "      <td>33.946812</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.926315</td>\n",
       "      <td>12.170369</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.110</td>\n",
       "      <td>36.715473</td>\n",
       "      <td>34.001553</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.266983</td>\n",
       "      <td>14.382359</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>5.072</td>\n",
       "      <td>36.106033</td>\n",
       "      <td>34.024689</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.545994</td>\n",
       "      <td>26.472795</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>4.958</td>\n",
       "      <td>36.358513</td>\n",
       "      <td>33.659855</td>\n",
       "      <td>814440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>vae</td>\n",
       "      <td>39.008423</td>\n",
       "      <td>14.467145</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>5.966</td>\n",
       "      <td>38.852318</td>\n",
       "      <td>36.682384</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>vae</td>\n",
       "      <td>38.667175</td>\n",
       "      <td>19.129324</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>5.719</td>\n",
       "      <td>38.680191</td>\n",
       "      <td>36.352863</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>vae</td>\n",
       "      <td>38.979324</td>\n",
       "      <td>34.872817</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>5.723</td>\n",
       "      <td>38.848206</td>\n",
       "      <td>36.269375</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>vae</td>\n",
       "      <td>38.695133</td>\n",
       "      <td>12.630501</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>5.551</td>\n",
       "      <td>38.644279</td>\n",
       "      <td>36.348225</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>vae</td>\n",
       "      <td>38.748814</td>\n",
       "      <td>17.442826</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>5.947</td>\n",
       "      <td>38.688698</td>\n",
       "      <td>36.160007</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>vae</td>\n",
       "      <td>38.889156</td>\n",
       "      <td>30.673074</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>6.142</td>\n",
       "      <td>38.818058</td>\n",
       "      <td>36.124516</td>\n",
       "      <td>408240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>vae</td>\n",
       "      <td>37.543560</td>\n",
       "      <td>12.749238</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>13.410</td>\n",
       "      <td>38.214378</td>\n",
       "      <td>35.169506</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.813862</td>\n",
       "      <td>16.094194</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>14.114</td>\n",
       "      <td>37.531429</td>\n",
       "      <td>34.111122</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.870815</td>\n",
       "      <td>27.120536</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>15.232</td>\n",
       "      <td>37.529591</td>\n",
       "      <td>34.496502</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.820236</td>\n",
       "      <td>9.214412</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>14.121</td>\n",
       "      <td>37.639351</td>\n",
       "      <td>35.006939</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.884148</td>\n",
       "      <td>14.414025</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>14.466</td>\n",
       "      <td>37.727360</td>\n",
       "      <td>33.920181</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.270718</td>\n",
       "      <td>22.198293</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>13.910</td>\n",
       "      <td>37.238293</td>\n",
       "      <td>34.164478</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>vae</td>\n",
       "      <td>38.372997</td>\n",
       "      <td>14.021271</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>15.860</td>\n",
       "      <td>39.710732</td>\n",
       "      <td>34.345455</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>vae</td>\n",
       "      <td>38.495579</td>\n",
       "      <td>23.470499</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>15.944</td>\n",
       "      <td>39.766102</td>\n",
       "      <td>33.392117</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>vae</td>\n",
       "      <td>38.923809</td>\n",
       "      <td>32.083009</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>19.259</td>\n",
       "      <td>40.100246</td>\n",
       "      <td>35.209141</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>vae</td>\n",
       "      <td>38.071442</td>\n",
       "      <td>10.996424</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>15.960</td>\n",
       "      <td>39.333412</td>\n",
       "      <td>34.086418</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>vae</td>\n",
       "      <td>38.318024</td>\n",
       "      <td>13.629519</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>18.670</td>\n",
       "      <td>39.773296</td>\n",
       "      <td>34.826351</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>vae</td>\n",
       "      <td>38.643250</td>\n",
       "      <td>27.899915</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>19.781</td>\n",
       "      <td>39.976521</td>\n",
       "      <td>34.897182</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>vae</td>\n",
       "      <td>38.954597</td>\n",
       "      <td>11.858646</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>17.206</td>\n",
       "      <td>40.122780</td>\n",
       "      <td>35.683022</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>vae</td>\n",
       "      <td>38.860466</td>\n",
       "      <td>17.161840</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>18.663</td>\n",
       "      <td>40.389641</td>\n",
       "      <td>34.772991</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>vae</td>\n",
       "      <td>38.730732</td>\n",
       "      <td>35.562833</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>16.547</td>\n",
       "      <td>39.933510</td>\n",
       "      <td>32.961514</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>vae</td>\n",
       "      <td>38.606010</td>\n",
       "      <td>10.635996</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>16.549</td>\n",
       "      <td>40.054543</td>\n",
       "      <td>34.491852</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>vae</td>\n",
       "      <td>38.703224</td>\n",
       "      <td>12.939351</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>17.616</td>\n",
       "      <td>40.053444</td>\n",
       "      <td>34.516350</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>vae</td>\n",
       "      <td>39.252869</td>\n",
       "      <td>20.584708</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>19.475</td>\n",
       "      <td>40.436749</td>\n",
       "      <td>35.260696</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>vae</td>\n",
       "      <td>41.724434</td>\n",
       "      <td>13.395561</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>21.477</td>\n",
       "      <td>41.960052</td>\n",
       "      <td>36.957325</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>vae</td>\n",
       "      <td>42.245285</td>\n",
       "      <td>21.199946</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>21.077</td>\n",
       "      <td>42.311222</td>\n",
       "      <td>36.085323</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>vae</td>\n",
       "      <td>43.671200</td>\n",
       "      <td>22.959535</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>26.819</td>\n",
       "      <td>43.866657</td>\n",
       "      <td>39.507103</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>vae</td>\n",
       "      <td>41.728619</td>\n",
       "      <td>11.741329</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>20.089</td>\n",
       "      <td>41.804096</td>\n",
       "      <td>35.673546</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>vae</td>\n",
       "      <td>41.663380</td>\n",
       "      <td>20.827550</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>20.758</td>\n",
       "      <td>41.899883</td>\n",
       "      <td>34.971832</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>vae</td>\n",
       "      <td>43.056198</td>\n",
       "      <td>21.003112</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>25.624</td>\n",
       "      <td>43.238808</td>\n",
       "      <td>38.184963</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>vae</td>\n",
       "      <td>42.844978</td>\n",
       "      <td>14.326285</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>21.229</td>\n",
       "      <td>42.992268</td>\n",
       "      <td>36.513130</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>vae</td>\n",
       "      <td>41.666096</td>\n",
       "      <td>16.889705</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>20.933</td>\n",
       "      <td>41.681541</td>\n",
       "      <td>35.931728</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>vae</td>\n",
       "      <td>43.974949</td>\n",
       "      <td>15.408413</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>30.089</td>\n",
       "      <td>44.255939</td>\n",
       "      <td>40.577141</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>vae</td>\n",
       "      <td>42.482197</td>\n",
       "      <td>10.209677</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>20.663</td>\n",
       "      <td>42.492107</td>\n",
       "      <td>36.517021</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>vae</td>\n",
       "      <td>42.242172</td>\n",
       "      <td>15.643077</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>21.805</td>\n",
       "      <td>42.457958</td>\n",
       "      <td>35.406273</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>vae</td>\n",
       "      <td>41.881481</td>\n",
       "      <td>21.757634</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>22.776</td>\n",
       "      <td>42.303612</td>\n",
       "      <td>36.324387</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>156 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    model_type  test_score  total_time  \\\n",
       "0          vae   39.458759   15.426934   \n",
       "1          vae   39.161125   19.651253   \n",
       "2          vae   39.361847   36.790556   \n",
       "3          vae   39.323124   12.772545   \n",
       "4          vae   39.155861   17.382276   \n",
       "5          vae   39.330284   33.281378   \n",
       "6          vae   39.997738   14.326173   \n",
       "7          vae   39.261269   17.181729   \n",
       "8          vae   39.362858   29.666096   \n",
       "9          vae   39.861931   14.487885   \n",
       "10         vae   39.847317   17.265143   \n",
       "11         vae   39.295494   27.626355   \n",
       "12         vae   36.961048   14.884972   \n",
       "13         vae   36.600670   20.822176   \n",
       "14         vae   36.782433   38.685860   \n",
       "15         vae   36.605301   12.527444   \n",
       "16         vae   36.477581   16.454788   \n",
       "17         vae   36.747940   31.721673   \n",
       "18         vae   37.333290   14.093270   \n",
       "19         vae   36.786739   19.671381   \n",
       "20         vae   36.614712   26.937424   \n",
       "21         vae   36.926315   12.170369   \n",
       "22         vae   36.266983   14.382359   \n",
       "23         vae   36.545994   26.472795   \n",
       "24         vae   39.008423   14.467145   \n",
       "25         vae   38.667175   19.129324   \n",
       "26         vae   38.979324   34.872817   \n",
       "27         vae   38.695133   12.630501   \n",
       "28         vae   38.748814   17.442826   \n",
       "29         vae   38.889156   30.673074   \n",
       "..         ...         ...         ...   \n",
       "126        vae   37.543560   12.749238   \n",
       "127        vae   36.813862   16.094194   \n",
       "128        vae   36.870815   27.120536   \n",
       "129        vae   36.820236    9.214412   \n",
       "130        vae   36.884148   14.414025   \n",
       "131        vae   36.270718   22.198293   \n",
       "132        vae   38.372997   14.021271   \n",
       "133        vae   38.495579   23.470499   \n",
       "134        vae   38.923809   32.083009   \n",
       "135        vae   38.071442   10.996424   \n",
       "136        vae   38.318024   13.629519   \n",
       "137        vae   38.643250   27.899915   \n",
       "138        vae   38.954597   11.858646   \n",
       "139        vae   38.860466   17.161840   \n",
       "140        vae   38.730732   35.562833   \n",
       "141        vae   38.606010   10.635996   \n",
       "142        vae   38.703224   12.939351   \n",
       "143        vae   39.252869   20.584708   \n",
       "144        vae   41.724434   13.395561   \n",
       "145        vae   42.245285   21.199946   \n",
       "146        vae   43.671200   22.959535   \n",
       "147        vae   41.728619   11.741329   \n",
       "148        vae   41.663380   20.827550   \n",
       "149        vae   43.056198   21.003112   \n",
       "150        vae   42.844978   14.326285   \n",
       "151        vae   41.666096   16.889705   \n",
       "152        vae   43.974949   15.408413   \n",
       "153        vae   42.482197   10.209677   \n",
       "154        vae   42.242172   15.643077   \n",
       "155        vae   41.881481   21.757634   \n",
       "\n",
       "                                               dataset  average_mismatches  \\\n",
       "0          synthetic_unimodal_data_length_100_gaussian               5.670   \n",
       "1          synthetic_unimodal_data_length_100_gaussian               5.782   \n",
       "2          synthetic_unimodal_data_length_100_gaussian               5.507   \n",
       "3          synthetic_unimodal_data_length_100_gaussian               5.661   \n",
       "4          synthetic_unimodal_data_length_100_gaussian               5.627   \n",
       "5          synthetic_unimodal_data_length_100_gaussian               5.622   \n",
       "6          synthetic_unimodal_data_length_100_gaussian               6.099   \n",
       "7          synthetic_unimodal_data_length_100_gaussian               5.647   \n",
       "8          synthetic_unimodal_data_length_100_gaussian               5.501   \n",
       "9          synthetic_unimodal_data_length_100_gaussian               5.503   \n",
       "10         synthetic_unimodal_data_length_100_gaussian               5.544   \n",
       "11         synthetic_unimodal_data_length_100_gaussian               5.439   \n",
       "12   synthetic_unimodal_data_length_100_skewed_gaus...               5.122   \n",
       "13   synthetic_unimodal_data_length_100_skewed_gaus...               5.053   \n",
       "14   synthetic_unimodal_data_length_100_skewed_gaus...               5.205   \n",
       "15   synthetic_unimodal_data_length_100_skewed_gaus...               5.314   \n",
       "16   synthetic_unimodal_data_length_100_skewed_gaus...               5.153   \n",
       "17   synthetic_unimodal_data_length_100_skewed_gaus...               5.157   \n",
       "18   synthetic_unimodal_data_length_100_skewed_gaus...               5.424   \n",
       "19   synthetic_unimodal_data_length_100_skewed_gaus...               5.201   \n",
       "20   synthetic_unimodal_data_length_100_skewed_gaus...               5.354   \n",
       "21   synthetic_unimodal_data_length_100_skewed_gaus...               5.110   \n",
       "22   synthetic_unimodal_data_length_100_skewed_gaus...               5.072   \n",
       "23   synthetic_unimodal_data_length_100_skewed_gaus...               4.958   \n",
       "24          synthetic_unimodal_data_length_100_uniform               5.966   \n",
       "25          synthetic_unimodal_data_length_100_uniform               5.719   \n",
       "26          synthetic_unimodal_data_length_100_uniform               5.723   \n",
       "27          synthetic_unimodal_data_length_100_uniform               5.551   \n",
       "28          synthetic_unimodal_data_length_100_uniform               5.947   \n",
       "29          synthetic_unimodal_data_length_100_uniform               6.142   \n",
       "..                                                 ...                 ...   \n",
       "126  synthetic_multimodal_data_modes_2_length_51_un...              13.410   \n",
       "127  synthetic_multimodal_data_modes_2_length_51_un...              14.114   \n",
       "128  synthetic_multimodal_data_modes_2_length_51_un...              15.232   \n",
       "129  synthetic_multimodal_data_modes_2_length_51_un...              14.121   \n",
       "130  synthetic_multimodal_data_modes_2_length_51_un...              14.466   \n",
       "131  synthetic_multimodal_data_modes_2_length_51_un...              13.910   \n",
       "132  synthetic_multimodal_data_modes_3_length_51_un...              15.860   \n",
       "133  synthetic_multimodal_data_modes_3_length_51_un...              15.944   \n",
       "134  synthetic_multimodal_data_modes_3_length_51_un...              19.259   \n",
       "135  synthetic_multimodal_data_modes_3_length_51_un...              15.960   \n",
       "136  synthetic_multimodal_data_modes_3_length_51_un...              18.670   \n",
       "137  synthetic_multimodal_data_modes_3_length_51_un...              19.781   \n",
       "138  synthetic_multimodal_data_modes_3_length_51_un...              17.206   \n",
       "139  synthetic_multimodal_data_modes_3_length_51_un...              18.663   \n",
       "140  synthetic_multimodal_data_modes_3_length_51_un...              16.547   \n",
       "141  synthetic_multimodal_data_modes_3_length_51_un...              16.549   \n",
       "142  synthetic_multimodal_data_modes_3_length_51_un...              17.616   \n",
       "143  synthetic_multimodal_data_modes_3_length_51_un...              19.475   \n",
       "144  synthetic_multimodal_data_modes_5_length_51_un...              21.477   \n",
       "145  synthetic_multimodal_data_modes_5_length_51_un...              21.077   \n",
       "146  synthetic_multimodal_data_modes_5_length_51_un...              26.819   \n",
       "147  synthetic_multimodal_data_modes_5_length_51_un...              20.089   \n",
       "148  synthetic_multimodal_data_modes_5_length_51_un...              20.758   \n",
       "149  synthetic_multimodal_data_modes_5_length_51_un...              25.624   \n",
       "150  synthetic_multimodal_data_modes_5_length_51_un...              21.229   \n",
       "151  synthetic_multimodal_data_modes_5_length_51_un...              20.933   \n",
       "152  synthetic_multimodal_data_modes_5_length_51_un...              30.089   \n",
       "153  synthetic_multimodal_data_modes_5_length_51_un...              20.663   \n",
       "154  synthetic_multimodal_data_modes_5_length_51_un...              21.805   \n",
       "155  synthetic_multimodal_data_modes_5_length_51_un...              22.776   \n",
       "\n",
       "     valid_score  train_score  num_params  exit_code  \\\n",
       "0      39.788689    36.521164    408240.0        0.0   \n",
       "1      39.471615    36.489025    408240.0        0.0   \n",
       "2      39.658100    36.507648    408240.0        0.0   \n",
       "3      39.590534    36.428577    408240.0        0.0   \n",
       "4      39.491261    36.368534    408240.0        0.0   \n",
       "5      39.599281    36.400875    408240.0        0.0   \n",
       "6      40.411022    37.095501    814440.0        0.0   \n",
       "7      39.467148    36.461933    814440.0        0.0   \n",
       "8      39.705418    36.083569    814440.0        0.0   \n",
       "9      40.134186    35.921974    814440.0        0.0   \n",
       "10     40.284241    34.536587    814440.0        0.0   \n",
       "11     39.516144    35.815998    814440.0        0.0   \n",
       "12     36.680405    33.846798    408240.0        0.0   \n",
       "13     36.350883    33.911247    408240.0        0.0   \n",
       "14     36.515648    33.963398    408240.0        0.0   \n",
       "15     36.369865    34.196621    408240.0        0.0   \n",
       "16     36.255791    33.836151    408240.0        0.0   \n",
       "17     36.491379    33.808632    408240.0        0.0   \n",
       "18     37.040230    34.442848    814440.0        0.0   \n",
       "19     36.522968    33.502495    814440.0        0.0   \n",
       "20     36.388496    33.946812    814440.0        0.0   \n",
       "21     36.715473    34.001553    814440.0        0.0   \n",
       "22     36.106033    34.024689    814440.0        0.0   \n",
       "23     36.358513    33.659855    814440.0        0.0   \n",
       "24     38.852318    36.682384    408240.0        0.0   \n",
       "25     38.680191    36.352863    408240.0        0.0   \n",
       "26     38.848206    36.269375    408240.0        0.0   \n",
       "27     38.644279    36.348225    408240.0        0.0   \n",
       "28     38.688698    36.160007    408240.0        0.0   \n",
       "29     38.818058    36.124516    408240.0        0.0   \n",
       "..           ...          ...         ...        ...   \n",
       "126    38.214378    35.169506    421460.0        0.0   \n",
       "127    37.531429    34.111122    421460.0        0.0   \n",
       "128    37.529591    34.496502    421460.0        0.0   \n",
       "129    37.639351    35.006939    421460.0        0.0   \n",
       "130    37.727360    33.920181    421460.0        0.0   \n",
       "131    37.238293    34.164478    421460.0        0.0   \n",
       "132    39.710732    34.345455    211260.0        0.0   \n",
       "133    39.766102    33.392117    211260.0        0.0   \n",
       "134    40.100246    35.209141    211260.0        0.0   \n",
       "135    39.333412    34.086418    211260.0        0.0   \n",
       "136    39.773296    34.826351    211260.0        0.0   \n",
       "137    39.976521    34.897182    211260.0        0.0   \n",
       "138    40.122780    35.683022    421460.0        0.0   \n",
       "139    40.389641    34.772991    421460.0        0.0   \n",
       "140    39.933510    32.961514    421460.0        0.0   \n",
       "141    40.054543    34.491852    421460.0        0.0   \n",
       "142    40.053444    34.516350    421460.0        0.0   \n",
       "143    40.436749    35.260696    421460.0        0.0   \n",
       "144    41.960052    36.957325    211260.0        0.0   \n",
       "145    42.311222    36.085323    211260.0        0.0   \n",
       "146    43.866657    39.507103    211260.0        0.0   \n",
       "147    41.804096    35.673546    211260.0        0.0   \n",
       "148    41.899883    34.971832    211260.0        0.0   \n",
       "149    43.238808    38.184963    211260.0        0.0   \n",
       "150    42.992268    36.513130    421460.0        0.0   \n",
       "151    41.681541    35.931728    421460.0        0.0   \n",
       "152    44.255939    40.577141    421460.0        0.0   \n",
       "153    42.492107    36.517021    421460.0        0.0   \n",
       "154    42.457958    35.406273    421460.0        0.0   \n",
       "155    42.303612    36.324387    421460.0        0.0   \n",
       "\n",
       "                                            name  \n",
       "0     vae_hidden_size_100_batch_size_10_lr_0.003  \n",
       "1     vae_hidden_size_100_batch_size_10_lr_0.001  \n",
       "2    vae_hidden_size_100_batch_size_10_lr_0.0003  \n",
       "3     vae_hidden_size_100_batch_size_20_lr_0.003  \n",
       "4     vae_hidden_size_100_batch_size_20_lr_0.001  \n",
       "5    vae_hidden_size_100_batch_size_20_lr_0.0003  \n",
       "6     vae_hidden_size_200_batch_size_10_lr_0.003  \n",
       "7     vae_hidden_size_200_batch_size_10_lr_0.001  \n",
       "8    vae_hidden_size_200_batch_size_10_lr_0.0003  \n",
       "9     vae_hidden_size_200_batch_size_20_lr_0.003  \n",
       "10    vae_hidden_size_200_batch_size_20_lr_0.001  \n",
       "11   vae_hidden_size_200_batch_size_20_lr_0.0003  \n",
       "12    vae_hidden_size_100_batch_size_10_lr_0.003  \n",
       "13    vae_hidden_size_100_batch_size_10_lr_0.001  \n",
       "14   vae_hidden_size_100_batch_size_10_lr_0.0003  \n",
       "15    vae_hidden_size_100_batch_size_20_lr_0.003  \n",
       "16    vae_hidden_size_100_batch_size_20_lr_0.001  \n",
       "17   vae_hidden_size_100_batch_size_20_lr_0.0003  \n",
       "18    vae_hidden_size_200_batch_size_10_lr_0.003  \n",
       "19    vae_hidden_size_200_batch_size_10_lr_0.001  \n",
       "20   vae_hidden_size_200_batch_size_10_lr_0.0003  \n",
       "21    vae_hidden_size_200_batch_size_20_lr_0.003  \n",
       "22    vae_hidden_size_200_batch_size_20_lr_0.001  \n",
       "23   vae_hidden_size_200_batch_size_20_lr_0.0003  \n",
       "24    vae_hidden_size_100_batch_size_10_lr_0.003  \n",
       "25    vae_hidden_size_100_batch_size_10_lr_0.001  \n",
       "26   vae_hidden_size_100_batch_size_10_lr_0.0003  \n",
       "27    vae_hidden_size_100_batch_size_20_lr_0.003  \n",
       "28    vae_hidden_size_100_batch_size_20_lr_0.001  \n",
       "29   vae_hidden_size_100_batch_size_20_lr_0.0003  \n",
       "..                                           ...  \n",
       "126   vae_hidden_size_200_batch_size_10_lr_0.003  \n",
       "127   vae_hidden_size_200_batch_size_10_lr_0.001  \n",
       "128  vae_hidden_size_200_batch_size_10_lr_0.0003  \n",
       "129   vae_hidden_size_200_batch_size_20_lr_0.003  \n",
       "130   vae_hidden_size_200_batch_size_20_lr_0.001  \n",
       "131  vae_hidden_size_200_batch_size_20_lr_0.0003  \n",
       "132   vae_hidden_size_100_batch_size_10_lr_0.003  \n",
       "133   vae_hidden_size_100_batch_size_10_lr_0.001  \n",
       "134  vae_hidden_size_100_batch_size_10_lr_0.0003  \n",
       "135   vae_hidden_size_100_batch_size_20_lr_0.003  \n",
       "136   vae_hidden_size_100_batch_size_20_lr_0.001  \n",
       "137  vae_hidden_size_100_batch_size_20_lr_0.0003  \n",
       "138   vae_hidden_size_200_batch_size_10_lr_0.003  \n",
       "139   vae_hidden_size_200_batch_size_10_lr_0.001  \n",
       "140  vae_hidden_size_200_batch_size_10_lr_0.0003  \n",
       "141   vae_hidden_size_200_batch_size_20_lr_0.003  \n",
       "142   vae_hidden_size_200_batch_size_20_lr_0.001  \n",
       "143  vae_hidden_size_200_batch_size_20_lr_0.0003  \n",
       "144   vae_hidden_size_100_batch_size_10_lr_0.003  \n",
       "145   vae_hidden_size_100_batch_size_10_lr_0.001  \n",
       "146  vae_hidden_size_100_batch_size_10_lr_0.0003  \n",
       "147   vae_hidden_size_100_batch_size_20_lr_0.003  \n",
       "148   vae_hidden_size_100_batch_size_20_lr_0.001  \n",
       "149  vae_hidden_size_100_batch_size_20_lr_0.0003  \n",
       "150   vae_hidden_size_200_batch_size_10_lr_0.003  \n",
       "151   vae_hidden_size_200_batch_size_10_lr_0.001  \n",
       "152  vae_hidden_size_200_batch_size_10_lr_0.0003  \n",
       "153   vae_hidden_size_200_batch_size_20_lr_0.003  \n",
       "154   vae_hidden_size_200_batch_size_20_lr_0.001  \n",
       "155  vae_hidden_size_200_batch_size_20_lr_0.0003  \n",
       "\n",
       "[156 rows x 10 columns]"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vae_df[\"test_score\"] = vae_df[\"test_score\"].apply(lambda x : eval(x)[0])\n",
    "vae_df[\"valid_score\"] = vae_df[\"valid_score\"].apply(lambda x : eval(x)[0])\n",
    "vae_df[\"train_score\"] = vae_df[\"train_score\"].apply(lambda x : eval(x)[0])\n",
    "vae_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12 92\n"
     ]
    }
   ],
   "source": [
    "rnn_drop_list = [i for i, model in enumerate(rnn_df[\"model_type\"].values) if model == '0.0']\n",
    "rnn_keep_list = [i for i, model in enumerate(rnn_df[\"model_type\"].values) if model != '0.0']\n",
    "print(len(rnn_drop_list), len(rnn_keep_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_type</th>\n",
       "      <th>test_score</th>\n",
       "      <th>total_time</th>\n",
       "      <th>dataset</th>\n",
       "      <th>average_mismatches</th>\n",
       "      <th>valid_score</th>\n",
       "      <th>train_score</th>\n",
       "      <th>num_params</th>\n",
       "      <th>exit_code</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>rnn</td>\n",
       "      <td>39.041250</td>\n",
       "      <td>327.334224</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>9.658</td>\n",
       "      <td>39.180859</td>\n",
       "      <td>36.785492</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>rnn</td>\n",
       "      <td>38.913168</td>\n",
       "      <td>533.507526</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>9.039</td>\n",
       "      <td>39.080164</td>\n",
       "      <td>36.267809</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>rnn</td>\n",
       "      <td>38.883285</td>\n",
       "      <td>1073.661136</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>10.898</td>\n",
       "      <td>39.112852</td>\n",
       "      <td>36.501531</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>rnn</td>\n",
       "      <td>39.304941</td>\n",
       "      <td>261.895198</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>12.492</td>\n",
       "      <td>39.444434</td>\n",
       "      <td>36.777391</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>rnn</td>\n",
       "      <td>38.893160</td>\n",
       "      <td>339.151133</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>10.791</td>\n",
       "      <td>39.059820</td>\n",
       "      <td>36.753918</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>rnn</td>\n",
       "      <td>38.821891</td>\n",
       "      <td>684.404752</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>10.848</td>\n",
       "      <td>38.987711</td>\n",
       "      <td>36.379570</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>rnn</td>\n",
       "      <td>36.262723</td>\n",
       "      <td>381.493295</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>8.086</td>\n",
       "      <td>35.983645</td>\n",
       "      <td>33.648223</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>rnn</td>\n",
       "      <td>36.129309</td>\n",
       "      <td>472.184616</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>10.021</td>\n",
       "      <td>35.847008</td>\n",
       "      <td>34.329863</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>rnn</td>\n",
       "      <td>36.183816</td>\n",
       "      <td>970.540609</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>10.314</td>\n",
       "      <td>35.869551</td>\n",
       "      <td>34.202328</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>rnn</td>\n",
       "      <td>36.367871</td>\n",
       "      <td>285.203783</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>9.451</td>\n",
       "      <td>36.070629</td>\n",
       "      <td>33.860328</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>rnn</td>\n",
       "      <td>36.141094</td>\n",
       "      <td>356.974194</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>8.049</td>\n",
       "      <td>35.923973</td>\n",
       "      <td>33.882578</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>rnn</td>\n",
       "      <td>36.168086</td>\n",
       "      <td>641.726103</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>10.217</td>\n",
       "      <td>35.864566</td>\n",
       "      <td>33.564586</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>rnn</td>\n",
       "      <td>38.366148</td>\n",
       "      <td>378.034331</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>9.599</td>\n",
       "      <td>38.299445</td>\n",
       "      <td>35.923527</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>rnn</td>\n",
       "      <td>38.185027</td>\n",
       "      <td>470.050228</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>10.427</td>\n",
       "      <td>38.294895</td>\n",
       "      <td>36.404875</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>rnn</td>\n",
       "      <td>38.258512</td>\n",
       "      <td>926.744784</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>11.667</td>\n",
       "      <td>38.307598</td>\n",
       "      <td>36.675168</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>rnn</td>\n",
       "      <td>38.522609</td>\n",
       "      <td>274.480432</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>10.642</td>\n",
       "      <td>38.591848</td>\n",
       "      <td>36.219410</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>rnn</td>\n",
       "      <td>38.307773</td>\n",
       "      <td>396.676757</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>10.250</td>\n",
       "      <td>38.201707</td>\n",
       "      <td>35.783719</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>rnn</td>\n",
       "      <td>38.245508</td>\n",
       "      <td>559.735744</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>12.966</td>\n",
       "      <td>38.235281</td>\n",
       "      <td>36.285891</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>rnn</td>\n",
       "      <td>38.270398</td>\n",
       "      <td>1490.091855</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>11.802</td>\n",
       "      <td>38.262762</td>\n",
       "      <td>36.095855</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>rnn</td>\n",
       "      <td>28.377768</td>\n",
       "      <td>63.586783</td>\n",
       "      <td>synthetic_unimodal_data_length_20_gaussian</td>\n",
       "      <td>6.011</td>\n",
       "      <td>27.682801</td>\n",
       "      <td>26.794246</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>rnn</td>\n",
       "      <td>28.286723</td>\n",
       "      <td>74.732424</td>\n",
       "      <td>synthetic_unimodal_data_length_20_gaussian</td>\n",
       "      <td>6.004</td>\n",
       "      <td>27.608195</td>\n",
       "      <td>26.949131</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>rnn</td>\n",
       "      <td>28.228463</td>\n",
       "      <td>107.130204</td>\n",
       "      <td>synthetic_unimodal_data_length_20_gaussian</td>\n",
       "      <td>6.039</td>\n",
       "      <td>27.624609</td>\n",
       "      <td>27.172973</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>rnn</td>\n",
       "      <td>28.313439</td>\n",
       "      <td>206.907092</td>\n",
       "      <td>synthetic_unimodal_data_length_20_gaussian</td>\n",
       "      <td>6.039</td>\n",
       "      <td>27.706686</td>\n",
       "      <td>27.206463</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>rnn</td>\n",
       "      <td>28.402613</td>\n",
       "      <td>53.461811</td>\n",
       "      <td>synthetic_unimodal_data_length_20_gaussian</td>\n",
       "      <td>5.834</td>\n",
       "      <td>27.795430</td>\n",
       "      <td>27.275594</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>rnn</td>\n",
       "      <td>28.294822</td>\n",
       "      <td>56.605158</td>\n",
       "      <td>synthetic_unimodal_data_length_20_gaussian</td>\n",
       "      <td>5.679</td>\n",
       "      <td>27.617699</td>\n",
       "      <td>27.253438</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>rnn</td>\n",
       "      <td>28.288178</td>\n",
       "      <td>82.576923</td>\n",
       "      <td>synthetic_unimodal_data_length_20_gaussian</td>\n",
       "      <td>5.662</td>\n",
       "      <td>27.660217</td>\n",
       "      <td>26.987375</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>rnn</td>\n",
       "      <td>28.312199</td>\n",
       "      <td>129.433289</td>\n",
       "      <td>synthetic_unimodal_data_length_20_gaussian</td>\n",
       "      <td>6.058</td>\n",
       "      <td>27.700510</td>\n",
       "      <td>27.105324</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>rnn</td>\n",
       "      <td>25.877109</td>\n",
       "      <td>52.660094</td>\n",
       "      <td>synthetic_unimodal_data_length_20_skewed_gaussian</td>\n",
       "      <td>5.495</td>\n",
       "      <td>25.969102</td>\n",
       "      <td>25.736549</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>rnn</td>\n",
       "      <td>25.749234</td>\n",
       "      <td>76.371028</td>\n",
       "      <td>synthetic_unimodal_data_length_20_skewed_gaussian</td>\n",
       "      <td>5.468</td>\n",
       "      <td>25.888912</td>\n",
       "      <td>25.135848</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>rnn</td>\n",
       "      <td>25.775895</td>\n",
       "      <td>114.090553</td>\n",
       "      <td>synthetic_unimodal_data_length_20_skewed_gaussian</td>\n",
       "      <td>5.425</td>\n",
       "      <td>25.869406</td>\n",
       "      <td>25.293008</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>rnn</td>\n",
       "      <td>33.787645</td>\n",
       "      <td>1081.808056</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>7.981</td>\n",
       "      <td>34.746117</td>\n",
       "      <td>32.035871</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>rnn</td>\n",
       "      <td>33.981641</td>\n",
       "      <td>124.240005</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>6.890</td>\n",
       "      <td>34.927422</td>\n",
       "      <td>31.818404</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>rnn</td>\n",
       "      <td>33.812152</td>\n",
       "      <td>154.081646</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>7.076</td>\n",
       "      <td>34.763008</td>\n",
       "      <td>32.069621</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>rnn</td>\n",
       "      <td>33.768129</td>\n",
       "      <td>242.906807</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>7.514</td>\n",
       "      <td>34.661430</td>\n",
       "      <td>32.204236</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>rnn</td>\n",
       "      <td>33.726180</td>\n",
       "      <td>623.180200</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>7.690</td>\n",
       "      <td>34.659250</td>\n",
       "      <td>31.641914</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>rnn</td>\n",
       "      <td>25.395535</td>\n",
       "      <td>1240.126062</td>\n",
       "      <td>gfp</td>\n",
       "      <td>17.808</td>\n",
       "      <td>24.985129</td>\n",
       "      <td>22.391420</td>\n",
       "      <td>330021.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>rnn</td>\n",
       "      <td>35.631988</td>\n",
       "      <td>169.658693</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>8.364</td>\n",
       "      <td>36.228512</td>\n",
       "      <td>32.911082</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>rnn</td>\n",
       "      <td>35.605438</td>\n",
       "      <td>216.761894</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>10.835</td>\n",
       "      <td>36.176953</td>\n",
       "      <td>33.747047</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>rnn</td>\n",
       "      <td>35.834684</td>\n",
       "      <td>605.084743</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>11.174</td>\n",
       "      <td>36.499324</td>\n",
       "      <td>33.501305</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>rnn</td>\n",
       "      <td>36.169508</td>\n",
       "      <td>1169.020653</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>13.262</td>\n",
       "      <td>36.781086</td>\n",
       "      <td>33.881211</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>rnn</td>\n",
       "      <td>35.826004</td>\n",
       "      <td>127.044826</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>9.040</td>\n",
       "      <td>36.477477</td>\n",
       "      <td>33.473340</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>rnn</td>\n",
       "      <td>35.541008</td>\n",
       "      <td>157.289691</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>10.317</td>\n",
       "      <td>36.125641</td>\n",
       "      <td>33.224434</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>rnn</td>\n",
       "      <td>35.462898</td>\n",
       "      <td>377.938907</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>10.723</td>\n",
       "      <td>36.114004</td>\n",
       "      <td>32.871695</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>rnn</td>\n",
       "      <td>35.728023</td>\n",
       "      <td>697.383602</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>12.545</td>\n",
       "      <td>36.413187</td>\n",
       "      <td>33.260758</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>rnn</td>\n",
       "      <td>36.477801</td>\n",
       "      <td>163.772760</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>12.213</td>\n",
       "      <td>37.729402</td>\n",
       "      <td>32.746607</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>rnn</td>\n",
       "      <td>36.742848</td>\n",
       "      <td>213.501277</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>13.606</td>\n",
       "      <td>38.061488</td>\n",
       "      <td>33.245238</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>rnn</td>\n",
       "      <td>37.228730</td>\n",
       "      <td>413.429590</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>15.021</td>\n",
       "      <td>38.690090</td>\n",
       "      <td>33.664105</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>rnn</td>\n",
       "      <td>38.193621</td>\n",
       "      <td>989.980696</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>19.245</td>\n",
       "      <td>39.828082</td>\n",
       "      <td>34.425359</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>rnn</td>\n",
       "      <td>36.827281</td>\n",
       "      <td>142.279447</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>11.925</td>\n",
       "      <td>38.068605</td>\n",
       "      <td>31.910273</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>rnn</td>\n",
       "      <td>36.533168</td>\n",
       "      <td>156.772378</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>16.580</td>\n",
       "      <td>37.974898</td>\n",
       "      <td>33.032988</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>rnn</td>\n",
       "      <td>36.771453</td>\n",
       "      <td>308.090046</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>14.313</td>\n",
       "      <td>38.155109</td>\n",
       "      <td>31.783441</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>rnn</td>\n",
       "      <td>37.421531</td>\n",
       "      <td>682.632111</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>17.059</td>\n",
       "      <td>38.857844</td>\n",
       "      <td>32.276232</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>rnn</td>\n",
       "      <td>39.655797</td>\n",
       "      <td>164.057840</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>19.471</td>\n",
       "      <td>39.620852</td>\n",
       "      <td>34.185230</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>rnn</td>\n",
       "      <td>39.959129</td>\n",
       "      <td>267.940756</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>20.335</td>\n",
       "      <td>39.981555</td>\n",
       "      <td>33.498277</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>rnn</td>\n",
       "      <td>41.270992</td>\n",
       "      <td>670.977205</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>21.469</td>\n",
       "      <td>41.324469</td>\n",
       "      <td>33.180602</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>rnn</td>\n",
       "      <td>43.054656</td>\n",
       "      <td>793.345631</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>28.623</td>\n",
       "      <td>43.364820</td>\n",
       "      <td>38.009324</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>rnn</td>\n",
       "      <td>39.624266</td>\n",
       "      <td>152.613585</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>19.121</td>\n",
       "      <td>39.693438</td>\n",
       "      <td>32.045396</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>rnn</td>\n",
       "      <td>39.490316</td>\n",
       "      <td>193.582643</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>17.596</td>\n",
       "      <td>39.558094</td>\n",
       "      <td>32.309951</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>rnn</td>\n",
       "      <td>40.275746</td>\n",
       "      <td>294.846441</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>21.989</td>\n",
       "      <td>40.416355</td>\n",
       "      <td>33.508980</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>rnn</td>\n",
       "      <td>41.335715</td>\n",
       "      <td>647.849923</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>23.782</td>\n",
       "      <td>41.434848</td>\n",
       "      <td>33.627953</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0001</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>92 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    model_type  test_score   total_time  \\\n",
       "0          rnn   39.041250   327.334224   \n",
       "1          rnn   38.913168   533.507526   \n",
       "2          rnn   38.883285  1073.661136   \n",
       "4          rnn   39.304941   261.895198   \n",
       "5          rnn   38.893160   339.151133   \n",
       "6          rnn   38.821891   684.404752   \n",
       "8          rnn   36.262723   381.493295   \n",
       "9          rnn   36.129309   472.184616   \n",
       "10         rnn   36.183816   970.540609   \n",
       "12         rnn   36.367871   285.203783   \n",
       "13         rnn   36.141094   356.974194   \n",
       "14         rnn   36.168086   641.726103   \n",
       "16         rnn   38.366148   378.034331   \n",
       "17         rnn   38.185027   470.050228   \n",
       "18         rnn   38.258512   926.744784   \n",
       "20         rnn   38.522609   274.480432   \n",
       "21         rnn   38.307773   396.676757   \n",
       "22         rnn   38.245508   559.735744   \n",
       "23         rnn   38.270398  1490.091855   \n",
       "24         rnn   28.377768    63.586783   \n",
       "25         rnn   28.286723    74.732424   \n",
       "26         rnn   28.228463   107.130204   \n",
       "27         rnn   28.313439   206.907092   \n",
       "28         rnn   28.402613    53.461811   \n",
       "29         rnn   28.294822    56.605158   \n",
       "30         rnn   28.288178    82.576923   \n",
       "31         rnn   28.312199   129.433289   \n",
       "32         rnn   25.877109    52.660094   \n",
       "33         rnn   25.749234    76.371028   \n",
       "34         rnn   25.775895   114.090553   \n",
       "..         ...         ...          ...   \n",
       "67         rnn   33.787645  1081.808056   \n",
       "68         rnn   33.981641   124.240005   \n",
       "69         rnn   33.812152   154.081646   \n",
       "70         rnn   33.768129   242.906807   \n",
       "71         rnn   33.726180   623.180200   \n",
       "76         rnn   25.395535  1240.126062   \n",
       "80         rnn   35.631988   169.658693   \n",
       "81         rnn   35.605438   216.761894   \n",
       "82         rnn   35.834684   605.084743   \n",
       "83         rnn   36.169508  1169.020653   \n",
       "84         rnn   35.826004   127.044826   \n",
       "85         rnn   35.541008   157.289691   \n",
       "86         rnn   35.462898   377.938907   \n",
       "87         rnn   35.728023   697.383602   \n",
       "88         rnn   36.477801   163.772760   \n",
       "89         rnn   36.742848   213.501277   \n",
       "90         rnn   37.228730   413.429590   \n",
       "91         rnn   38.193621   989.980696   \n",
       "92         rnn   36.827281   142.279447   \n",
       "93         rnn   36.533168   156.772378   \n",
       "94         rnn   36.771453   308.090046   \n",
       "95         rnn   37.421531   682.632111   \n",
       "96         rnn   39.655797   164.057840   \n",
       "97         rnn   39.959129   267.940756   \n",
       "98         rnn   41.270992   670.977205   \n",
       "99         rnn   43.054656   793.345631   \n",
       "100        rnn   39.624266   152.613585   \n",
       "101        rnn   39.490316   193.582643   \n",
       "102        rnn   40.275746   294.846441   \n",
       "103        rnn   41.335715   647.849923   \n",
       "\n",
       "                                               dataset  average_mismatches  \\\n",
       "0          synthetic_unimodal_data_length_100_gaussian               9.658   \n",
       "1          synthetic_unimodal_data_length_100_gaussian               9.039   \n",
       "2          synthetic_unimodal_data_length_100_gaussian              10.898   \n",
       "4          synthetic_unimodal_data_length_100_gaussian              12.492   \n",
       "5          synthetic_unimodal_data_length_100_gaussian              10.791   \n",
       "6          synthetic_unimodal_data_length_100_gaussian              10.848   \n",
       "8    synthetic_unimodal_data_length_100_skewed_gaus...               8.086   \n",
       "9    synthetic_unimodal_data_length_100_skewed_gaus...              10.021   \n",
       "10   synthetic_unimodal_data_length_100_skewed_gaus...              10.314   \n",
       "12   synthetic_unimodal_data_length_100_skewed_gaus...               9.451   \n",
       "13   synthetic_unimodal_data_length_100_skewed_gaus...               8.049   \n",
       "14   synthetic_unimodal_data_length_100_skewed_gaus...              10.217   \n",
       "16          synthetic_unimodal_data_length_100_uniform               9.599   \n",
       "17          synthetic_unimodal_data_length_100_uniform              10.427   \n",
       "18          synthetic_unimodal_data_length_100_uniform              11.667   \n",
       "20          synthetic_unimodal_data_length_100_uniform              10.642   \n",
       "21          synthetic_unimodal_data_length_100_uniform              10.250   \n",
       "22          synthetic_unimodal_data_length_100_uniform              12.966   \n",
       "23          synthetic_unimodal_data_length_100_uniform              11.802   \n",
       "24          synthetic_unimodal_data_length_20_gaussian               6.011   \n",
       "25          synthetic_unimodal_data_length_20_gaussian               6.004   \n",
       "26          synthetic_unimodal_data_length_20_gaussian               6.039   \n",
       "27          synthetic_unimodal_data_length_20_gaussian               6.039   \n",
       "28          synthetic_unimodal_data_length_20_gaussian               5.834   \n",
       "29          synthetic_unimodal_data_length_20_gaussian               5.679   \n",
       "30          synthetic_unimodal_data_length_20_gaussian               5.662   \n",
       "31          synthetic_unimodal_data_length_20_gaussian               6.058   \n",
       "32   synthetic_unimodal_data_length_20_skewed_gaussian               5.495   \n",
       "33   synthetic_unimodal_data_length_20_skewed_gaussian               5.468   \n",
       "34   synthetic_unimodal_data_length_20_skewed_gaussian               5.425   \n",
       "..                                                 ...                 ...   \n",
       "67           synthetic_unimodal_data_length_50_uniform               7.981   \n",
       "68           synthetic_unimodal_data_length_50_uniform               6.890   \n",
       "69           synthetic_unimodal_data_length_50_uniform               7.076   \n",
       "70           synthetic_unimodal_data_length_50_uniform               7.514   \n",
       "71           synthetic_unimodal_data_length_50_uniform               7.690   \n",
       "76                                                 gfp              17.808   \n",
       "80   synthetic_multimodal_data_modes_2_length_51_un...               8.364   \n",
       "81   synthetic_multimodal_data_modes_2_length_51_un...              10.835   \n",
       "82   synthetic_multimodal_data_modes_2_length_51_un...              11.174   \n",
       "83   synthetic_multimodal_data_modes_2_length_51_un...              13.262   \n",
       "84   synthetic_multimodal_data_modes_2_length_51_un...               9.040   \n",
       "85   synthetic_multimodal_data_modes_2_length_51_un...              10.317   \n",
       "86   synthetic_multimodal_data_modes_2_length_51_un...              10.723   \n",
       "87   synthetic_multimodal_data_modes_2_length_51_un...              12.545   \n",
       "88   synthetic_multimodal_data_modes_3_length_51_un...              12.213   \n",
       "89   synthetic_multimodal_data_modes_3_length_51_un...              13.606   \n",
       "90   synthetic_multimodal_data_modes_3_length_51_un...              15.021   \n",
       "91   synthetic_multimodal_data_modes_3_length_51_un...              19.245   \n",
       "92   synthetic_multimodal_data_modes_3_length_51_un...              11.925   \n",
       "93   synthetic_multimodal_data_modes_3_length_51_un...              16.580   \n",
       "94   synthetic_multimodal_data_modes_3_length_51_un...              14.313   \n",
       "95   synthetic_multimodal_data_modes_3_length_51_un...              17.059   \n",
       "96   synthetic_multimodal_data_modes_5_length_51_un...              19.471   \n",
       "97   synthetic_multimodal_data_modes_5_length_51_un...              20.335   \n",
       "98   synthetic_multimodal_data_modes_5_length_51_un...              21.469   \n",
       "99   synthetic_multimodal_data_modes_5_length_51_un...              28.623   \n",
       "100  synthetic_multimodal_data_modes_5_length_51_un...              19.121   \n",
       "101  synthetic_multimodal_data_modes_5_length_51_un...              17.596   \n",
       "102  synthetic_multimodal_data_modes_5_length_51_un...              21.989   \n",
       "103  synthetic_multimodal_data_modes_5_length_51_un...              23.782   \n",
       "\n",
       "     valid_score  train_score  num_params  exit_code  \\\n",
       "0      39.180859    36.785492     84820.0        0.0   \n",
       "1      39.080164    36.267809     84820.0        0.0   \n",
       "2      39.112852    36.501531     84820.0        0.0   \n",
       "4      39.444434    36.777391    329620.0        0.0   \n",
       "5      39.059820    36.753918    329620.0        0.0   \n",
       "6      38.987711    36.379570    329620.0        0.0   \n",
       "8      35.983645    33.648223     84820.0        0.0   \n",
       "9      35.847008    34.329863     84820.0        0.0   \n",
       "10     35.869551    34.202328     84820.0        0.0   \n",
       "12     36.070629    33.860328    329620.0        0.0   \n",
       "13     35.923973    33.882578    329620.0        0.0   \n",
       "14     35.864566    33.564586    329620.0        0.0   \n",
       "16     38.299445    35.923527     84820.0        0.0   \n",
       "17     38.294895    36.404875     84820.0        0.0   \n",
       "18     38.307598    36.675168     84820.0        0.0   \n",
       "20     38.591848    36.219410    329620.0        0.0   \n",
       "21     38.201707    35.783719    329620.0        0.0   \n",
       "22     38.235281    36.285891    329620.0        0.0   \n",
       "23     38.262762    36.095855    329620.0        0.0   \n",
       "24     27.682801    26.794246     84820.0        0.0   \n",
       "25     27.608195    26.949131     84820.0        0.0   \n",
       "26     27.624609    27.172973     84820.0        0.0   \n",
       "27     27.706686    27.206463     84820.0        0.0   \n",
       "28     27.795430    27.275594    329620.0        0.0   \n",
       "29     27.617699    27.253438    329620.0        0.0   \n",
       "30     27.660217    26.987375    329620.0        0.0   \n",
       "31     27.700510    27.105324    329620.0        0.0   \n",
       "32     25.969102    25.736549     84820.0        0.0   \n",
       "33     25.888912    25.135848     84820.0        0.0   \n",
       "34     25.869406    25.293008     84820.0        0.0   \n",
       "..           ...          ...         ...        ...   \n",
       "67     34.746117    32.035871     84820.0        0.0   \n",
       "68     34.927422    31.818404    329620.0        0.0   \n",
       "69     34.763008    32.069621    329620.0        0.0   \n",
       "70     34.661430    32.204236    329620.0        0.0   \n",
       "71     34.659250    31.641914    329620.0        0.0   \n",
       "76     24.985129    22.391420    330021.0        0.0   \n",
       "80     36.228512    32.911082     84820.0        0.0   \n",
       "81     36.176953    33.747047     84820.0        0.0   \n",
       "82     36.499324    33.501305     84820.0        0.0   \n",
       "83     36.781086    33.881211     84820.0        0.0   \n",
       "84     36.477477    33.473340    329620.0        0.0   \n",
       "85     36.125641    33.224434    329620.0        0.0   \n",
       "86     36.114004    32.871695    329620.0        0.0   \n",
       "87     36.413187    33.260758    329620.0        0.0   \n",
       "88     37.729402    32.746607     84820.0        0.0   \n",
       "89     38.061488    33.245238     84820.0        0.0   \n",
       "90     38.690090    33.664105     84820.0        0.0   \n",
       "91     39.828082    34.425359     84820.0        0.0   \n",
       "92     38.068605    31.910273    329620.0        0.0   \n",
       "93     37.974898    33.032988    329620.0        0.0   \n",
       "94     38.155109    31.783441    329620.0        0.0   \n",
       "95     38.857844    32.276232    329620.0        0.0   \n",
       "96     39.620852    34.185230     84820.0        0.0   \n",
       "97     39.981555    33.498277     84820.0        0.0   \n",
       "98     41.324469    33.180602     84820.0        0.0   \n",
       "99     43.364820    38.009324     84820.0        0.0   \n",
       "100    39.693438    32.045396    329620.0        0.0   \n",
       "101    39.558094    32.309951    329620.0        0.0   \n",
       "102    40.416355    33.508980    329620.0        0.0   \n",
       "103    41.434848    33.627953    329620.0        0.0   \n",
       "\n",
       "                              name  \n",
       "0     rnn_hidden_size_100_lr_0.003  \n",
       "1     rnn_hidden_size_100_lr_0.001  \n",
       "2    rnn_hidden_size_100_lr_0.0003  \n",
       "4     rnn_hidden_size_200_lr_0.003  \n",
       "5     rnn_hidden_size_200_lr_0.001  \n",
       "6    rnn_hidden_size_200_lr_0.0003  \n",
       "8     rnn_hidden_size_100_lr_0.003  \n",
       "9     rnn_hidden_size_100_lr_0.001  \n",
       "10   rnn_hidden_size_100_lr_0.0003  \n",
       "12    rnn_hidden_size_200_lr_0.003  \n",
       "13    rnn_hidden_size_200_lr_0.001  \n",
       "14   rnn_hidden_size_200_lr_0.0003  \n",
       "16    rnn_hidden_size_100_lr_0.003  \n",
       "17    rnn_hidden_size_100_lr_0.001  \n",
       "18   rnn_hidden_size_100_lr_0.0003  \n",
       "20    rnn_hidden_size_200_lr_0.003  \n",
       "21    rnn_hidden_size_200_lr_0.001  \n",
       "22   rnn_hidden_size_200_lr_0.0003  \n",
       "23   rnn_hidden_size_200_lr_0.0001  \n",
       "24    rnn_hidden_size_100_lr_0.003  \n",
       "25    rnn_hidden_size_100_lr_0.001  \n",
       "26   rnn_hidden_size_100_lr_0.0003  \n",
       "27   rnn_hidden_size_100_lr_0.0001  \n",
       "28    rnn_hidden_size_200_lr_0.003  \n",
       "29    rnn_hidden_size_200_lr_0.001  \n",
       "30   rnn_hidden_size_200_lr_0.0003  \n",
       "31   rnn_hidden_size_200_lr_0.0001  \n",
       "32    rnn_hidden_size_100_lr_0.003  \n",
       "33    rnn_hidden_size_100_lr_0.001  \n",
       "34   rnn_hidden_size_100_lr_0.0003  \n",
       "..                             ...  \n",
       "67   rnn_hidden_size_100_lr_0.0001  \n",
       "68    rnn_hidden_size_200_lr_0.003  \n",
       "69    rnn_hidden_size_200_lr_0.001  \n",
       "70   rnn_hidden_size_200_lr_0.0003  \n",
       "71   rnn_hidden_size_200_lr_0.0001  \n",
       "76    rnn_hidden_size_200_lr_0.003  \n",
       "80    rnn_hidden_size_100_lr_0.003  \n",
       "81    rnn_hidden_size_100_lr_0.001  \n",
       "82   rnn_hidden_size_100_lr_0.0003  \n",
       "83   rnn_hidden_size_100_lr_0.0001  \n",
       "84    rnn_hidden_size_200_lr_0.003  \n",
       "85    rnn_hidden_size_200_lr_0.001  \n",
       "86   rnn_hidden_size_200_lr_0.0003  \n",
       "87   rnn_hidden_size_200_lr_0.0001  \n",
       "88    rnn_hidden_size_100_lr_0.003  \n",
       "89    rnn_hidden_size_100_lr_0.001  \n",
       "90   rnn_hidden_size_100_lr_0.0003  \n",
       "91   rnn_hidden_size_100_lr_0.0001  \n",
       "92    rnn_hidden_size_200_lr_0.003  \n",
       "93    rnn_hidden_size_200_lr_0.001  \n",
       "94   rnn_hidden_size_200_lr_0.0003  \n",
       "95   rnn_hidden_size_200_lr_0.0001  \n",
       "96    rnn_hidden_size_100_lr_0.003  \n",
       "97    rnn_hidden_size_100_lr_0.001  \n",
       "98   rnn_hidden_size_100_lr_0.0003  \n",
       "99   rnn_hidden_size_100_lr_0.0001  \n",
       "100   rnn_hidden_size_200_lr_0.003  \n",
       "101   rnn_hidden_size_200_lr_0.001  \n",
       "102  rnn_hidden_size_200_lr_0.0003  \n",
       "103  rnn_hidden_size_200_lr_0.0001  \n",
       "\n",
       "[92 rows x 10 columns]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnn_df = rnn_df.iloc[rnn_keep_list, :]\n",
    "rnn_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34 18\n"
     ]
    }
   ],
   "source": [
    "hmm_drop_list = [i for i, model in enumerate(hmm_df[\"model_type\"].values) if model == '0.0']\n",
    "hmm_keep_list = [i for i, model in enumerate(hmm_df[\"model_type\"].values) if model != '0.0']\n",
    "print(len(hmm_keep_list), len(hmm_drop_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_type</th>\n",
       "      <th>test_score</th>\n",
       "      <th>total_time</th>\n",
       "      <th>dataset</th>\n",
       "      <th>average_mismatches</th>\n",
       "      <th>valid_score</th>\n",
       "      <th>train_score</th>\n",
       "      <th>num_params</th>\n",
       "      <th>exit_code</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>hmm</td>\n",
       "      <td>163.729458</td>\n",
       "      <td>672.123741</td>\n",
       "      <td>synthetic_unimodal_data_length_100_gaussian</td>\n",
       "      <td>90.335</td>\n",
       "      <td>163.819790</td>\n",
       "      <td>163.462043</td>\n",
       "      <td>800.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>hmm</td>\n",
       "      <td>160.439516</td>\n",
       "      <td>584.543961</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>90.671</td>\n",
       "      <td>160.227315</td>\n",
       "      <td>160.183613</td>\n",
       "      <td>800.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>hmm</td>\n",
       "      <td>138.283641</td>\n",
       "      <td>670.506927</td>\n",
       "      <td>synthetic_unimodal_data_length_100_skewed_gaus...</td>\n",
       "      <td>90.365</td>\n",
       "      <td>138.019667</td>\n",
       "      <td>137.851505</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>hmm</td>\n",
       "      <td>161.177640</td>\n",
       "      <td>246.149052</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>90.686</td>\n",
       "      <td>161.168688</td>\n",
       "      <td>161.434284</td>\n",
       "      <td>800.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>hmm</td>\n",
       "      <td>127.280070</td>\n",
       "      <td>899.934500</td>\n",
       "      <td>synthetic_unimodal_data_length_100_uniform</td>\n",
       "      <td>89.869</td>\n",
       "      <td>127.157544</td>\n",
       "      <td>127.213086</td>\n",
       "      <td>2400.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>hmm</td>\n",
       "      <td>30.578423</td>\n",
       "      <td>93.825385</td>\n",
       "      <td>synthetic_unimodal_data_length_20_gaussian</td>\n",
       "      <td>10.029</td>\n",
       "      <td>30.097017</td>\n",
       "      <td>30.222726</td>\n",
       "      <td>800.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>hmm</td>\n",
       "      <td>33.690124</td>\n",
       "      <td>318.300480</td>\n",
       "      <td>synthetic_unimodal_data_length_20_gaussian</td>\n",
       "      <td>13.709</td>\n",
       "      <td>33.164068</td>\n",
       "      <td>33.360533</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>hmm</td>\n",
       "      <td>31.618671</td>\n",
       "      <td>533.252563</td>\n",
       "      <td>synthetic_unimodal_data_length_20_gaussian</td>\n",
       "      <td>13.828</td>\n",
       "      <td>30.967193</td>\n",
       "      <td>31.185973</td>\n",
       "      <td>2400.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>hmm</td>\n",
       "      <td>32.385205</td>\n",
       "      <td>377.175702</td>\n",
       "      <td>synthetic_unimodal_data_length_20_gaussian</td>\n",
       "      <td>14.633</td>\n",
       "      <td>31.873334</td>\n",
       "      <td>32.050543</td>\n",
       "      <td>3500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>hmm</td>\n",
       "      <td>32.220333</td>\n",
       "      <td>69.040670</td>\n",
       "      <td>synthetic_unimodal_data_length_20_skewed_gaussian</td>\n",
       "      <td>13.279</td>\n",
       "      <td>32.293674</td>\n",
       "      <td>32.497731</td>\n",
       "      <td>800.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>hmm</td>\n",
       "      <td>26.155596</td>\n",
       "      <td>151.421632</td>\n",
       "      <td>synthetic_unimodal_data_length_20_skewed_gaussian</td>\n",
       "      <td>8.430</td>\n",
       "      <td>26.205717</td>\n",
       "      <td>26.274437</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>hmm</td>\n",
       "      <td>28.630247</td>\n",
       "      <td>177.973732</td>\n",
       "      <td>synthetic_unimodal_data_length_20_skewed_gaussian</td>\n",
       "      <td>13.844</td>\n",
       "      <td>28.687373</td>\n",
       "      <td>28.775390</td>\n",
       "      <td>2400.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>hmm</td>\n",
       "      <td>27.959026</td>\n",
       "      <td>349.081254</td>\n",
       "      <td>synthetic_unimodal_data_length_20_skewed_gaussian</td>\n",
       "      <td>14.153</td>\n",
       "      <td>28.036499</td>\n",
       "      <td>28.151720</td>\n",
       "      <td>3500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>hmm</td>\n",
       "      <td>30.299214</td>\n",
       "      <td>95.458056</td>\n",
       "      <td>synthetic_unimodal_data_length_20_uniform</td>\n",
       "      <td>9.982</td>\n",
       "      <td>30.443394</td>\n",
       "      <td>30.153763</td>\n",
       "      <td>800.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>hmm</td>\n",
       "      <td>33.467269</td>\n",
       "      <td>103.940329</td>\n",
       "      <td>synthetic_unimodal_data_length_20_uniform</td>\n",
       "      <td>14.048</td>\n",
       "      <td>33.588177</td>\n",
       "      <td>33.340336</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>hmm</td>\n",
       "      <td>30.725757</td>\n",
       "      <td>522.775589</td>\n",
       "      <td>synthetic_unimodal_data_length_20_uniform</td>\n",
       "      <td>10.843</td>\n",
       "      <td>30.879700</td>\n",
       "      <td>30.551452</td>\n",
       "      <td>2400.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>hmm</td>\n",
       "      <td>29.487527</td>\n",
       "      <td>812.683749</td>\n",
       "      <td>synthetic_unimodal_data_length_20_uniform</td>\n",
       "      <td>10.597</td>\n",
       "      <td>29.629142</td>\n",
       "      <td>29.331068</td>\n",
       "      <td>3500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>hmm</td>\n",
       "      <td>78.123778</td>\n",
       "      <td>149.141079</td>\n",
       "      <td>synthetic_unimodal_data_length_50_gaussian</td>\n",
       "      <td>43.880</td>\n",
       "      <td>78.726581</td>\n",
       "      <td>78.383688</td>\n",
       "      <td>800.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>hmm</td>\n",
       "      <td>62.896197</td>\n",
       "      <td>193.520639</td>\n",
       "      <td>synthetic_unimodal_data_length_50_gaussian</td>\n",
       "      <td>42.508</td>\n",
       "      <td>63.241722</td>\n",
       "      <td>62.951611</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>hmm</td>\n",
       "      <td>49.870443</td>\n",
       "      <td>607.648293</td>\n",
       "      <td>synthetic_unimodal_data_length_50_gaussian</td>\n",
       "      <td>42.432</td>\n",
       "      <td>50.335326</td>\n",
       "      <td>49.897661</td>\n",
       "      <td>3500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>hmm</td>\n",
       "      <td>78.473292</td>\n",
       "      <td>90.185113</td>\n",
       "      <td>synthetic_unimodal_data_length_50_skewed_gaussian</td>\n",
       "      <td>44.292</td>\n",
       "      <td>78.136042</td>\n",
       "      <td>78.188168</td>\n",
       "      <td>800.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>hmm</td>\n",
       "      <td>65.198984</td>\n",
       "      <td>193.754603</td>\n",
       "      <td>synthetic_unimodal_data_length_50_skewed_gaussian</td>\n",
       "      <td>43.334</td>\n",
       "      <td>64.927422</td>\n",
       "      <td>64.724603</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>hmm</td>\n",
       "      <td>54.254814</td>\n",
       "      <td>481.134427</td>\n",
       "      <td>synthetic_unimodal_data_length_50_skewed_gaussian</td>\n",
       "      <td>44.023</td>\n",
       "      <td>53.911938</td>\n",
       "      <td>53.553983</td>\n",
       "      <td>2400.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>hmm</td>\n",
       "      <td>72.625271</td>\n",
       "      <td>343.336584</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>44.302</td>\n",
       "      <td>73.375649</td>\n",
       "      <td>72.111776</td>\n",
       "      <td>800.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>hmm</td>\n",
       "      <td>65.433488</td>\n",
       "      <td>735.702525</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>43.899</td>\n",
       "      <td>66.122538</td>\n",
       "      <td>64.885167</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>hmm</td>\n",
       "      <td>59.212019</td>\n",
       "      <td>827.843845</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>44.350</td>\n",
       "      <td>60.062504</td>\n",
       "      <td>58.703601</td>\n",
       "      <td>3500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>hmm</td>\n",
       "      <td>96.584219</td>\n",
       "      <td>373.222957</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>44.821</td>\n",
       "      <td>97.024315</td>\n",
       "      <td>96.908910</td>\n",
       "      <td>800.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>hmm</td>\n",
       "      <td>86.712826</td>\n",
       "      <td>786.754198</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>44.475</td>\n",
       "      <td>87.126085</td>\n",
       "      <td>86.832014</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>hmm</td>\n",
       "      <td>107.822839</td>\n",
       "      <td>377.984134</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>44.457</td>\n",
       "      <td>108.401017</td>\n",
       "      <td>107.438812</td>\n",
       "      <td>800.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>hmm</td>\n",
       "      <td>95.682237</td>\n",
       "      <td>424.253978</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>43.969</td>\n",
       "      <td>96.637159</td>\n",
       "      <td>95.116697</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>hmm</td>\n",
       "      <td>90.428119</td>\n",
       "      <td>553.035205</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>43.739</td>\n",
       "      <td>91.054503</td>\n",
       "      <td>89.620096</td>\n",
       "      <td>2400.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>hmm</td>\n",
       "      <td>83.710733</td>\n",
       "      <td>964.705426</td>\n",
       "      <td>synthetic_multimodal_data_modes_3_length_51_un...</td>\n",
       "      <td>44.238</td>\n",
       "      <td>84.449199</td>\n",
       "      <td>82.838196</td>\n",
       "      <td>3500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>hmm</td>\n",
       "      <td>119.753795</td>\n",
       "      <td>367.962556</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>44.513</td>\n",
       "      <td>119.791103</td>\n",
       "      <td>118.794048</td>\n",
       "      <td>800.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>hmm</td>\n",
       "      <td>109.621701</td>\n",
       "      <td>780.603045</td>\n",
       "      <td>synthetic_multimodal_data_modes_5_length_51_un...</td>\n",
       "      <td>44.241</td>\n",
       "      <td>109.809288</td>\n",
       "      <td>108.936058</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   model_type  test_score  total_time  \\\n",
       "0         hmm  163.729458  672.123741   \n",
       "4         hmm  160.439516  584.543961   \n",
       "5         hmm  138.283641  670.506927   \n",
       "8         hmm  161.177640  246.149052   \n",
       "10        hmm  127.280070  899.934500   \n",
       "12        hmm   30.578423   93.825385   \n",
       "13        hmm   33.690124  318.300480   \n",
       "14        hmm   31.618671  533.252563   \n",
       "15        hmm   32.385205  377.175702   \n",
       "16        hmm   32.220333   69.040670   \n",
       "17        hmm   26.155596  151.421632   \n",
       "18        hmm   28.630247  177.973732   \n",
       "19        hmm   27.959026  349.081254   \n",
       "20        hmm   30.299214   95.458056   \n",
       "21        hmm   33.467269  103.940329   \n",
       "22        hmm   30.725757  522.775589   \n",
       "23        hmm   29.487527  812.683749   \n",
       "24        hmm   78.123778  149.141079   \n",
       "25        hmm   62.896197  193.520639   \n",
       "27        hmm   49.870443  607.648293   \n",
       "28        hmm   78.473292   90.185113   \n",
       "29        hmm   65.198984  193.754603   \n",
       "30        hmm   54.254814  481.134427   \n",
       "32        hmm   72.625271  343.336584   \n",
       "33        hmm   65.433488  735.702525   \n",
       "35        hmm   59.212019  827.843845   \n",
       "40        hmm   96.584219  373.222957   \n",
       "41        hmm   86.712826  786.754198   \n",
       "44        hmm  107.822839  377.984134   \n",
       "45        hmm   95.682237  424.253978   \n",
       "46        hmm   90.428119  553.035205   \n",
       "47        hmm   83.710733  964.705426   \n",
       "48        hmm  119.753795  367.962556   \n",
       "49        hmm  109.621701  780.603045   \n",
       "\n",
       "                                              dataset  average_mismatches  \\\n",
       "0         synthetic_unimodal_data_length_100_gaussian              90.335   \n",
       "4   synthetic_unimodal_data_length_100_skewed_gaus...              90.671   \n",
       "5   synthetic_unimodal_data_length_100_skewed_gaus...              90.365   \n",
       "8          synthetic_unimodal_data_length_100_uniform              90.686   \n",
       "10         synthetic_unimodal_data_length_100_uniform              89.869   \n",
       "12         synthetic_unimodal_data_length_20_gaussian              10.029   \n",
       "13         synthetic_unimodal_data_length_20_gaussian              13.709   \n",
       "14         synthetic_unimodal_data_length_20_gaussian              13.828   \n",
       "15         synthetic_unimodal_data_length_20_gaussian              14.633   \n",
       "16  synthetic_unimodal_data_length_20_skewed_gaussian              13.279   \n",
       "17  synthetic_unimodal_data_length_20_skewed_gaussian               8.430   \n",
       "18  synthetic_unimodal_data_length_20_skewed_gaussian              13.844   \n",
       "19  synthetic_unimodal_data_length_20_skewed_gaussian              14.153   \n",
       "20          synthetic_unimodal_data_length_20_uniform               9.982   \n",
       "21          synthetic_unimodal_data_length_20_uniform              14.048   \n",
       "22          synthetic_unimodal_data_length_20_uniform              10.843   \n",
       "23          synthetic_unimodal_data_length_20_uniform              10.597   \n",
       "24         synthetic_unimodal_data_length_50_gaussian              43.880   \n",
       "25         synthetic_unimodal_data_length_50_gaussian              42.508   \n",
       "27         synthetic_unimodal_data_length_50_gaussian              42.432   \n",
       "28  synthetic_unimodal_data_length_50_skewed_gaussian              44.292   \n",
       "29  synthetic_unimodal_data_length_50_skewed_gaussian              43.334   \n",
       "30  synthetic_unimodal_data_length_50_skewed_gaussian              44.023   \n",
       "32          synthetic_unimodal_data_length_50_uniform              44.302   \n",
       "33          synthetic_unimodal_data_length_50_uniform              43.899   \n",
       "35          synthetic_unimodal_data_length_50_uniform              44.350   \n",
       "40  synthetic_multimodal_data_modes_2_length_51_un...              44.821   \n",
       "41  synthetic_multimodal_data_modes_2_length_51_un...              44.475   \n",
       "44  synthetic_multimodal_data_modes_3_length_51_un...              44.457   \n",
       "45  synthetic_multimodal_data_modes_3_length_51_un...              43.969   \n",
       "46  synthetic_multimodal_data_modes_3_length_51_un...              43.739   \n",
       "47  synthetic_multimodal_data_modes_3_length_51_un...              44.238   \n",
       "48  synthetic_multimodal_data_modes_5_length_51_un...              44.513   \n",
       "49  synthetic_multimodal_data_modes_5_length_51_un...              44.241   \n",
       "\n",
       "    valid_score  train_score  num_params  exit_code                name  \n",
       "0    163.819790   163.462043       800.0        0.0  hmm_hidden_size_20  \n",
       "4    160.227315   160.183613       800.0        0.0  hmm_hidden_size_20  \n",
       "5    138.019667   137.851505      1500.0        0.0  hmm_hidden_size_30  \n",
       "8    161.168688   161.434284       800.0        0.0  hmm_hidden_size_20  \n",
       "10   127.157544   127.213086      2400.0        0.0  hmm_hidden_size_40  \n",
       "12    30.097017    30.222726       800.0        0.0  hmm_hidden_size_20  \n",
       "13    33.164068    33.360533      1500.0        0.0  hmm_hidden_size_30  \n",
       "14    30.967193    31.185973      2400.0        0.0  hmm_hidden_size_40  \n",
       "15    31.873334    32.050543      3500.0        0.0  hmm_hidden_size_50  \n",
       "16    32.293674    32.497731       800.0        0.0  hmm_hidden_size_20  \n",
       "17    26.205717    26.274437      1500.0        0.0  hmm_hidden_size_30  \n",
       "18    28.687373    28.775390      2400.0        0.0  hmm_hidden_size_40  \n",
       "19    28.036499    28.151720      3500.0        0.0  hmm_hidden_size_50  \n",
       "20    30.443394    30.153763       800.0        0.0  hmm_hidden_size_20  \n",
       "21    33.588177    33.340336      1500.0        0.0  hmm_hidden_size_30  \n",
       "22    30.879700    30.551452      2400.0        0.0  hmm_hidden_size_40  \n",
       "23    29.629142    29.331068      3500.0        0.0  hmm_hidden_size_50  \n",
       "24    78.726581    78.383688       800.0        0.0  hmm_hidden_size_20  \n",
       "25    63.241722    62.951611      1500.0        0.0  hmm_hidden_size_30  \n",
       "27    50.335326    49.897661      3500.0        0.0  hmm_hidden_size_50  \n",
       "28    78.136042    78.188168       800.0        0.0  hmm_hidden_size_20  \n",
       "29    64.927422    64.724603      1500.0        0.0  hmm_hidden_size_30  \n",
       "30    53.911938    53.553983      2400.0        0.0  hmm_hidden_size_40  \n",
       "32    73.375649    72.111776       800.0        0.0  hmm_hidden_size_20  \n",
       "33    66.122538    64.885167      1500.0        0.0  hmm_hidden_size_30  \n",
       "35    60.062504    58.703601      3500.0        0.0  hmm_hidden_size_50  \n",
       "40    97.024315    96.908910       800.0        0.0  hmm_hidden_size_20  \n",
       "41    87.126085    86.832014      1500.0        0.0  hmm_hidden_size_30  \n",
       "44   108.401017   107.438812       800.0        0.0  hmm_hidden_size_20  \n",
       "45    96.637159    95.116697      1500.0        0.0  hmm_hidden_size_30  \n",
       "46    91.054503    89.620096      2400.0        0.0  hmm_hidden_size_40  \n",
       "47    84.449199    82.838196      3500.0        0.0  hmm_hidden_size_50  \n",
       "48   119.791103   118.794048       800.0        0.0  hmm_hidden_size_20  \n",
       "49   109.809288   108.936058      1500.0        0.0  hmm_hidden_size_30  "
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hmm_df = hmm_df.iloc[hmm_keep_list, :]\n",
    "hmm_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiments = pd.concat([hmm_df, vae_df, rnn_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_type</th>\n",
       "      <th>test_score</th>\n",
       "      <th>total_time</th>\n",
       "      <th>dataset</th>\n",
       "      <th>average_mismatches</th>\n",
       "      <th>valid_score</th>\n",
       "      <th>train_score</th>\n",
       "      <th>num_params</th>\n",
       "      <th>exit_code</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>vae</td>\n",
       "      <td>25.018938</td>\n",
       "      <td>29.326605</td>\n",
       "      <td>gfp</td>\n",
       "      <td>3.791</td>\n",
       "      <td>24.741735</td>\n",
       "      <td>22.846964</td>\n",
       "      <td>2016638.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>vae</td>\n",
       "      <td>25.071152</td>\n",
       "      <td>32.220650</td>\n",
       "      <td>gfp</td>\n",
       "      <td>3.900</td>\n",
       "      <td>24.748516</td>\n",
       "      <td>22.717083</td>\n",
       "      <td>1010838.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>vae</td>\n",
       "      <td>25.114323</td>\n",
       "      <td>24.529130</td>\n",
       "      <td>gfp</td>\n",
       "      <td>3.786</td>\n",
       "      <td>24.866001</td>\n",
       "      <td>22.634626</td>\n",
       "      <td>1010838.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>vae</td>\n",
       "      <td>25.136610</td>\n",
       "      <td>25.838103</td>\n",
       "      <td>gfp</td>\n",
       "      <td>4.040</td>\n",
       "      <td>24.939550</td>\n",
       "      <td>22.832769</td>\n",
       "      <td>1010838.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>vae</td>\n",
       "      <td>25.325895</td>\n",
       "      <td>56.105991</td>\n",
       "      <td>gfp</td>\n",
       "      <td>3.961</td>\n",
       "      <td>24.929785</td>\n",
       "      <td>22.330002</td>\n",
       "      <td>2016638.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>vae</td>\n",
       "      <td>25.328659</td>\n",
       "      <td>41.292221</td>\n",
       "      <td>gfp</td>\n",
       "      <td>4.282</td>\n",
       "      <td>25.014442</td>\n",
       "      <td>22.320091</td>\n",
       "      <td>2016638.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>rnn</td>\n",
       "      <td>25.395535</td>\n",
       "      <td>1240.126062</td>\n",
       "      <td>gfp</td>\n",
       "      <td>17.808</td>\n",
       "      <td>24.985129</td>\n",
       "      <td>22.391420</td>\n",
       "      <td>330021.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>vae</td>\n",
       "      <td>25.430275</td>\n",
       "      <td>37.323750</td>\n",
       "      <td>gfp</td>\n",
       "      <td>7.390</td>\n",
       "      <td>25.168547</td>\n",
       "      <td>22.649450</td>\n",
       "      <td>1010838.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>vae</td>\n",
       "      <td>25.498753</td>\n",
       "      <td>28.973421</td>\n",
       "      <td>gfp</td>\n",
       "      <td>3.809</td>\n",
       "      <td>25.301153</td>\n",
       "      <td>22.404305</td>\n",
       "      <td>2016638.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>vae</td>\n",
       "      <td>25.564478</td>\n",
       "      <td>25.492081</td>\n",
       "      <td>gfp</td>\n",
       "      <td>3.635</td>\n",
       "      <td>25.189240</td>\n",
       "      <td>23.138323</td>\n",
       "      <td>2016638.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>vae</td>\n",
       "      <td>25.610201</td>\n",
       "      <td>60.120387</td>\n",
       "      <td>gfp</td>\n",
       "      <td>3.756</td>\n",
       "      <td>25.210026</td>\n",
       "      <td>22.415464</td>\n",
       "      <td>1010838.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>vae</td>\n",
       "      <td>26.141891</td>\n",
       "      <td>54.419998</td>\n",
       "      <td>gfp</td>\n",
       "      <td>34.967</td>\n",
       "      <td>25.842421</td>\n",
       "      <td>23.402271</td>\n",
       "      <td>2016638.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>vae</td>\n",
       "      <td>27.981173</td>\n",
       "      <td>41.538969</td>\n",
       "      <td>gfp</td>\n",
       "      <td>86.056</td>\n",
       "      <td>27.435287</td>\n",
       "      <td>24.981028</td>\n",
       "      <td>1010838.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>rnn</td>\n",
       "      <td>35.462898</td>\n",
       "      <td>377.938907</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>10.723</td>\n",
       "      <td>36.114004</td>\n",
       "      <td>32.871695</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>rnn</td>\n",
       "      <td>35.541008</td>\n",
       "      <td>157.289691</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>10.317</td>\n",
       "      <td>36.125641</td>\n",
       "      <td>33.224434</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>rnn</td>\n",
       "      <td>35.605438</td>\n",
       "      <td>216.761894</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>10.835</td>\n",
       "      <td>36.176953</td>\n",
       "      <td>33.747047</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>rnn</td>\n",
       "      <td>35.631988</td>\n",
       "      <td>169.658693</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>8.364</td>\n",
       "      <td>36.228512</td>\n",
       "      <td>32.911082</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>rnn</td>\n",
       "      <td>35.728023</td>\n",
       "      <td>697.383602</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>12.545</td>\n",
       "      <td>36.413187</td>\n",
       "      <td>33.260758</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>rnn</td>\n",
       "      <td>35.826004</td>\n",
       "      <td>127.044826</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>9.040</td>\n",
       "      <td>36.477477</td>\n",
       "      <td>33.473340</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>rnn</td>\n",
       "      <td>35.834684</td>\n",
       "      <td>605.084743</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>11.174</td>\n",
       "      <td>36.499324</td>\n",
       "      <td>33.501305</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>rnn</td>\n",
       "      <td>36.169508</td>\n",
       "      <td>1169.020653</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>13.262</td>\n",
       "      <td>36.781086</td>\n",
       "      <td>33.881211</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.270718</td>\n",
       "      <td>22.198293</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>13.910</td>\n",
       "      <td>37.238293</td>\n",
       "      <td>34.164478</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.376476</td>\n",
       "      <td>16.107120</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>13.019</td>\n",
       "      <td>37.269409</td>\n",
       "      <td>33.810802</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.386944</td>\n",
       "      <td>13.026769</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>12.928</td>\n",
       "      <td>37.277016</td>\n",
       "      <td>34.594135</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.456425</td>\n",
       "      <td>26.322335</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>14.730</td>\n",
       "      <td>37.437649</td>\n",
       "      <td>34.789349</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.495571</td>\n",
       "      <td>10.940336</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>12.696</td>\n",
       "      <td>37.507469</td>\n",
       "      <td>34.015968</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.646713</td>\n",
       "      <td>30.016130</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>16.003</td>\n",
       "      <td>37.476822</td>\n",
       "      <td>34.983650</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.660088</td>\n",
       "      <td>19.990564</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>14.140</td>\n",
       "      <td>37.430695</td>\n",
       "      <td>34.740582</td>\n",
       "      <td>211260.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.813862</td>\n",
       "      <td>16.094194</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>14.114</td>\n",
       "      <td>37.531429</td>\n",
       "      <td>34.111122</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>vae</td>\n",
       "      <td>36.820236</td>\n",
       "      <td>9.214412</td>\n",
       "      <td>synthetic_multimodal_data_modes_2_length_51_un...</td>\n",
       "      <td>14.121</td>\n",
       "      <td>37.639351</td>\n",
       "      <td>35.006939</td>\n",
       "      <td>421460.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>252</th>\n",
       "      <td>vae</td>\n",
       "      <td>32.347607</td>\n",
       "      <td>11.491580</td>\n",
       "      <td>synthetic_unimodal_data_length_50_skewed_gaussian</td>\n",
       "      <td>5.299</td>\n",
       "      <td>31.985619</td>\n",
       "      <td>31.019852</td>\n",
       "      <td>207240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253</th>\n",
       "      <td>rnn</td>\n",
       "      <td>32.441963</td>\n",
       "      <td>126.943657</td>\n",
       "      <td>synthetic_unimodal_data_length_50_skewed_gaussian</td>\n",
       "      <td>6.098</td>\n",
       "      <td>32.060486</td>\n",
       "      <td>30.509834</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>vae</td>\n",
       "      <td>32.539562</td>\n",
       "      <td>9.666118</td>\n",
       "      <td>synthetic_unimodal_data_length_50_skewed_gaussian</td>\n",
       "      <td>5.209</td>\n",
       "      <td>32.208920</td>\n",
       "      <td>31.005478</td>\n",
       "      <td>413440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>vae</td>\n",
       "      <td>32.808731</td>\n",
       "      <td>12.305997</td>\n",
       "      <td>synthetic_unimodal_data_length_50_skewed_gaussian</td>\n",
       "      <td>5.380</td>\n",
       "      <td>32.431664</td>\n",
       "      <td>31.103584</td>\n",
       "      <td>413440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256</th>\n",
       "      <td>hmm</td>\n",
       "      <td>54.254814</td>\n",
       "      <td>481.134427</td>\n",
       "      <td>synthetic_unimodal_data_length_50_skewed_gaussian</td>\n",
       "      <td>44.023</td>\n",
       "      <td>53.911938</td>\n",
       "      <td>53.553983</td>\n",
       "      <td>2400.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257</th>\n",
       "      <td>hmm</td>\n",
       "      <td>65.198984</td>\n",
       "      <td>193.754603</td>\n",
       "      <td>synthetic_unimodal_data_length_50_skewed_gaussian</td>\n",
       "      <td>43.334</td>\n",
       "      <td>64.927422</td>\n",
       "      <td>64.724603</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258</th>\n",
       "      <td>hmm</td>\n",
       "      <td>78.473292</td>\n",
       "      <td>90.185113</td>\n",
       "      <td>synthetic_unimodal_data_length_50_skewed_gaussian</td>\n",
       "      <td>44.292</td>\n",
       "      <td>78.136042</td>\n",
       "      <td>78.188168</td>\n",
       "      <td>800.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>259</th>\n",
       "      <td>vae</td>\n",
       "      <td>33.399937</td>\n",
       "      <td>36.304336</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>5.535</td>\n",
       "      <td>34.326412</td>\n",
       "      <td>32.038525</td>\n",
       "      <td>207240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260</th>\n",
       "      <td>vae</td>\n",
       "      <td>33.415684</td>\n",
       "      <td>26.502703</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>5.298</td>\n",
       "      <td>34.286011</td>\n",
       "      <td>32.178432</td>\n",
       "      <td>207240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261</th>\n",
       "      <td>vae</td>\n",
       "      <td>33.451778</td>\n",
       "      <td>20.352989</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>5.534</td>\n",
       "      <td>34.397305</td>\n",
       "      <td>32.104565</td>\n",
       "      <td>413440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>vae</td>\n",
       "      <td>33.488804</td>\n",
       "      <td>22.199394</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>6.000</td>\n",
       "      <td>34.455612</td>\n",
       "      <td>32.340164</td>\n",
       "      <td>413440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263</th>\n",
       "      <td>vae</td>\n",
       "      <td>33.513599</td>\n",
       "      <td>12.558914</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>5.578</td>\n",
       "      <td>34.430592</td>\n",
       "      <td>32.189384</td>\n",
       "      <td>207240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>vae</td>\n",
       "      <td>33.588066</td>\n",
       "      <td>16.039454</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>5.353</td>\n",
       "      <td>34.446808</td>\n",
       "      <td>32.343678</td>\n",
       "      <td>207240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>265</th>\n",
       "      <td>vae</td>\n",
       "      <td>33.637699</td>\n",
       "      <td>11.745106</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>5.454</td>\n",
       "      <td>34.589878</td>\n",
       "      <td>31.977812</td>\n",
       "      <td>413440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>rnn</td>\n",
       "      <td>33.704949</td>\n",
       "      <td>199.372288</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>7.253</td>\n",
       "      <td>34.665965</td>\n",
       "      <td>32.063582</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>rnn</td>\n",
       "      <td>33.726180</td>\n",
       "      <td>623.180200</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>7.690</td>\n",
       "      <td>34.659250</td>\n",
       "      <td>31.641914</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268</th>\n",
       "      <td>vae</td>\n",
       "      <td>33.754108</td>\n",
       "      <td>10.352012</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>5.369</td>\n",
       "      <td>34.625782</td>\n",
       "      <td>32.618462</td>\n",
       "      <td>207240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269</th>\n",
       "      <td>rnn</td>\n",
       "      <td>33.768129</td>\n",
       "      <td>242.906807</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>7.514</td>\n",
       "      <td>34.661430</td>\n",
       "      <td>32.204236</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270</th>\n",
       "      <td>rnn</td>\n",
       "      <td>33.769184</td>\n",
       "      <td>373.934419</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>7.337</td>\n",
       "      <td>34.773758</td>\n",
       "      <td>32.046834</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>271</th>\n",
       "      <td>vae</td>\n",
       "      <td>33.774891</td>\n",
       "      <td>14.363359</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>5.622</td>\n",
       "      <td>34.664455</td>\n",
       "      <td>32.137012</td>\n",
       "      <td>413440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272</th>\n",
       "      <td>rnn</td>\n",
       "      <td>33.787645</td>\n",
       "      <td>1081.808056</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>7.981</td>\n",
       "      <td>34.746117</td>\n",
       "      <td>32.035871</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.0001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>273</th>\n",
       "      <td>rnn</td>\n",
       "      <td>33.812152</td>\n",
       "      <td>154.081646</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>7.076</td>\n",
       "      <td>34.763008</td>\n",
       "      <td>32.069621</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>274</th>\n",
       "      <td>rnn</td>\n",
       "      <td>33.812191</td>\n",
       "      <td>145.959079</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>7.078</td>\n",
       "      <td>34.794715</td>\n",
       "      <td>31.901131</td>\n",
       "      <td>84820.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_100_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>275</th>\n",
       "      <td>vae</td>\n",
       "      <td>33.946857</td>\n",
       "      <td>10.882909</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>5.720</td>\n",
       "      <td>34.760155</td>\n",
       "      <td>32.825520</td>\n",
       "      <td>207240.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_100_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>276</th>\n",
       "      <td>rnn</td>\n",
       "      <td>33.981641</td>\n",
       "      <td>124.240005</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>6.890</td>\n",
       "      <td>34.927422</td>\n",
       "      <td>31.818404</td>\n",
       "      <td>329620.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>rnn_hidden_size_200_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>277</th>\n",
       "      <td>vae</td>\n",
       "      <td>34.295147</td>\n",
       "      <td>9.448185</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>5.574</td>\n",
       "      <td>35.126652</td>\n",
       "      <td>31.995180</td>\n",
       "      <td>413440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_20_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278</th>\n",
       "      <td>vae</td>\n",
       "      <td>34.603245</td>\n",
       "      <td>11.136064</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>5.304</td>\n",
       "      <td>35.481850</td>\n",
       "      <td>33.083820</td>\n",
       "      <td>413440.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>vae_hidden_size_200_batch_size_10_lr_0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279</th>\n",
       "      <td>hmm</td>\n",
       "      <td>59.212019</td>\n",
       "      <td>827.843845</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>44.350</td>\n",
       "      <td>60.062504</td>\n",
       "      <td>58.703601</td>\n",
       "      <td>3500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280</th>\n",
       "      <td>hmm</td>\n",
       "      <td>65.433488</td>\n",
       "      <td>735.702525</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>43.899</td>\n",
       "      <td>66.122538</td>\n",
       "      <td>64.885167</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>hmm</td>\n",
       "      <td>72.625271</td>\n",
       "      <td>343.336584</td>\n",
       "      <td>synthetic_unimodal_data_length_50_uniform</td>\n",
       "      <td>44.302</td>\n",
       "      <td>73.375649</td>\n",
       "      <td>72.111776</td>\n",
       "      <td>800.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hmm_hidden_size_20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>282 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    model_type  test_score   total_time  \\\n",
       "0          vae   25.018938    29.326605   \n",
       "1          vae   25.071152    32.220650   \n",
       "2          vae   25.114323    24.529130   \n",
       "3          vae   25.136610    25.838103   \n",
       "4          vae   25.325895    56.105991   \n",
       "5          vae   25.328659    41.292221   \n",
       "6          rnn   25.395535  1240.126062   \n",
       "7          vae   25.430275    37.323750   \n",
       "8          vae   25.498753    28.973421   \n",
       "9          vae   25.564478    25.492081   \n",
       "10         vae   25.610201    60.120387   \n",
       "11         vae   26.141891    54.419998   \n",
       "12         vae   27.981173    41.538969   \n",
       "13         rnn   35.462898   377.938907   \n",
       "14         rnn   35.541008   157.289691   \n",
       "15         rnn   35.605438   216.761894   \n",
       "16         rnn   35.631988   169.658693   \n",
       "17         rnn   35.728023   697.383602   \n",
       "18         rnn   35.826004   127.044826   \n",
       "19         rnn   35.834684   605.084743   \n",
       "20         rnn   36.169508  1169.020653   \n",
       "21         vae   36.270718    22.198293   \n",
       "22         vae   36.376476    16.107120   \n",
       "23         vae   36.386944    13.026769   \n",
       "24         vae   36.456425    26.322335   \n",
       "25         vae   36.495571    10.940336   \n",
       "26         vae   36.646713    30.016130   \n",
       "27         vae   36.660088    19.990564   \n",
       "28         vae   36.813862    16.094194   \n",
       "29         vae   36.820236     9.214412   \n",
       "..         ...         ...          ...   \n",
       "252        vae   32.347607    11.491580   \n",
       "253        rnn   32.441963   126.943657   \n",
       "254        vae   32.539562     9.666118   \n",
       "255        vae   32.808731    12.305997   \n",
       "256        hmm   54.254814   481.134427   \n",
       "257        hmm   65.198984   193.754603   \n",
       "258        hmm   78.473292    90.185113   \n",
       "259        vae   33.399937    36.304336   \n",
       "260        vae   33.415684    26.502703   \n",
       "261        vae   33.451778    20.352989   \n",
       "262        vae   33.488804    22.199394   \n",
       "263        vae   33.513599    12.558914   \n",
       "264        vae   33.588066    16.039454   \n",
       "265        vae   33.637699    11.745106   \n",
       "266        rnn   33.704949   199.372288   \n",
       "267        rnn   33.726180   623.180200   \n",
       "268        vae   33.754108    10.352012   \n",
       "269        rnn   33.768129   242.906807   \n",
       "270        rnn   33.769184   373.934419   \n",
       "271        vae   33.774891    14.363359   \n",
       "272        rnn   33.787645  1081.808056   \n",
       "273        rnn   33.812152   154.081646   \n",
       "274        rnn   33.812191   145.959079   \n",
       "275        vae   33.946857    10.882909   \n",
       "276        rnn   33.981641   124.240005   \n",
       "277        vae   34.295147     9.448185   \n",
       "278        vae   34.603245    11.136064   \n",
       "279        hmm   59.212019   827.843845   \n",
       "280        hmm   65.433488   735.702525   \n",
       "281        hmm   72.625271   343.336584   \n",
       "\n",
       "                                               dataset  average_mismatches  \\\n",
       "0                                                  gfp               3.791   \n",
       "1                                                  gfp               3.900   \n",
       "2                                                  gfp               3.786   \n",
       "3                                                  gfp               4.040   \n",
       "4                                                  gfp               3.961   \n",
       "5                                                  gfp               4.282   \n",
       "6                                                  gfp              17.808   \n",
       "7                                                  gfp               7.390   \n",
       "8                                                  gfp               3.809   \n",
       "9                                                  gfp               3.635   \n",
       "10                                                 gfp               3.756   \n",
       "11                                                 gfp              34.967   \n",
       "12                                                 gfp              86.056   \n",
       "13   synthetic_multimodal_data_modes_2_length_51_un...              10.723   \n",
       "14   synthetic_multimodal_data_modes_2_length_51_un...              10.317   \n",
       "15   synthetic_multimodal_data_modes_2_length_51_un...              10.835   \n",
       "16   synthetic_multimodal_data_modes_2_length_51_un...               8.364   \n",
       "17   synthetic_multimodal_data_modes_2_length_51_un...              12.545   \n",
       "18   synthetic_multimodal_data_modes_2_length_51_un...               9.040   \n",
       "19   synthetic_multimodal_data_modes_2_length_51_un...              11.174   \n",
       "20   synthetic_multimodal_data_modes_2_length_51_un...              13.262   \n",
       "21   synthetic_multimodal_data_modes_2_length_51_un...              13.910   \n",
       "22   synthetic_multimodal_data_modes_2_length_51_un...              13.019   \n",
       "23   synthetic_multimodal_data_modes_2_length_51_un...              12.928   \n",
       "24   synthetic_multimodal_data_modes_2_length_51_un...              14.730   \n",
       "25   synthetic_multimodal_data_modes_2_length_51_un...              12.696   \n",
       "26   synthetic_multimodal_data_modes_2_length_51_un...              16.003   \n",
       "27   synthetic_multimodal_data_modes_2_length_51_un...              14.140   \n",
       "28   synthetic_multimodal_data_modes_2_length_51_un...              14.114   \n",
       "29   synthetic_multimodal_data_modes_2_length_51_un...              14.121   \n",
       "..                                                 ...                 ...   \n",
       "252  synthetic_unimodal_data_length_50_skewed_gaussian               5.299   \n",
       "253  synthetic_unimodal_data_length_50_skewed_gaussian               6.098   \n",
       "254  synthetic_unimodal_data_length_50_skewed_gaussian               5.209   \n",
       "255  synthetic_unimodal_data_length_50_skewed_gaussian               5.380   \n",
       "256  synthetic_unimodal_data_length_50_skewed_gaussian              44.023   \n",
       "257  synthetic_unimodal_data_length_50_skewed_gaussian              43.334   \n",
       "258  synthetic_unimodal_data_length_50_skewed_gaussian              44.292   \n",
       "259          synthetic_unimodal_data_length_50_uniform               5.535   \n",
       "260          synthetic_unimodal_data_length_50_uniform               5.298   \n",
       "261          synthetic_unimodal_data_length_50_uniform               5.534   \n",
       "262          synthetic_unimodal_data_length_50_uniform               6.000   \n",
       "263          synthetic_unimodal_data_length_50_uniform               5.578   \n",
       "264          synthetic_unimodal_data_length_50_uniform               5.353   \n",
       "265          synthetic_unimodal_data_length_50_uniform               5.454   \n",
       "266          synthetic_unimodal_data_length_50_uniform               7.253   \n",
       "267          synthetic_unimodal_data_length_50_uniform               7.690   \n",
       "268          synthetic_unimodal_data_length_50_uniform               5.369   \n",
       "269          synthetic_unimodal_data_length_50_uniform               7.514   \n",
       "270          synthetic_unimodal_data_length_50_uniform               7.337   \n",
       "271          synthetic_unimodal_data_length_50_uniform               5.622   \n",
       "272          synthetic_unimodal_data_length_50_uniform               7.981   \n",
       "273          synthetic_unimodal_data_length_50_uniform               7.076   \n",
       "274          synthetic_unimodal_data_length_50_uniform               7.078   \n",
       "275          synthetic_unimodal_data_length_50_uniform               5.720   \n",
       "276          synthetic_unimodal_data_length_50_uniform               6.890   \n",
       "277          synthetic_unimodal_data_length_50_uniform               5.574   \n",
       "278          synthetic_unimodal_data_length_50_uniform               5.304   \n",
       "279          synthetic_unimodal_data_length_50_uniform              44.350   \n",
       "280          synthetic_unimodal_data_length_50_uniform              43.899   \n",
       "281          synthetic_unimodal_data_length_50_uniform              44.302   \n",
       "\n",
       "     valid_score  train_score  num_params  exit_code  \\\n",
       "0      24.741735    22.846964   2016638.0        0.0   \n",
       "1      24.748516    22.717083   1010838.0        0.0   \n",
       "2      24.866001    22.634626   1010838.0        0.0   \n",
       "3      24.939550    22.832769   1010838.0        0.0   \n",
       "4      24.929785    22.330002   2016638.0        0.0   \n",
       "5      25.014442    22.320091   2016638.0        0.0   \n",
       "6      24.985129    22.391420    330021.0        0.0   \n",
       "7      25.168547    22.649450   1010838.0        0.0   \n",
       "8      25.301153    22.404305   2016638.0        0.0   \n",
       "9      25.189240    23.138323   2016638.0        0.0   \n",
       "10     25.210026    22.415464   1010838.0        0.0   \n",
       "11     25.842421    23.402271   2016638.0        0.0   \n",
       "12     27.435287    24.981028   1010838.0        0.0   \n",
       "13     36.114004    32.871695    329620.0        0.0   \n",
       "14     36.125641    33.224434    329620.0        0.0   \n",
       "15     36.176953    33.747047     84820.0        0.0   \n",
       "16     36.228512    32.911082     84820.0        0.0   \n",
       "17     36.413187    33.260758    329620.0        0.0   \n",
       "18     36.477477    33.473340    329620.0        0.0   \n",
       "19     36.499324    33.501305     84820.0        0.0   \n",
       "20     36.781086    33.881211     84820.0        0.0   \n",
       "21     37.238293    34.164478    421460.0        0.0   \n",
       "22     37.269409    33.810802    211260.0        0.0   \n",
       "23     37.277016    34.594135    211260.0        0.0   \n",
       "24     37.437649    34.789349    211260.0        0.0   \n",
       "25     37.507469    34.015968    211260.0        0.0   \n",
       "26     37.476822    34.983650    211260.0        0.0   \n",
       "27     37.430695    34.740582    211260.0        0.0   \n",
       "28     37.531429    34.111122    421460.0        0.0   \n",
       "29     37.639351    35.006939    421460.0        0.0   \n",
       "..           ...          ...         ...        ...   \n",
       "252    31.985619    31.019852    207240.0        0.0   \n",
       "253    32.060486    30.509834    329620.0        0.0   \n",
       "254    32.208920    31.005478    413440.0        0.0   \n",
       "255    32.431664    31.103584    413440.0        0.0   \n",
       "256    53.911938    53.553983      2400.0        0.0   \n",
       "257    64.927422    64.724603      1500.0        0.0   \n",
       "258    78.136042    78.188168       800.0        0.0   \n",
       "259    34.326412    32.038525    207240.0        0.0   \n",
       "260    34.286011    32.178432    207240.0        0.0   \n",
       "261    34.397305    32.104565    413440.0        0.0   \n",
       "262    34.455612    32.340164    413440.0        0.0   \n",
       "263    34.430592    32.189384    207240.0        0.0   \n",
       "264    34.446808    32.343678    207240.0        0.0   \n",
       "265    34.589878    31.977812    413440.0        0.0   \n",
       "266    34.665965    32.063582     84820.0        0.0   \n",
       "267    34.659250    31.641914    329620.0        0.0   \n",
       "268    34.625782    32.618462    207240.0        0.0   \n",
       "269    34.661430    32.204236    329620.0        0.0   \n",
       "270    34.773758    32.046834     84820.0        0.0   \n",
       "271    34.664455    32.137012    413440.0        0.0   \n",
       "272    34.746117    32.035871     84820.0        0.0   \n",
       "273    34.763008    32.069621    329620.0        0.0   \n",
       "274    34.794715    31.901131     84820.0        0.0   \n",
       "275    34.760155    32.825520    207240.0        0.0   \n",
       "276    34.927422    31.818404    329620.0        0.0   \n",
       "277    35.126652    31.995180    413440.0        0.0   \n",
       "278    35.481850    33.083820    413440.0        0.0   \n",
       "279    60.062504    58.703601      3500.0        0.0   \n",
       "280    66.122538    64.885167      1500.0        0.0   \n",
       "281    73.375649    72.111776       800.0        0.0   \n",
       "\n",
       "                                            name  \n",
       "0     vae_hidden_size_200_batch_size_10_lr_0.001  \n",
       "1     vae_hidden_size_100_batch_size_10_lr_0.001  \n",
       "2     vae_hidden_size_100_batch_size_20_lr_0.003  \n",
       "3     vae_hidden_size_100_batch_size_10_lr_0.003  \n",
       "4    vae_hidden_size_200_batch_size_10_lr_0.0003  \n",
       "5     vae_hidden_size_200_batch_size_20_lr_0.001  \n",
       "6                   rnn_hidden_size_200_lr_0.003  \n",
       "7     vae_hidden_size_100_batch_size_20_lr_0.001  \n",
       "8     vae_hidden_size_200_batch_size_20_lr_0.003  \n",
       "9     vae_hidden_size_200_batch_size_10_lr_0.003  \n",
       "10   vae_hidden_size_100_batch_size_10_lr_0.0003  \n",
       "11   vae_hidden_size_200_batch_size_20_lr_0.0003  \n",
       "12   vae_hidden_size_100_batch_size_20_lr_0.0003  \n",
       "13                 rnn_hidden_size_200_lr_0.0003  \n",
       "14                  rnn_hidden_size_200_lr_0.001  \n",
       "15                  rnn_hidden_size_100_lr_0.001  \n",
       "16                  rnn_hidden_size_100_lr_0.003  \n",
       "17                 rnn_hidden_size_200_lr_0.0001  \n",
       "18                  rnn_hidden_size_200_lr_0.003  \n",
       "19                 rnn_hidden_size_100_lr_0.0003  \n",
       "20                 rnn_hidden_size_100_lr_0.0001  \n",
       "21   vae_hidden_size_200_batch_size_20_lr_0.0003  \n",
       "22    vae_hidden_size_100_batch_size_20_lr_0.001  \n",
       "23    vae_hidden_size_100_batch_size_10_lr_0.003  \n",
       "24   vae_hidden_size_100_batch_size_20_lr_0.0003  \n",
       "25    vae_hidden_size_100_batch_size_20_lr_0.003  \n",
       "26   vae_hidden_size_100_batch_size_10_lr_0.0003  \n",
       "27    vae_hidden_size_100_batch_size_10_lr_0.001  \n",
       "28    vae_hidden_size_200_batch_size_10_lr_0.001  \n",
       "29    vae_hidden_size_200_batch_size_20_lr_0.003  \n",
       "..                                           ...  \n",
       "252   vae_hidden_size_100_batch_size_10_lr_0.003  \n",
       "253                 rnn_hidden_size_200_lr_0.003  \n",
       "254   vae_hidden_size_200_batch_size_20_lr_0.003  \n",
       "255   vae_hidden_size_200_batch_size_10_lr_0.003  \n",
       "256                           hmm_hidden_size_40  \n",
       "257                           hmm_hidden_size_30  \n",
       "258                           hmm_hidden_size_20  \n",
       "259  vae_hidden_size_100_batch_size_10_lr_0.0003  \n",
       "260  vae_hidden_size_100_batch_size_20_lr_0.0003  \n",
       "261  vae_hidden_size_200_batch_size_20_lr_0.0003  \n",
       "262  vae_hidden_size_200_batch_size_10_lr_0.0003  \n",
       "263   vae_hidden_size_100_batch_size_20_lr_0.001  \n",
       "264   vae_hidden_size_100_batch_size_10_lr_0.001  \n",
       "265   vae_hidden_size_200_batch_size_20_lr_0.001  \n",
       "266                 rnn_hidden_size_100_lr_0.001  \n",
       "267                rnn_hidden_size_200_lr_0.0001  \n",
       "268   vae_hidden_size_100_batch_size_20_lr_0.003  \n",
       "269                rnn_hidden_size_200_lr_0.0003  \n",
       "270                rnn_hidden_size_100_lr_0.0003  \n",
       "271   vae_hidden_size_200_batch_size_10_lr_0.001  \n",
       "272                rnn_hidden_size_100_lr_0.0001  \n",
       "273                 rnn_hidden_size_200_lr_0.001  \n",
       "274                 rnn_hidden_size_100_lr_0.003  \n",
       "275   vae_hidden_size_100_batch_size_10_lr_0.003  \n",
       "276                 rnn_hidden_size_200_lr_0.003  \n",
       "277   vae_hidden_size_200_batch_size_20_lr_0.003  \n",
       "278   vae_hidden_size_200_batch_size_10_lr_0.003  \n",
       "279                           hmm_hidden_size_50  \n",
       "280                           hmm_hidden_size_30  \n",
       "281                           hmm_hidden_size_20  \n",
       "\n",
       "[282 rows x 10 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "experiments = experiments.sort_values(by=[\"dataset\", \"test_score\"], axis = 0).reset_index(drop=True)\n",
    "experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiments.to_csv(\"./logs/optimized_results/grid_search_results.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "wild_type = get_wild_type_amino_acid_sequence().lower()\n",
    "sequence_lst = pd.read_csv(\"./data/gfp_data.csv\")[\"aaSequence\"].values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.7105123508567712\n"
     ]
    }
   ],
   "source": [
    "print(np.mean([count_substring_mismatch(seq, wild_type) for seq in sequence_lst]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEWCAYAAACufwpNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHBVJREFUeJzt3XmYZXV95/H3h1VlkUZaBLqxHUQSIC5IgESjjAuyJDZxXEelQSKuERMnEeNMMC7z4GOMkWCMqAhEIxINQgSDPYTRRAVZRHZChyU0NmuzKSMG+c4f51dyKKq6bzen+lbB+/U856lzfmf7nnur7qfOcs9JVSFJ0hDWG3cBkqRHD0NFkjQYQ0WSNBhDRZI0GENFkjQYQ0WSNBhDRWskyduS3JzkJ0metIbzfiDJF2eqNg0jyW8luWrcdaxKkkry9HHXoYczVDSyJBsCfwHsU1WbVtXt465Jw6uqf6mqndblOpPsnWT5ulynZoahojWxNfA44LJxFyJpdjJU9BBJdkvywyT3JPn7JF9J8uEkzwAmDoncmeSf2/SV5F1JrklyW5KPJVnV79Xj2jLvSXJhkmf11n1Ekn9v4y5P8ru9cU9P8u0kd7X1fKU37leSLE2yMslVSV69iu07uNV6T5Jrk7y+N+5NSa5IckeSM5M8tTfupUmubOs/ptXye23cQw7rJVnUXpcN2vATk3w+yYokN7bXc/1ePf+a5M/beq9Nsl9vWVsm+UKSH7fxX++N++0kFyW5M8n3kjxzFdtdSd6e5Oq27R9KskOb7+4kJyfZqE37kL2GJO9tdd/TXt8X97b775N8sY27JMkzkrwvyS1JbkiyT285h7TX9572HryltW8CfBPYth1W/UmSbZOsn+RPer8TFyRZ2Nusl7TtuTPJp5Jkde9lOp9o9d3dat51utdNa6Gq7OyoKoCNgOuBw4ENgVcAPwc+3MYvAgrYoDdPAWcDWwLbA/8G/N40y/8A8J/AK9vy/wdwLbBhG/8qYFu6f3ZeA/wU2KaN+zLw/jbuccDzW/smwA3AIcAGwHOA24Cdp1j/JsDdwE5teBtgl9a/GFgG/Gpbzv8EvtfGbQXc06v7D4D7J7azbdcXe+t5yOsEnAJ8pq3/ycAPgLe0cQe31+TNwPrA24AfA2njTwe+Asxr635ha38OcAuwZ5tvCXAdsPE0r30BpwKbA7sA9wFnAf8FeCJwObCkTbs3sLz179Re321727ZDb7t/BrysvWYntvfz/a3WNwPX9mo4ANgBCPBC4F5gt8nr7E3/R8AlrYYAzwKe1NuebwBb0P3e3QrsO8J7+TLggjZf2jTbjPtv79HUjb0Au9nTAS8Abpz4QGtt/8rqQ2Xf3vDbgbOmWf4HgHN6w+sBK4Dfmmb6i4DFrf9E4FhgwaRpXgP8y6S2zwBHTrG8TYA7gf8GPH7SuG8Ch06q7V7gqcBBk+oOsJwRQoXukOF9/fUBrwPObv0HA8t6457Q5n0KXeg9AMybYls+DXxoUttVtNCZYvoCntcbvgB4b2/448Bftv69eTBUnk4XXi+hhf+k93Npb/h3gJ8A67fhzdp6t5impq8Dh09e56TtWbyK7Xl+b/hk4IgR3ssX0f3jsxew3rj/5h6NnYe/1LctcGO1v8TmhhHm609zfVvOaqetqgfoPpy3BUhyUO9wzp3ArnR7CQB/TPdh/oMklyV5U2t/KrDnxDxtvtfTfSg/RFX9lC6E3gqsSHJ6kl/pLeeTvWWsbOvbrtXXr7sY7XWZWO6GbX0Ty/4M3R7LhJt6y7639W4KLARWVtUd0yz3PZO2eyGrfu1v7vX/vymGN508Q1UtA95NFyC3JDkpSX8dk5dxW1X9ojc8sS0k2S/JOekOU94J7M+D7+9UFgL/vorxN/X67+3VP+17WVX/DBwDfKptz7FJNl/FOrSGDBX1rQC26x+bpvvDXp3+NNvTHb5Z7bTpzr0sAH7cjnl/Fngn3SGOLYBL6T4MqKqbqurNVbUt8Bbgr9NdUnoD8O2q2qLXbVpVb5tq5VV1ZlW9lG4v4Mq2Ttpy3jJpOY+vqu+116VfdyZt80/p9jAm9APtBro9la16y928qnZZxWvUn3fLJFtMM+4jk+p9QlV9eYTlrpGq+ruqej7dh3UBH13TZSTZGPga8OfA1u39PYP2/rblTnYD3eGyNbWq95KqOrqqngvsDDyD7jCbBmKoqO/7wC+AdybZIMliYI8R5vujJPPaSdTD6c4BTOe5SV7RTmK/m+4D9xy6Q1NFd2ycJIfQ7anQhl+VZEEbvKNN+wDdcfVnJHljkg1b9+tJfnXyipNsnWRxOzF8H92hmgfa6L8B3pdklzbtE5O8qo07HdilV/e7eGhwXAS8IMn2SZ4IvG9iRFWtAL4FfDzJ5knWayfIX7iK16g/7zfpAnRe27YXtNGfBd6aZM928nmTJAck2Wx1y10TSXZK8qIWCj+j2/t4YDWzTWUjYGO69/f+dBcj7NMbfzPwpPb6Tfgc8KEkO7ZtfGZG+27UtO9l+93YM93l8T9t27Q226NpGCr6par6Od3J+UPpzj28ge5D+77VzHoq3TH6i+g+gD+/mmlfQxcMbwReUVX/WVWX0x3X/z7dB8yvAd/tzffrwLlJfgKcRncs/pqquofuw+m1dHtIN9H9J73xFOteD/jDNt1KupPFb2vbfkqb76Qkd9PtJe3Xxt1GdxHBUcDtwI792qpqKV2QXtxeh29MWu9BdB+ql7ft/irdntIo3kh3Iv9KunMb727rPJ/uRPgxbZnL6M7PDG1juu2+je61fTK90BxVe5/eRXfu4w7gv9O9jxPjr6S7GOOadthqW7rvRJ1MF8p30/1ePX6EdU37XtJdqPDZVsP1dO/nx9Z0ezS9iStMpCklORf4m6r6wjTjC9ixHXt/zEjyf+lOzn9u3LVIs4l7KnqIJC9M8pR2+GsJ8Ezgn8Zdl6S5YYNxF6BZZye6Qw6bANcAr2zH9iVptTz8JUkajIe/JEmDecwd/tpqq61q0aJF4y5DkuaUCy644Laqmr+66R5zobJo0SLOP//8cZchSXNKkutHmc7DX5KkwRgqkqTBGCqSpMEYKpKkwRgqkqTBGCqSpMEYKpKkwRgqkqTBGCqSpME85r5RrzWz6IjTx7Le6446YCzrlfTIuKciSRqMoSJJGoyhIkkajKEiSRqMoSJJGoyhIkkajKEiSRqMoSJJGoyhIkkajKEiSRqMoSJJGoyhIkkajKEiSRqMoSJJGoyhIkkajKEiSRqMoSJJGoyhIkkazIyFSpKFSc5OcnmSy5Ic3tq3TLI0ydXt57zWniRHJ1mW5OIku/WWtaRNf3WSJb325ya5pM1zdJLM1PZIklZvJvdU7gfeU1U7A3sB70iyM3AEcFZV7Qic1YYB9gN2bN1hwKehCyHgSGBPYA/gyIkgatO8uTffvjO4PZKk1ZixUKmqFVV1Yeu/B7gC2A5YDJzQJjsBOLD1LwZOrM45wBZJtgFeBiytqpVVdQewFNi3jdu8qs6pqgJO7C1LkjQG6+ScSpJFwHOAc4Gtq2pFG3UTsHXr3w64oTfb8ta2qvblU7RPtf7Dkpyf5Pxbb731EW2LJGl6Mx4qSTYFvga8u6ru7o9rexg10zVU1bFVtXtV7T5//vyZXp0kPWbNaKgk2ZAuUL5UVf/Qmm9uh65oP29p7TcCC3uzL2htq2pfMEW7JGlMZvLqrwCfB66oqr/ojToNmLiCawlwaq/9oHYV2F7AXe0w2ZnAPknmtRP0+wBntnF3J9mrreug3rIkSWOwwQwu+3nAG4FLklzU2v4EOAo4OcmhwPXAq9u4M4D9gWXAvcAhAFW1MsmHgPPadB+sqpWt/+3A8cDjgW+2TpI0JjMWKlX1r8B03xt58RTTF/COaZZ1HHDcFO3nA7s+gjIlSQPyG/WSpMEYKpKkwRgqkqTBGCqSpMEYKpKkwRgqkqTBGCqSpMEYKpKkwRgqkqTBGCqSpMEYKpKkwRgqkqTBGCqSpMEYKpKkwRgqkqTBGCqSpMEYKpKkwRgqkqTBGCqSpMEYKpKkwRgqkqTBGCqSpMEYKpKkwRgqkqTBGCqSpMEYKpKkwRgqkqTBGCqSpMEYKpKkwRgqkqTBGCqSpMEYKpKkwRgqkqTBGCqSpMEYKpKkwRgqkqTBGCqSpMHMWKgkOS7JLUku7bV9IMmNSS5q3f69ce9LsizJVUle1mvft7UtS3JEr/1pSc5t7V9JstFMbYskaTQzuadyPLDvFO2fqKpnt+4MgCQ7A68Fdmnz/HWS9ZOsD3wK2A/YGXhdmxbgo21ZTwfuAA6dwW2RJI1gxkKlqr4DrBxx8sXASVV1X1VdCywD9mjdsqq6pqp+DpwELE4S4EXAV9v8JwAHDroBkqQ1No5zKu9McnE7PDavtW0H3NCbZnlrm679ScCdVXX/pPYpJTksyflJzr/11luH2g5J0iTrOlQ+DewAPBtYAXx8Xay0qo6tqt2ravf58+evi1VK0mPSButyZVV180R/ks8C32iDNwILe5MuaG1M0347sEWSDdreSn96SdKYrNM9lSTb9AZ/F5i4Muw04LVJNk7yNGBH4AfAecCO7UqvjehO5p9WVQWcDbyyzb8EOHVdbIMkaXoztqeS5MvA3sBWSZYDRwJ7J3k2UMB1wFsAquqyJCcDlwP3A++oql+05bwTOBNYHziuqi5rq3gvcFKSDwM/BD4/U9siSRrNjIVKVb1uiuZpP/ir6iPAR6ZoPwM4Y4r2a+iuDpMkzRJ+o16SNBhDRZI0GENFkjQYQ0WSNBhDRZI0GENFkjQYQ0WSNBhDRZI0mHV67y+tnUVHnD7uEiRpJO6pSJIGY6hIkgYzUqgkOTzJ5ul8PsmFSfaZ6eIkSXPLqHsqb6qqu4F9gHnAG4GjZqwqSdKcNGqopP3cH/jbdvv5rGJ6SdJj0KihckGSb9GFyplJNgMemLmyJElz0aiXFB9K91z5a6rq3iRPAg6ZubIkSXPRqHsqS6vqwqq6E6Cqbgc+MXNlSZLmolXuqSR5HPAEukcCz+PB8yibA9vNcG2SpDlmdYe/3gK8G9gWuIAHQ+Vu4JgZrEuSNAetMlSq6pPAJ5P8flX91TqqSZI0R410or6q/irJbwKL+vNU1YkzVJckaQ4aKVSS/C2wA3AR8IvWXIChIkn6pVEvKd4d2LmqaiaLkSTNbaNeUnwp8JSZLESSNPeNuqeyFXB5kh8A9000VtXLZ6QqSdKcNGqofGAmi5AkPTqMevXXt2e6EEnS3Dfq1V/30F3tBbARsCHw06rafKYKkyTNPaPuqWw20Z8kwGJgr5kqSpI0N63x44Sr83XgZTNQjyRpDhv18NcreoPr0X1v5WczUpEkac4a9eqv3+n13w9cR3cITJKkXxr1nIoP5JIkrdZI51SSLEhySpJbWve1JAtmujhJ0twy6on6LwCn0T1XZVvgH1ubJEm/NGqozK+qL1TV/a07Hpg/g3VJkuagUUPl9iRvSLJ+694A3D6ThUmS5p5RQ+VNwKuBm4AVwCuBg1c1Q5Lj2vmXS3ttWyZZmuTq9nNea0+So5MsS3Jxkt168yxp01+dZEmv/blJLmnzHN2+lClJGqNRQ+WDwJKqml9VT6YLmT9bzTzHA/tOajsCOKuqdgTOasMA+wE7tu4w4NPQhRBwJLAnsAdw5EQQtWne3Jtv8rokSevYqKHyzKq6Y2KgqlYCz1nVDFX1HWDlpObFwAmt/wTgwF77ie3b+ucAWyTZhu5b+0uramVb/1Jg3zZu86o6pz047MTesiRJYzJqqKzX20OY2IMY9YuTfVtX1YrWfxOwdevfDrihN93y1raq9uVTtEuSxmjUYPg48P0kf9+GXwV85JGsuKoqyTp5PHGSw+gOq7H99tuvi1VK0mPSSHsqVXUi8Arg5ta9oqr+di3Wd3M7dEX7eUtrvxFY2JtuQWtbVfuCKdqnq//Yqtq9qnafP98roSVppox8l+Kquryqjmnd5Wu5vtOAiSu4lgCn9toPaleB7QXc1Q6TnQnsk2ReO/y2D3BmG3d3kr3aVV8H9ZYlSRqTtTkvMpIkXwb2BrZKspzuKq6jgJOTHApcT3eZMsAZwP7AMuBe4BDoLghI8iHgvDbdB9tFAgBvp7vC7PHAN1snSRqjGQuVqnrdNKNePMW0BbxjmuUcBxw3Rfv5wK6PpEZJ0rDW+CFdkiRNx1CRJA3GUJEkDcZQkSQNxlCRJA3GUJEkDcZQkSQNxlCRJA3GUJEkDcZQkSQNxlCRJA3GUJEkDcZQkSQNxlCRJA3GUJEkDcZQkSQNxlCRJA3GUJEkDWbGHicszVWLjjh9LOu97qgDxrJeaUjuqUiSBuOeimalce0tSHpk3FORJA3GUJEkDcZQkSQNxlCRJA3GUJEkDcZQkSQNxlCRJA3GUJEkDcZQkSQNxlCRJA3GUJEkDcZQkSQNxlCRJA3GUJEkDcZQkSQNxlCRJA3GUJEkDWYsoZLkuiSXJLkoyfmtbcskS5Nc3X7Oa+1JcnSSZUkuTrJbbzlL2vRXJ1kyjm2RJD1onHsq/7Wqnl1Vu7fhI4CzqmpH4Kw2DLAfsGPrDgM+DV0IAUcCewJ7AEdOBJEkaTxm0+GvxcAJrf8E4MBe+4nVOQfYIsk2wMuApVW1sqruAJYC+67roiVJD9pgTOst4FtJCvhMVR0LbF1VK9r4m4CtW/92wA29eZe3tunaHybJYXR7OWy//fZrXfSiI05f63kl6bFgXKHy/Kq6McmTgaVJruyPrKpqgTOIFlrHAuy+++6DLVeS9FBjOfxVVTe2n7cAp9CdE7m5Hdai/bylTX4jsLA3+4LWNl27JGlM1nmoJNkkyWYT/cA+wKXAacDEFVxLgFNb/2nAQe0qsL2Au9phsjOBfZLMayfo92ltkqQxGcfhr62BU5JMrP/vquqfkpwHnJzkUOB64NVt+jOA/YFlwL3AIQBVtTLJh4Dz2nQfrKqV624zJEmTrfNQqaprgGdN0X478OIp2gt4xzTLOg44bugaJUlrZzZdUixJmuMMFUnSYAwVSdJgDBVJ0mAMFUnSYAwVSdJgDBVJ0mAMFUnSYAwVSdJgDBVJ0mAMFUnSYAwVSdJgDBVJ0mAMFUnSYAwVSdJgDBVJ0mAMFUnSYAwVSdJgDBVJ0mAMFUnSYAwVSdJgDBVJ0mAMFUnSYAwVSdJgDBVJ0mAMFUnSYDYYdwGSOouOOH1s677uqAPGtm49urinIkkajKEiSRqMoSJJGoznVCSN7XyO53IefdxTkSQNxlCRJA3GUJEkDcZQkSQNxlCRJA3GUJEkDWbOh0qSfZNclWRZkiPGXY8kPZbN6e+pJFkf+BTwUmA5cF6S06rq8vFWJmkUfj/m0WdOhwqwB7Csqq4BSHISsBgwVCRNy5t3zpy5HirbATf0hpcDe06eKMlhwGFt8CdJrlrL9W0F3LaW864rs73G2V4fzP4aZ3t9MPtrHFt9+ejIk8621/Cpo0w010NlJFV1LHDsI11OkvOravcBSpoxs73G2V4fzP4aZ3t9MPtrnO31wdyocSpz/UT9jcDC3vCC1iZJGoO5HirnATsmeVqSjYDXAqeNuSZJesya04e/qur+JO8EzgTWB46rqstmcJWP+BDaOjDba5zt9cHsr3G21wezv8bZXh/MjRofJlU17hokSY8Sc/3wlyRpFjFUJEmDMVRGMNtvBZNkYZKzk1ye5LIkh4+7pqkkWT/JD5N8Y9y1TCXJFkm+muTKJFck+Y1x1zRZkj9o7/GlSb6c5HGzoKbjktyS5NJe25ZJlia5uv2cN8vq+1h7ny9OckqSLcZV33Q19sa9J0kl2Wocta0pQ2U1ereC2Q/YGXhdkp3HW9XD3A+8p6p2BvYC3jELawQ4HLhi3EWswieBf6qqXwGexSyrNcl2wLuA3atqV7qLU1473qoAOB7Yd1LbEcBZVbUjcFYbHpfjeXh9S4Fdq+qZwL8B71vXRU1yPA+vkSQLgX2A/1jXBa0tQ2X1fnkrmKr6OTBxK5hZo6pWVNWFrf8eug/D7cZb1UMlWQAcAHxu3LVMJckTgRcAnweoqp9X1Z3jrWpKGwCPT7IB8ATgx2Ouh6r6DrByUvNi4ITWfwJw4Dotqmeq+qrqW1V1fxs8h+47bmMzzWsI8Angj4E5c0WVobJ6U90KZlZ9YPclWQQ8Bzh3vJU8zF/S/XE8MO5CpvE04FbgC+0Q3eeSbDLuovqq6kbgz+n+a10B3FVV3xpvVdPauqpWtP6bgK3HWcxqvAn45riLmCzJYuDGqvrRuGtZE4bKo0iSTYGvAe+uqrvHXc+EJL8N3FJVF4y7llXYANgN+HRVPQf4KeM9ZPMw7bzEYroA3BbYJMkbxlvV6lX3vYVZ+Z92kvfTHT7+0rhr6UvyBOBPgD8ddy1rylBZvTlxK5gkG9IFypeq6h/GXc8kzwNenuQ6usOHL0ryxfGW9DDLgeVVNbGH91W6kJlNXgJcW1W3VtV/Av8A/OaYa5rOzUm2AWg/bxlzPQ+T5GDgt4HX1+z7wt4OdP88/Kj93SwALkzylLFWNQJDZfVm/a1gkoTuXMAVVfUX465nsqp6X1UtqKpFdK/fP1fVrPoPu6puAm5IslNrejGz7xEK/wHsleQJ7T1/MbPsYoKe04AlrX8JcOoYa3mYJPvSHY59eVXdO+56JquqS6rqyVW1qP3dLAd2a7+ns5qhshrtZN7ErWCuAE6e4VvBrI3nAW+k2wO4qHX7j7uoOej3gS8luRh4NvC/x1zPQ7S9qK8CFwKX0P39jv1WHkm+DHwf2CnJ8iSHAkcBL01yNd0e1lGzrL5jgM2Ape3v5W/GVd8qapyTvE2LJGkw7qlIkgZjqEiSBmOoSJIGY6hIkgZjqEiSBmOoSGspyctn8q7VSQ5c3Y1Bk+w9W+/6rMcmQ0VaS1V1WlXN5PcvDqS7M7Y0Zxgq0hSSLGrP2zg+yb8l+VKSlyT5bntGyB5JDk5yTJv+Ve0ZJz9K8p3WdnCSr7fniVyX5J1J/rDdsPKcJFu26d6c5Lw279faN+Z/E3g58LH25bwdkjw9yf9p012YZIdW7qa958B8qX3bniTPTfLtJBckObN325R3pXv2zsVJTlrnL64e3arKzs5uUgcsorvR4K/R/fN1AXAcELqbOn4dOBg4pk1/CbBd69+i/TwYWEb3ze35wF3AW9u4T9Dd+BPgSb31fhj4/dZ/PPDK3rhzgd9t/Y+ju/X93m25C1qd3weeD2wIfA+Y36Z/DXBc6/8xsHG/Vju7oboNBsgl6dHq2qq6BCDJZXQPnaokl9CFTt93geOTnEx3o8cJZ1f3jJt7ktwF/GNrvwR4ZuvfNcmHgS2ATeluCfQQSTajC61TAKrqZ60d4AdVtbwNX9RquxPYle42JNA90GviVvQX092O5ut04SgNxlCRpndfr/+B3vADTPrbqaq3JtmT7kFkFyR57hos43jgwKr6Ubtz7t6PoM5ftOUGuKyqpnok8gF0DyT7HeD9SX6tHnxglfSIeE5FGkCSHarq3Kr6U7qHfS1c3Tw9mwEr2uMLXt9rv6eNo+3tLE9yYFvfxu2ZG9O5Cpif5Dfa9Bsm2SXJesDCqjobeC/wRLq9I2kQhoo0jI8luSTJpXTnMtbkaX3/i+58yXeBK3vtJwF/1E7s70B3J+p3tbsofw+Y9tka1T36+pXAR5P8CLiI7tkr6wNfbIfwfggcXbPzscmao7xLsSRpMO6pSJIGY6hIkgZjqEiSBmOoSJIGY6hIkgZjqEiSBmOoSJIG8/8Bjay3WPyzP1wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist([count_substring_mismatch(seq, wild_type) for seq in sequence_lst])\n",
    "plt.xlabel(\"mismatches\")\n",
    "plt.ylabel(\"counts\")\n",
    "plt.title(\"gfp base sequence mismatches\")\n",
    "plt.savefig(\"./logs/gfp/gfp_mismatches.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'model_type': 'rnn',\n",
       " 'base_log': 'logs/gfp/rnn/',\n",
       " 'name': 'rnn_default_medium',\n",
       " 'input': 4998,\n",
       " 'hidden_size': 50,\n",
       " 'latent_dim': -1,\n",
       " 'seq_length': 238,\n",
       " 'pseudo_count': 1,\n",
       " 'n_jobs': 1,\n",
       " 'device': 'cpu',\n",
       " 'learning_rate': 0.001,\n",
       " 'epochs': 100,\n",
       " 'batch_size': 10,\n",
       " 'layers': 1,\n",
       " 'dataset': 'gfp',\n",
       " 'num_data': 1000,\n",
       " 'early_stopping': True,\n",
       " 'patience': 10}"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnn_default_args()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'model_type': 'vae',\n",
       " 'base_log': 'logs/gfp/vae/',\n",
       " 'name': 'vae_default_medium',\n",
       " 'input': 4998,\n",
       " 'hidden_size': 200,\n",
       " 'latent_dim': 20,\n",
       " 'seq_length': 238,\n",
       " 'pseudo_count': 1,\n",
       " 'n_jobs': 1,\n",
       " 'device': 'cpu',\n",
       " 'learning_rate': 0.001,\n",
       " 'epochs': 100,\n",
       " 'batch_size': 10,\n",
       " 'layers': 1,\n",
       " 'dataset': 'gfp',\n",
       " 'num_data': 1000,\n",
       " 'early_stopping': True,\n",
       " 'patience': 10}"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vae_default_args()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'model_type': 'hmm',\n",
       " 'base_log': 'logs/gfp/hmm/',\n",
       " 'name': 'hmm_default_medium',\n",
       " 'input': 4998,\n",
       " 'hidden_size': 25,\n",
       " 'latent_dim': -1,\n",
       " 'seq_length': 238,\n",
       " 'pseudo_count': 1,\n",
       " 'n_jobs': 1,\n",
       " 'device': 'cpu',\n",
       " 'learning_rate': 0.001,\n",
       " 'epochs': 100,\n",
       " 'batch_size': 10,\n",
       " 'layers': 1,\n",
       " 'dataset': 'gfp',\n",
       " 'num_data': 1000,\n",
       " 'early_stopping': True,\n",
       " 'patience': 10}"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hmm_default_args()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}