example_w2vec_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# modules to import \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import sys\n",
    "import seaborn as sns\n",
    "# this to set the plot styles \n",
    "sns.set_context('paper', font_scale = 1.5, rc = {\"lines.linewidth\": 1.5})\n",
    "sns.set_style('ticks', {'axes.grid': False, \n",
    "                        'grid.linestyle': '', \n",
    "                        'font.family':'sans-serif', \n",
    "                        'font.sans-serif':'Myriad Pro',\n",
    "                        'text.color': '0',\n",
    "                        'xtick.color': '0',\n",
    "                        'ytick.color': '0'\n",
    "                           })\n",
    "\n",
    "\n",
    "sys.path.insert(1, '/work/06658/pramesh/maverick2/notebooks/')\n",
    "from nlp_wvec_utils import *\n",
    "# Ignore  the warnings\n",
    "import warnings\n",
    "warnings.filterwarnings('always')\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>off_id</th>\n",
       "      <th>on_id</th>\n",
       "      <th>source_sequence</th>\n",
       "      <th>sequence_id</th>\n",
       "      <th>pre_seq</th>\n",
       "      <th>promoter</th>\n",
       "      <th>trigger</th>\n",
       "      <th>loop1</th>\n",
       "      <th>switch</th>\n",
       "      <th>loop2</th>\n",
       "      <th>...</th>\n",
       "      <th>on_value</th>\n",
       "      <th>off_value</th>\n",
       "      <th>delta_onoff</th>\n",
       "      <th>on_qc</th>\n",
       "      <th>off_qc</th>\n",
       "      <th>delta_qc_onoff</th>\n",
       "      <th>min_toehold_sequence</th>\n",
       "      <th>toehold_quartile</th>\n",
       "      <th>quartile_label</th>\n",
       "      <th>quartile_rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>85396</th>\n",
       "      <td>AACCAAACACACAAACGCACCCTGGGAAGGTGTCAACTGCTGATGT...</td>\n",
       "      <td>CTACACATCAGCAGTTGACACCTTCCCAGGAACCAAACACACAAAC...</td>\n",
       "      <td>human_ZNF281</td>\n",
       "      <td>human_ZNF281_tile_227</td>\n",
       "      <td>CTCTGGGCTAACTGTCGCGC</td>\n",
       "      <td>TAATACGACTCACTATAGGG</td>\n",
       "      <td>CTACACATCAGCAGTTGACACCTTCCCAGG</td>\n",
       "      <td>AACCAAACACACAAACGCAC</td>\n",
       "      <td>CCTGGGAAGGTGTCAACTGCTGATGTGTAG</td>\n",
       "      <td>AACAGAGGAGA</td>\n",
       "      <td>...</td>\n",
       "      <td>0.154633</td>\n",
       "      <td>0.039786</td>\n",
       "      <td>0.114848</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.1</td>\n",
       "      <td>1.1</td>\n",
       "      <td>CCTGGGAAGGTGTCAACTGCTGATGTGTAGAACAGAGGAGACTACA...</td>\n",
       "      <td>Q2</td>\n",
       "      <td>Bad</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154635</th>\n",
       "      <td>AACCAAACACACAAACGCACGGGACCAACATTACAATGCATTCTAA...</td>\n",
       "      <td>CCCTTTAGAATGCATTGTAATGTTGGTCCCAACCAAACACACAAAC...</td>\n",
       "      <td>random_sequences</td>\n",
       "      <td>random_sequence_86097</td>\n",
       "      <td>CTCTGGGCTAACTGTCGCGC</td>\n",
       "      <td>TAATACGACTCACTATAGGG</td>\n",
       "      <td>CCCTTTAGAATGCATTGTAATGTTGGTCCC</td>\n",
       "      <td>AACCAAACACACAAACGCAC</td>\n",
       "      <td>GGGACCAACATTACAATGCATTCTAAAGGG</td>\n",
       "      <td>AACAGAGGAGA</td>\n",
       "      <td>...</td>\n",
       "      <td>0.034915</td>\n",
       "      <td>0.142617</td>\n",
       "      <td>-0.107702</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.1</td>\n",
       "      <td>1.1</td>\n",
       "      <td>GGGACCAACATTACAATGCATTCTAAAGGGAACAGAGGAGACCCTT...</td>\n",
       "      <td>Q1</td>\n",
       "      <td>Bad</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188168</th>\n",
       "      <td>AACCAAACACACAAACGCACTATTATTCAGTTTGTAAACGAGGATA...</td>\n",
       "      <td>TTTTTATCCTCGTTTACAAACTGAATAATAAACCAAACACACAAAC...</td>\n",
       "      <td>smallpox</td>\n",
       "      <td>smallpox_tile_21424</td>\n",
       "      <td>CTCTGGGCTAACTGTCGCGC</td>\n",
       "      <td>TAATACGACTCACTATAGGG</td>\n",
       "      <td>TTTTTATCCTCGTTTACAAACTGAATAATA</td>\n",
       "      <td>AACCAAACACACAAACGCAC</td>\n",
       "      <td>TATTATTCAGTTTGTAAACGAGGATAAAAA</td>\n",
       "      <td>AACAGAGGAGA</td>\n",
       "      <td>...</td>\n",
       "      <td>0.624165</td>\n",
       "      <td>0.305005</td>\n",
       "      <td>0.319161</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>TATTATTCAGTTTGTAAACGAGGATAAAAAAACAGAGGAGATTTTT...</td>\n",
       "      <td>Q3</td>\n",
       "      <td>Bad</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156905</th>\n",
       "      <td>AACCAAACACACAAACGCACGGGCTTTCTCCACGAAAATGACCTTG...</td>\n",
       "      <td>ACTCCAAGGTCATTTTCGTGGAGAAAGCCCAACCAAACACACAAAC...</td>\n",
       "      <td>human_NFATC1</td>\n",
       "      <td>human_NFATC1_tile_188</td>\n",
       "      <td>CTCTGGGCTAACTGTCGCGC</td>\n",
       "      <td>TAATACGACTCACTATAGGG</td>\n",
       "      <td>ACTCCAAGGTCATTTTCGTGGAGAAAGCCC</td>\n",
       "      <td>AACCAAACACACAAACGCAC</td>\n",
       "      <td>GGGCTTTCTCCACGAAAATGACCTTGGAGT</td>\n",
       "      <td>AACAGAGGAGA</td>\n",
       "      <td>...</td>\n",
       "      <td>0.057950</td>\n",
       "      <td>0.063176</td>\n",
       "      <td>-0.005226</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.1</td>\n",
       "      <td>1.1</td>\n",
       "      <td>GGGCTTTCTCCACGAAAATGACCTTGGAGTAACAGAGGAGAACTCC...</td>\n",
       "      <td>Q1</td>\n",
       "      <td>Bad</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128239</th>\n",
       "      <td>AACCAAACACACAAACGCACGATTATATAGGACAATTTGACATGAG...</td>\n",
       "      <td>GAATCTCATGTCAAATTGTCCTATATAATCAACCAAACACACAAAC...</td>\n",
       "      <td>smallpox</td>\n",
       "      <td>smallpox_tile_11433</td>\n",
       "      <td>CTCTGGGCTAACTGTCGCGC</td>\n",
       "      <td>TAATACGACTCACTATAGGG</td>\n",
       "      <td>GAATCTCATGTCAAATTGTCCTATATAATC</td>\n",
       "      <td>AACCAAACACACAAACGCAC</td>\n",
       "      <td>GATTATATAGGACAATTTGACATGAGATTC</td>\n",
       "      <td>AACAGAGGAGA</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.515572</td>\n",
       "      <td>0.484428</td>\n",
       "      <td>1.1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.1</td>\n",
       "      <td>GATTATATAGGACAATTTGACATGAGATTCAACAGAGGAGAGAATC...</td>\n",
       "      <td>Q4</td>\n",
       "      <td>Good</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   off_id  \\\n",
       "85396   AACCAAACACACAAACGCACCCTGGGAAGGTGTCAACTGCTGATGT...   \n",
       "154635  AACCAAACACACAAACGCACGGGACCAACATTACAATGCATTCTAA...   \n",
       "188168  AACCAAACACACAAACGCACTATTATTCAGTTTGTAAACGAGGATA...   \n",
       "156905  AACCAAACACACAAACGCACGGGCTTTCTCCACGAAAATGACCTTG...   \n",
       "128239  AACCAAACACACAAACGCACGATTATATAGGACAATTTGACATGAG...   \n",
       "\n",
       "                                                    on_id   source_sequence  \\\n",
       "85396   CTACACATCAGCAGTTGACACCTTCCCAGGAACCAAACACACAAAC...      human_ZNF281   \n",
       "154635  CCCTTTAGAATGCATTGTAATGTTGGTCCCAACCAAACACACAAAC...  random_sequences   \n",
       "188168  TTTTTATCCTCGTTTACAAACTGAATAATAAACCAAACACACAAAC...          smallpox   \n",
       "156905  ACTCCAAGGTCATTTTCGTGGAGAAAGCCCAACCAAACACACAAAC...      human_NFATC1   \n",
       "128239  GAATCTCATGTCAAATTGTCCTATATAATCAACCAAACACACAAAC...          smallpox   \n",
       "\n",
       "                  sequence_id               pre_seq              promoter  \\\n",
       "85396   human_ZNF281_tile_227  CTCTGGGCTAACTGTCGCGC  TAATACGACTCACTATAGGG   \n",
       "154635  random_sequence_86097  CTCTGGGCTAACTGTCGCGC  TAATACGACTCACTATAGGG   \n",
       "188168    smallpox_tile_21424  CTCTGGGCTAACTGTCGCGC  TAATACGACTCACTATAGGG   \n",
       "156905  human_NFATC1_tile_188  CTCTGGGCTAACTGTCGCGC  TAATACGACTCACTATAGGG   \n",
       "128239    smallpox_tile_11433  CTCTGGGCTAACTGTCGCGC  TAATACGACTCACTATAGGG   \n",
       "\n",
       "                               trigger                 loop1  \\\n",
       "85396   CTACACATCAGCAGTTGACACCTTCCCAGG  AACCAAACACACAAACGCAC   \n",
       "154635  CCCTTTAGAATGCATTGTAATGTTGGTCCC  AACCAAACACACAAACGCAC   \n",
       "188168  TTTTTATCCTCGTTTACAAACTGAATAATA  AACCAAACACACAAACGCAC   \n",
       "156905  ACTCCAAGGTCATTTTCGTGGAGAAAGCCC  AACCAAACACACAAACGCAC   \n",
       "128239  GAATCTCATGTCAAATTGTCCTATATAATC  AACCAAACACACAAACGCAC   \n",
       "\n",
       "                                switch        loop2  ...  on_value off_value  \\\n",
       "85396   CCTGGGAAGGTGTCAACTGCTGATGTGTAG  AACAGAGGAGA  ...  0.154633  0.039786   \n",
       "154635  GGGACCAACATTACAATGCATTCTAAAGGG  AACAGAGGAGA  ...  0.034915  0.142617   \n",
       "188168  TATTATTCAGTTTGTAAACGAGGATAAAAA  AACAGAGGAGA  ...  0.624165  0.305005   \n",
       "156905  GGGCTTTCTCCACGAAAATGACCTTGGAGT  AACAGAGGAGA  ...  0.057950  0.063176   \n",
       "128239  GATTATATAGGACAATTTGACATGAGATTC  AACAGAGGAGA  ...  1.000000  0.515572   \n",
       "\n",
       "       delta_onoff on_qc off_qc  delta_qc_onoff  \\\n",
       "85396     0.114848   3.0    1.1             1.1   \n",
       "154635   -0.107702   2.0    1.1             1.1   \n",
       "188168    0.319161   4.0    3.0             3.0   \n",
       "156905   -0.005226   2.0    1.1             1.1   \n",
       "128239    0.484428   1.1    4.0             1.1   \n",
       "\n",
       "                                     min_toehold_sequence  toehold_quartile  \\\n",
       "85396   CCTGGGAAGGTGTCAACTGCTGATGTGTAGAACAGAGGAGACTACA...                Q2   \n",
       "154635  GGGACCAACATTACAATGCATTCTAAAGGGAACAGAGGAGACCCTT...                Q1   \n",
       "188168  TATTATTCAGTTTGTAAACGAGGATAAAAAAACAGAGGAGATTTTT...                Q3   \n",
       "156905  GGGCTTTCTCCACGAAAATGACCTTGGAGTAACAGAGGAGAACTCC...                Q1   \n",
       "128239  GATTATATAGGACAATTTGACATGAGATTCAACAGAGGAGAGAATC...                Q4   \n",
       "\n",
       "        quartile_label  quartile_rating  \n",
       "85396              Bad                0  \n",
       "154635             Bad                0  \n",
       "188168             Bad                0  \n",
       "156905             Bad                0  \n",
       "128239            Good                1  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load in the toehold dataset and preprocess for ML \n",
    "df = pd.read_csv('/work/06658/pramesh/maverick2/excels/newQC_toehold_data.csv', comment = '#')\n",
    "\n",
    "# rename the columns \n",
    "rename_dict = {\n",
    "    \"Unnamed: 0\" : \"toehold_id\",\n",
    "    \"onoff_value\" : \"delta_onoff\",\n",
    "    \"onoff_qc\" : \"delta_qc_onoff\",\n",
    "    \"switch_sequence\" : \"min_toehold_sequence\"\n",
    "}\n",
    "\n",
    "# clean up df to get rid of NaN and low qc reads \n",
    "df = df.rename(columns = rename_dict)\n",
    "df = df.dropna() # throw out nan's \n",
    "ngs_qc_onind = df['on_qc'] >= 1.1 # keep all the acceptable reads for the ON\n",
    "ngs_qc_offind = df['off_qc'] >= 1.1 # keep all the acceptable reads for the OFF\n",
    "df = df.loc[ngs_qc_onind & ngs_qc_offind, :]\n",
    "\n",
    "# bin the toeholds by their ON/OFF ratio into quartlies since the data were collected as 4 bins\n",
    "df['toehold_quartile'] = pd.qcut(df['delta_onoff'], q = 4, labels = ['Q1', 'Q2', 'Q3', 'Q4'])\n",
    "\n",
    "df_top = df[df['toehold_quartile'] == 'Q4']\n",
    "df_top['quartile_label'] = 'Good'\n",
    "df_rest = df[df['toehold_quartile'] != 'Q4']\n",
    "df_rest['quartile_label'] = 'Bad'\n",
    "df_classify = pd.concat([df_top, df_rest], axis = 0)\n",
    "\n",
    "# instantiate the label encoder\n",
    "label_encoder = LabelEncoder()\n",
    "df_classify['quartile_rating'] = label_encoder.fit_transform(df_classify['quartile_label'])\n",
    "df_classify = df_classify.sample(frac = 1)\n",
    "\n",
    "# we now have a df of sequences that we will train a classifier\n",
    "df_classify.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embeddings trained and saved!\n"
     ]
    }
   ],
   "source": [
    "# parameters to train the skip-gram embedding (takes a long time)\n",
    "ngram_size = 3\n",
    "ngram_stride = 3\n",
    "dim_embedding = 400\n",
    "\n",
    "# train the skip-gram embedding and save it as a text file \n",
    "df_classify, embeddings_index =  train_sg_embedding(df_classify = df_classify, \n",
    "                                                    ngram_size = ngram_size,\n",
    "                                                    ngram_stride = ngram_stride, \n",
    "                                                    dim_embedding = dim_embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split the data into training and testing sets and get the embedding lookup\n",
    "x_train, x_test, y_train, y_test, embedding_matrix, max_length = create_ds_splits(df_classify = df_classify, \n",
    "                                                                                  embeddings_index = embeddings_index, \n",
    "                                                                                  dim_embedding = dim_embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the optimal parameters were determined using grid-search so we define a series of functions for the \n",
    "# different architectures \n",
    "\n",
    "# create lstm grid-search model\n",
    "def create_lstm_gs_model(embedding_matrix = embedding_matrix, max_length = max_length, neurons = 64, \n",
    "                         dropout_rate1 = 0.0, dropout_rate2 = 0.0, weight_constraint = 0):\n",
    "    \"\"\"\n",
    "    This function creates a simple LSTM with the embedding as the inputs. \n",
    "    \"\"\"\n",
    "    \n",
    "    num_words = embedding_matrix.shape[0]\n",
    "    dim_embedding = embedding_matrix.shape[1]\n",
    "    \n",
    "    # instantiate model\n",
    "    model = Sequential()\n",
    "    embedding_layer = Embedding(input_dim = num_words, output_dim = dim_embedding, \n",
    "                                weights = [embedding_matrix],\n",
    "                                input_length = max_length,\n",
    "                                trainable = False)\n",
    "    model.add(embedding_layer)\n",
    "    model.add(Dropout(dropout_rate1))\n",
    "    model.add(LSTM(units = neurons, kernel_constraint = max_norm(weight_constraint)))\n",
    "    model.add(Dropout(dropout_rate2))\n",
    "    model.add(Dense(1, activation = 'sigmoid', kernel_initializer = 'he_uniform')) # binary classification \n",
    "    model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "def create_bidlstm_gs_model(embedding_matrix = embedding_matrix, max_length = max_length, neurons = 64,\n",
    "                            dropout_rate1 = 0.0, dropout_rate2 = 0.0,  weight_constraint = 0):\n",
    "    \"\"\"\n",
    "    This function creates a Bi-directional LSTM with the embedding as the inputs. \n",
    "    \"\"\"\n",
    "    \n",
    "    num_words = embedding_matrix.shape[0]\n",
    "    dim_embedding = embedding_matrix.shape[1]\n",
    "    \n",
    "    # instantiate model\n",
    "    model = Sequential()\n",
    "    embedding_layer = Embedding(input_dim = num_words, output_dim = dim_embedding, \n",
    "                                weights = [embedding_matrix],\n",
    "                                input_length = max_length,\n",
    "                                trainable = False)\n",
    "    model.add(embedding_layer)\n",
    "    model.add(Dropout(dropout_rate1))\n",
    "    model.add(Bidirectional(LSTM(units = neurons, kernel_constraint = max_norm(weight_constraint))))\n",
    "    model.add(Dropout(dropout_rate2))\n",
    "    model.add(Dense(1, activation = 'sigmoid', kernel_initializer = 'he_uniform')) # binary classification \n",
    "    model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "def create_self_atten_gs_model(embedding_matrix = embedding_matrix, max_length = max_length, neurons = 64,\n",
    "                               dropout_rate1 = 0.0, dropout_rate2 = 0.0,  weight_constraint = 0):\n",
    "    \"\"\"\n",
    "    This function creates a Bi-directional LSTM model with a Self-Attention layer\n",
    "    \"\"\"\n",
    "    \n",
    "    num_words = embedding_matrix.shape[0]\n",
    "    dim_embedding = embedding_matrix.shape[1]\n",
    "    \n",
    "    # instantiate model\n",
    "    model = Sequential()\n",
    "    embedding_layer = Embedding(input_dim = num_words, output_dim = dim_embedding, \n",
    "                                weights = [embedding_matrix],\n",
    "                                input_length = max_length,\n",
    "                                trainable = False)\n",
    "    model.add(embedding_layer)\n",
    "    model.add(Dropout(dropout_rate1))\n",
    "    model.add(Bidirectional(LSTM(units = neurons, return_sequences = True, recurrent_activation = 'relu', \n",
    "                                 kernel_constraint = max_norm(weight_constraint))))\n",
    "    model.add(SeqSelfAttention(attention_width = 6, attention_activation = 'sigmoid'))\n",
    "    model.add(Flatten()) # to get one output neuron \n",
    "    model.add(Dropout(dropout_rate2))\n",
    "    model.add(Dense(1, activation = 'sigmoid', kernel_initializer = 'he_uniform')) # binary classification \n",
    "    model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])\n",
    " \n",
    "    return model\n",
    "\n",
    "def create_cnn_lstm_model(embedding_matrix = embedding_matrix, max_length = max_length, neurons = 64,\n",
    "                          dropout_rate1 = 0.0, dropout_rate2 = 0.0, weight_constraint = 0):\n",
    "    \"\"\"\n",
    "    This function creates a CNN-LSTM model \n",
    "    \"\"\"\n",
    "    \n",
    "    num_words = embedding_matrix.shape[0]\n",
    "    dim_embedding = embedding_matrix.shape[1]\n",
    "    \n",
    "    # instantiate model\n",
    "    model = Sequential()\n",
    "    embedding_layer = Embedding(input_dim = num_words, output_dim = dim_embedding, \n",
    "                                weights = [embedding_matrix],\n",
    "                                input_length = max_length,\n",
    "                                trainable = False)\n",
    "    model.add(embedding_layer)\n",
    "    model.add(Dropout(dropout_rate1))\n",
    "    model.add(Conv1D(filters = filters, kernel_size = 2, padding = 'same', activation = 'relu'))\n",
    "    model.add(MaxPooling1D(pool_size = 4))\n",
    "    model.add(Dropout(dropout_rate2))\n",
    "    model.add(LSTM(units = neurons, kernel_constraint = max_norm(weight_constraint)))\n",
    "    model.add(Dense(1, activation = 'sigmoid', kernel_initializer = 'he_uniform')) # binary classification \n",
    "    model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])\n",
    " \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the grid search parameters (takes a very long time)\n",
    "weight_constraint = [1, 2, 3, 4]\n",
    "dropout_rate1 = [0.2, 0.3, 0.4]\n",
    "dropout_rate2 = [0.1, 0.2, 0.3]\n",
    "neurons = [64, 128, 256]\n",
    "param_grid = dict(dropout_rate1 = dropout_rate1,\n",
    "                  dropout_rate2 = dropout_rate2,\n",
    "                  neurons = neurons,\n",
    "                  weight_constraint = weight_constraint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# step through each model architecture and find the best parameters\n",
    "model = KerasClassifier(build_fn = create_lstm_gs_model, epochs = 100, batch_size = 128, verbose = 1)\n",
    "\n",
    "# instantiate the loss-history class\n",
    "history = LossHistory()\n",
    "grid = GridSearchCV(estimator = model, param_grid = param_grid, n_jobs = -1, cv = 5)\n",
    "    \n",
    "grid_result = grid.fit(x_train, y_train, validation_split = 0.1, callbacks = [history])\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history, accs, aucs, model = train_lstm(x_train, x_test, y_train, y_test, embedding_matrix, max_length)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}