Generative_Models_50_Authors_0.90_RESULT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X\n"
     ]
    }
   ],
   "source": [
    "print(\"X\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1000000\n",
      "2000000\n",
      "3000000\n",
      "4000000\n",
      "CPU times: user 43.1 s, sys: 5.39 s, total: 48.5 s\n",
      "Wall time: 23min 52s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "import json\n",
    "reviews = []\n",
    "with open(\"yelp_academic_dataset_review.json\") as f:\n",
    "    index = 0\n",
    "    for line in f:\n",
    "        if index % 1000000 == 0:\n",
    "            print(index)\n",
    "        index += 1\n",
    "        reviews.append(json.loads(line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'useful': 0, 'date': '2011-10-10', 'type': 'review', 'review_id': 'NxL8SIC5yqOdnlXCg18IBg', 'text': \"If you enjoy service by someone who is as competent as he is personable, I would recommend Corey Kaplan highly. The time he has spent here has been very productive and working with him educational and enjoyable. I hope not to need him again (though this is highly unlikely) but knowing he is there if I do is very nice. By the way, I'm not from El Centro, CA. but Scottsdale, AZ.\", 'stars': 5, 'funny': 0, 'business_id': '2aFiy99vNLklCx3T_tGS9A', 'cool': 0, 'user_id': 'KpkOkG6RIf4Ra25Lhhxf1A'}\n"
     ]
    }
   ],
   "source": [
    "print(reviews[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "prolific_reviewers = Counter([review['user_id'] for review in reviews]).most_common(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "keep_ids = {pr[0] : 0 for pr in prolific_reviewers}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "by_author = {} # author : \"review 1\\n review 2\\n review 3\"\n",
    "for review in reviews:\n",
    "    uid = review['user_id']\n",
    "    if uid in keep_ids:\n",
    "        uid = review['user_id']\n",
    "        if uid in by_author:\n",
    "            by_author[uid] += \"\\n{}\".format(review['text'])\n",
    "        else:\n",
    "            by_author[uid] = \"{}\".format(review['text'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(by_author)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(276136, 'ffPY_bHX8vLebHu8LBEqfg'),\n",
       " (278427, 'PeLGa5vUR8_mcsn-fn42Jg'),\n",
       " (338936, 'Q4Qfu-3vYtL1LRm2X1b0Gg'),\n",
       " (351311, 'cMEtAiW60I5wE_vLfTxoJQ'),\n",
       " (370129, 'iDlkZO2iILS8Jwfdy7DP9A')]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check that we have at least 200000 characters for each author\n",
    "sorted([(len(by_author[key]), key) for key in by_author])[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_chunks(l, n):\n",
    "    n = max(1, n)\n",
    "    return [l[i:i+n] for i in range(0, len(l), n)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "train_texts = []  # the first 100 000 chars from each author\n",
    "train_labels = [] # each author\n",
    "test_texts = []   # 100 texts of 1000 characters each (second 100 000 chars of each author)\n",
    "test_labels = []  # each author * 100\n",
    "\n",
    "author_int = {author: i for i,author in enumerate(by_author)}\n",
    "int_author = {author_int[author]: author for author in author_int}\n",
    "\n",
    "for author in by_author:\n",
    "    train_text = by_author[author][:50000]\n",
    "    train_label = author_int[author]\n",
    "    train_texts.append(train_text)\n",
    "    train_labels.append(train_label)\n",
    "    \n",
    "    short_texts = get_chunks(by_author[author][50000:100000], 1000)\n",
    "    for text in short_texts:\n",
    "        test_texts.append(text)\n",
    "        test_labels.append(author_int[author])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = {\n",
    "    \"train_texts\": train_texts,\n",
    "    \"train_labels\": train_labels,\n",
    "    \"test_texts\": test_texts,\n",
    "    \"test_labels\": test_labels\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open(\"train_test_texts_labels.pickle\", \"wb\") as f:\n",
    "    pickle.dump(data, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open(\"train_test_texts_labels.pickle\", \"rb\") as f:\n",
    "    data = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_texts = data['train_texts']\n",
    "train_labels = data['train_labels']\n",
    "test_texts = data['test_texts']\n",
    "test_labels = data['test_labels']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50 2500\n"
     ]
    }
   ],
   "source": [
    "print(len(train_texts), len(test_texts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50000"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_texts[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# vectorization - chars to ints\n",
    "import string\n",
    "import random\n",
    "import sys\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from keras.models import load_model\n",
    "\n",
    "def sample(preds, temperature=1.0):\n",
    "    \"\"\"Sample predictions from a probability array\"\"\"\n",
    "    preds = np.asarray(preds).astype('float64')\n",
    "    preds = np.log(preds + 1e-6) / temperature\n",
    "    exp_preds = np.exp(preds)\n",
    "    preds = exp_preds / np.sum(exp_preds)\n",
    "    probas = np.random.multinomial(1, preds, 1)\n",
    "    return np.argmax(probas)\n",
    "\n",
    "def generate(model, diversity=0.5, text=\"\"):\n",
    "    \"\"\"Generate text from a model\"\"\"\n",
    "    start_index = random.randint(0, len(text) - maxlen - 1)\n",
    "    generated = ''\n",
    "    sentence = text[start_index: start_index + maxlen]\n",
    "    generated += sentence\n",
    "    print('----- Generating with seed: \"' + sentence + '\"')\n",
    "    sys.stdout.write(generated)\n",
    "\n",
    "    for i in range(5000):\n",
    "        x = np.zeros((1, maxlen), dtype=np.int)\n",
    "        for t, char in enumerate(sentence):\n",
    "            try:\n",
    "                x[0, t] = char_indices[char]\n",
    "            except:\n",
    "                print(sentence)\n",
    "        preds = model.predict(x, verbose=0)[0][0]\n",
    "        next_index = sample(preds, diversity)\n",
    "        next_char = indices_char[next_index]\n",
    "        generated += next_char\n",
    "        sentence = sentence[1:] + next_char\n",
    "        sys.stdout.write(next_char)\n",
    "        sys.stdout.flush()\n",
    "    return\n",
    "\n",
    "def vectorize(text):\n",
    "    \"\"\"Convert text into character sequences\"\"\"\n",
    "    step = 3\n",
    "    sentences = []\n",
    "    next_chars = []\n",
    "    for i in range(0, len(text) - maxlen, step):\n",
    "        sentences.append(text[i: i + maxlen])\n",
    "        next_chars.append(text[i + maxlen])\n",
    "    X = np.zeros((len(sentences), maxlen), dtype=np.int)\n",
    "    y = np.zeros((len(sentences), len(chars)), dtype=np.bool)\n",
    "    for i, sentence in enumerate(sentences):\n",
    "        for t, char in enumerate(sentence):\n",
    "            X[i, t] = char_indices[char]\n",
    "        y[i, char_indices[next_chars[i]]] = 1\n",
    "    return X, y\n",
    "\n",
    "def clean_text(text, charset):\n",
    "    text = \" \".join(text.split())  # all white space is one space\n",
    "    text = \"\".join([x for x in text if x in charset])  # remove characters that we don't care about\n",
    "    return text\n",
    "\n",
    "def get_model(modelfile, freeze=False):\n",
    "    model = load_model(modelfile)\n",
    "    if freeze:\n",
    "        for layer in model.layers[:6]:\n",
    "            layer.trainable = False\n",
    "    return model\n",
    "\n",
    "chars = \" \" + string.ascii_letters + string.punctuation  # sorted to keep indices consistent\n",
    "charset = set(chars)  # for lookup\n",
    "char_indices = dict((c, i) for i, c in enumerate(chars))\n",
    "indices_char = dict((i, c) for i, c in enumerate(chars))\n",
    "\n",
    "maxlen = 100  # must match length which generated model - the sequence length\n",
    "\n",
    "# load a pretrained language model\n",
    "modelfile = \"charlm2/model_middlemarch_cnn.hdf5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing pretrained character embeds...\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Embedding, Dropout, BatchNormalization, GRU, Dense\n",
    "\n",
    "# https://blog.keras.io/using-pre-trained-word-embeddings-in-a-keras-model.html\n",
    "print('Processing pretrained character embeds...')\n",
    "embedding_vectors = {}\n",
    "with open(\"glove.840B.300d-char.txt\", \"r\") as f:\n",
    "    for line in f:\n",
    "        line_split = line.strip().split(\" \")\n",
    "        vec = np.array(line_split[1:], dtype=float)\n",
    "        char = line_split[0]\n",
    "        embedding_vectors[char] = vec\n",
    "\n",
    "embedding_matrix = np.zeros((len(chars), 300))\n",
    "#embedding_matrix = np.random.uniform(-1, 1, (len(chars), 300))\n",
    "for char, i in char_indices.items():\n",
    "    #print (\"{}, {}\".format(char, i))\n",
    "    embedding_vector = embedding_vectors.get(char)\n",
    "    if embedding_vector is not None:\n",
    "        embedding_matrix[i] = embedding_vector\n",
    "\n",
    "def get_gru_model(use_embeddings=False):\n",
    "    model = Sequential()\n",
    "    if use_embeddings:\n",
    "        model.add(Embedding(input_dim=len(charset), output_dim=300, weights=[embedding_matrix]))\n",
    "    else:\n",
    "        model.add(Embedding(input_dim=len(charset), output_dim=300))\n",
    "    model.add(Dropout(0.1))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(GRU(256))\n",
    "    model.add(Dropout(0.3))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dense(85, activation='softmax'))\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 14831 samples, validate on 1648 samples\n",
      "Epoch 1/10\n",
      "14831/14831 [==============================] - 19s - loss: 3.1480 - val_loss: 3.9208\n",
      "Epoch 2/10\n",
      "14831/14831 [==============================] - 16s - loss: 2.4809 - val_loss: 3.2844\n",
      "Epoch 3/10\n",
      "14831/14831 [==============================] - 17s - loss: 2.2282 - val_loss: 2.7332\n",
      "Epoch 4/10\n",
      "14831/14831 [==============================] - 16s - loss: 2.0630 - val_loss: 2.3528\n",
      "Epoch 5/10\n",
      "14831/14831 [==============================] - 16s - loss: 1.9500 - val_loss: 2.1056\n",
      "Epoch 6/10\n",
      "14831/14831 [==============================] - 16s - loss: 1.8630 - val_loss: 2.0248\n",
      "Epoch 7/10\n",
      "14831/14831 [==============================] - 16s - loss: 1.7832 - val_loss: 2.0104\n",
      "Epoch 8/10\n",
      "14831/14831 [==============================] - 16s - loss: 1.7099 - val_loss: 2.0018\n",
      "Epoch 9/10\n",
      "14831/14831 [==============================] - 16s - loss: 1.6386 - val_loss: 1.9920\n",
      "Epoch 10/10\n",
      "14831/14831 [==============================] - 16s - loss: 1.5568 - val_loss: 2.0394\n",
      "CPU times: user 4min 10s, sys: 18.8 s, total: 4min 29s\n",
      "Wall time: 2min 55s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "test_model = get_gru_model()\n",
    "X, y = vectorize(clean_text(train_texts[0], charset))\n",
    "test_model.fit(X, y, epochs=10, batch_size=128, validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 / 50\n",
      "1 / 50\n",
      "2 / 50\n",
      "3 / 50\n",
      "4 / 50\n",
      "5 / 50\n",
      "6 / 50\n",
      "7 / 50\n",
      "8 / 50\n",
      "9 / 50\n",
      "10 / 50\n",
      "11 / 50\n",
      "12 / 50\n",
      "13 / 50\n",
      "14 / 50\n",
      "16 / 50\n",
      "17 / 50\n",
      "18 / 50\n",
      "19 / 50\n",
      "20 / 50\n",
      "21 / 50\n",
      "22 / 50\n",
      "23 / 50\n",
      "24 / 50\n",
      "25 / 50\n",
      "26 / 50\n",
      "27 / 50\n",
      "28 / 50\n",
      "29 / 50\n",
      "30 / 50\n",
      "31 / 50\n",
      "32 / 50\n",
      "33 / 50\n",
      "34 / 50\n",
      "35 / 50\n",
      "36 / 50\n",
      "37 / 50\n",
      "38 / 50\n",
      "39 / 50\n",
      "40 / 50\n",
      "41 / 50\n",
      "42 / 50\n",
      "43 / 50\n",
      "44 / 50\n",
      "45 / 50\n",
      "46 / 50\n",
      "47 / 50\n",
      "48 / 50\n",
      "49 / 50\n",
      "CPU times: user 3h 45min 50s, sys: 17min 12s, total: 4h 3min 2s\n",
      "Wall time: 2h 35min 14s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "author_models = []  # [(author_model, author_id), (author_model, author_id), ...] - ids are ints\n",
    "for i, train_text in enumerate(train_texts):\n",
    "    print(\"{} / {}\".format(i, len(train_texts)))\n",
    "    ct = clean_text(train_text, charset)\n",
    "    am = get_gru_model()\n",
    "    X, y = vectorize(ct)\n",
    "    am.fit(X, y, epochs=10, batch_size=128, verbose=0)\n",
    "    author_models.append((am, train_labels[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "print(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50\n",
      "2500\n",
      "2500\n"
     ]
    }
   ],
   "source": [
    "print(len(author_models))\n",
    "print(len(test_texts))\n",
    "print(len(test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "182.5832"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from statistics import mean\n",
    "word_counts = [text.count(\" \") for text in test_texts]\n",
    "mean(word_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 182 words is quite short\n",
    "# Try to join 5 tests texts together\n",
    "longer_test_texts = get_chunks(test_texts, 5)\n",
    "longer_test_labels = get_chunks(test_labels, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all([len(set(x)) == 1 for x in longer_test_labels])  # Make sure that all combined labels are the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "longer_test_texts = ['\\n'.join(chunk) for chunk in longer_test_texts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "longer_test_labels = [chunk[0] for chunk in longer_test_labels]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(longer_test_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 / 500 .................................................. 0:04:53.823562\n",
      "1 / 500 .................................................. 0:01:29.357596\n",
      "2 / 500 .................................................. 0:01:29.403903\n",
      "3 / 500 .................................................. 0:01:29.457545\n",
      "4 / 500 .................................................. 0:01:29.418732\n",
      "5 / 500 .................................................. 0:01:29.402586\n",
      "6 / 500 .................................................. 0:01:29.338355\n",
      "7 / 500 .................................................. 0:01:29.299863\n",
      "8 / 500 .................................................. 0:01:29.201831\n",
      "9 / 500 .................................................. 0:01:29.215665\n",
      "10 / 500 .................................................. 0:01:29.218467\n",
      "11 / 500 .................................................. 0:01:29.156110\n",
      "12 / 500 .................................................. 0:01:29.178234\n",
      "13 / 500 .................................................. 0:01:29.257610\n",
      "14 / 500 .................................................. 0:01:29.327733\n",
      "15 / 500 .................................................. 0:01:29.421585\n",
      "16 / 500 .................................................. 0:01:29.383154\n",
      "17 / 500 .................................................. 0:01:29.310579\n",
      "18 / 500 .................................................. 0:01:29.286682\n",
      "19 / 500 .................................................. 0:01:29.370293\n",
      "20 / 500 .................................................. 0:01:29.478384\n",
      "21 / 500 .................................................. 0:01:29.339718\n",
      "22 / 500 .................................................. 0:01:29.308610\n",
      "23 / 500 .................................................. 0:01:29.285973\n",
      "24 / 500 .................................................. 0:01:29.284933\n",
      "25 / 500 .................................................. 0:01:29.271699\n",
      "26 / 500 .................................................. 0:01:29.310228\n",
      "27 / 500 .................................................. 0:01:29.279211\n",
      "28 / 500 .................................................. 0:01:29.303016\n",
      "29 / 500 .................................................. 0:01:29.298737\n",
      "30 / 500 .................................................. 0:01:29.152147\n",
      "31 / 500 .................................................. 0:01:29.194914\n",
      "32 / 500 .................................................. 0:01:27.611336\n",
      "33 / 500 .................................................. 0:01:29.188967\n",
      "34 / 500 .................................................. 0:01:29.159004\n",
      "35 / 500 .................................................. 0:01:29.423076\n",
      "36 / 500 .................................................. 0:01:29.170725\n",
      "37 / 500 .................................................. 0:01:29.245325\n",
      "38 / 500 .................................................. 0:01:29.155090\n",
      "39 / 500 .................................................. 0:01:29.187568\n",
      "40 / 500 .................................................. 0:01:27.786165\n",
      "41 / 500 .................................................. 0:01:29.294818\n",
      "42 / 500 .................................................. 0:01:28.379429\n",
      "43 / 500 .................................................. 0:01:31.315155\n",
      "44 / 500 .................................................. 0:01:29.598726\n",
      "45 / 500 .................................................. 0:01:27.999974\n",
      "46 / 500 .................................................. 0:01:29.178190\n",
      "47 / 500 .................................................. 0:01:29.180526\n",
      "48 / 500 .................................................. 0:01:29.261730\n",
      "49 / 500 .................................................. 0:01:29.194141\n",
      "50 / 500 .................................................. 0:01:29.330793\n",
      "51 / 500 .................................................. 0:01:29.300381\n",
      "52 / 500 .................................................. 0:01:29.251712\n",
      "53 / 500 .................................................. 0:01:29.269716\n",
      "54 / 500 .................................................. 0:01:29.280158\n",
      "55 / 500 .................................................. 0:01:29.286157\n",
      "56 / 500 .................................................. 0:01:29.268443\n",
      "57 / 500 .................................................. 0:01:30.518024\n",
      "58 / 500 .................................................. 0:01:31.304159\n",
      "59 / 500 .................................................. 0:01:31.310046\n",
      "60 / 500 .................................................. 0:01:31.321730\n",
      "61 / 500 .................................................. 0:01:31.435841\n",
      "62 / 500 .................................................. 0:01:31.384364\n",
      "63 / 500 .................................................. 0:01:31.420921\n",
      "64 / 500 .................................................. 0:01:31.373423\n",
      "65 / 500 .................................................. 0:01:31.402962\n",
      "66 / 500 .................................................. 0:01:31.332208\n",
      "67 / 500 .................................................. 0:01:31.337926\n",
      "68 / 500 .................................................. 0:01:31.324558\n",
      "69 / 500 .................................................. 0:01:31.294203\n",
      "70 / 500 .................................................. 0:01:31.243505\n",
      "71 / 500 .................................................. 0:01:31.228101\n",
      "72 / 500 .................................................. 0:01:29.596148\n",
      "73 / 500 .................................................. 0:01:29.555263\n",
      "74 / 500 .................................................. 0:01:31.239782\n",
      "75 / 500 .................................................. 0:01:29.546468\n",
      "76 / 500 .................................................. 0:01:29.548159\n",
      "77 / 500 .................................................. 0:01:29.473878\n",
      "78 / 500 .................................................. 0:01:29.461476\n",
      "79 / 500 .................................................. 0:01:31.228102\n",
      "80 / 500 .................................................. 0:01:31.399889\n",
      "81 / 500 .................................................. 0:01:31.331286\n",
      "82 / 500 .................................................. 0:01:31.312273\n",
      "83 / 500 .................................................. 0:01:31.353722\n",
      "84 / 500 .................................................. 0:01:31.370625\n",
      "85 / 500 .................................................. 0:01:31.218542\n",
      "86 / 500 .................................................. 0:01:31.332046\n",
      "87 / 500 .................................................. 0:01:29.336997\n",
      "88 / 500 .................................................. 0:01:29.240632\n",
      "89 / 500 .................................................. 0:01:29.274116\n",
      "90 / 500 .................................................. 0:01:29.326343\n",
      "91 / 500 .................................................. 0:01:29.260267\n",
      "92 / 500 .................................................. 0:01:29.294301\n",
      "93 / 500 .................................................. 0:01:29.276195\n",
      "94 / 500 .................................................. 0:01:30.900014\n",
      "95 / 500 .................................................. 0:01:29.383476\n",
      "96 / 500 .................................................. 0:01:29.396899\n",
      "97 / 500 .................................................. 0:01:29.356665\n",
      "98 / 500 .................................................. 0:01:29.369516\n",
      "99 / 500 .................................................. 0:01:31.275173\n",
      "100 / 500 .................................................. 0:01:31.347635\n",
      "101 / 500 .................................................. 0:01:31.393623\n",
      "102 / 500 .................................................. 0:01:31.347320\n",
      "103 / 500 .................................................. 0:01:31.375181\n",
      "104 / 500 .................................................. 0:01:31.311148\n",
      "105 / 500 .................................................. 0:01:31.387397\n",
      "106 / 500 .................................................. 0:01:31.367275\n",
      "107 / 500 .................................................. 0:01:31.338472\n",
      "108 / 500 .................................................. 0:01:31.328217\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "109 / 500 .................................................. 0:01:31.387294\n",
      "110 / 500 .................................................. 0:01:29.615783\n",
      "111 / 500 .................................................. 0:01:31.243825\n",
      "112 / 500 .................................................. 0:01:31.158846\n",
      "113 / 500 .................................................. 0:01:31.100170\n",
      "114 / 500 .................................................. 0:01:31.254315\n",
      "115 / 500 .................................................. 0:01:31.249152\n",
      "116 / 500 .................................................. 0:01:31.238190\n",
      "117 / 500 .................................................. 0:01:31.247292\n",
      "118 / 500 .................................................. 0:01:31.156884\n",
      "119 / 500 .................................................. 0:01:29.593372\n",
      "120 / 500 .................................................. 0:01:31.333271\n",
      "121 / 500 .................................................. 0:01:31.355232\n",
      "122 / 500 .................................................. 0:01:31.128229\n",
      "123 / 500 .................................................. 0:01:31.173251\n",
      "124 / 500 .................................................. 0:01:31.214446\n",
      "125 / 500 .................................................. 0:01:31.267996\n",
      "126 / 500 .................................................. 0:01:31.282000\n",
      "127 / 500 .................................................. 0:01:31.285395\n",
      "128 / 500 .................................................. 0:01:31.345288\n",
      "129 / 500 .................................................. 0:01:31.306279\n",
      "130 / 500 .................................................. 0:01:31.289196\n",
      "131 / 500 .................................................. 0:01:31.269467\n",
      "132 / 500 .................................................. 0:01:31.285420\n",
      "133 / 500 .................................................. 0:01:31.275581\n",
      "134 / 500 .................................................. 0:01:31.463589\n",
      "135 / 500 .................................................. 0:01:31.358504\n",
      "136 / 500 .................................................. 0:01:31.314517\n",
      "137 / 500 .................................................. 0:01:31.379302\n",
      "138 / 500 .................................................. 0:01:31.361220\n",
      "139 / 500 .................................................. 0:01:31.377806\n",
      "140 / 500 .................................................. 0:01:31.355625\n",
      "141 / 500 .................................................. 0:01:31.397016\n",
      "142 / 500 .................................................. 0:01:31.362150\n",
      "143 / 500 .................................................. 0:01:31.290609\n",
      "144 / 500 .................................................. 0:01:31.338056\n",
      "145 / 500 .................................................. 0:01:31.301164\n",
      "146 / 500 .................................................. 0:01:31.368888\n",
      "147 / 500 .................................................. 0:01:31.384695\n",
      "148 / 500 .................................................. 0:01:31.399871\n",
      "149 / 500 .................................................. 0:01:31.347783\n",
      "150 / 500 .................................................. 0:01:31.315141\n",
      "151 / 500 .................................................. 0:01:29.587088\n",
      "152 / 500 .................................................. 0:01:29.632068\n",
      "153 / 500 .................................................. 0:01:29.634604\n",
      "154 / 500 .................................................. 0:01:31.227712\n",
      "155 / 500 .................................................. 0:01:31.208059\n",
      "156 / 500 .................................................. 0:01:29.620271\n",
      "157 / 500 .................................................. 0:01:31.359682\n",
      "158 / 500 .................................................. 0:01:29.602609\n",
      "159 / 500 .................................................. 0:01:31.230380\n",
      "160 / 500 .................................................. 0:01:31.329277\n",
      "161 / 500 .................................................. 0:01:31.349188\n",
      "162 / 500 .................................................. 0:01:31.279552\n",
      "163 / 500 .................................................. 0:01:31.305497\n",
      "164 / 500 .................................................. 0:01:31.312618\n",
      "165 / 500 .................................................. 0:01:31.264765\n",
      "166 / 500 .................................................. 0:01:31.318770\n",
      "167 / 500 .................................................. 0:01:31.261898\n",
      "168 / 500 .................................................. 0:01:29.651436\n",
      "169 / 500 .................................................. 0:01:29.205188\n",
      "170 / 500 .................................................. 0:01:29.277407\n",
      "171 / 500 .................................................. 0:01:29.245709\n",
      "172 / 500 .................................................. 0:01:29.253370\n",
      "173 / 500 .................................................. 0:01:29.219863\n",
      "174 / 500 .................................................. 0:01:29.154296\n",
      "175 / 500 .................................................. 0:01:29.307466\n",
      "176 / 500 .................................................. 0:01:29.314791\n",
      "177 / 500 .................................................. 0:01:29.423372\n",
      "178 / 500 .................................................. 0:01:29.234599\n",
      "179 / 500 .................................................. 0:01:29.268434\n",
      "180 / 500 .................................................. 0:01:29.249972\n",
      "181 / 500 .................................................. 0:01:29.298749\n",
      "182 / 500 .................................................. 0:01:29.280976\n",
      "183 / 500 .................................................. 0:01:29.219671\n",
      "184 / 500 .................................................. 0:01:29.286034\n",
      "185 / 500 .................................................. 0:01:29.248134\n",
      "186 / 500 .................................................. 0:01:29.258958\n",
      "187 / 500 .................................................. 0:01:29.237072\n",
      "188 / 500 .................................................. 0:01:29.259329\n",
      "189 / 500 .................................................. 0:01:29.267715\n",
      "190 / 500 .................................................. 0:01:29.155892\n",
      "191 / 500 .................................................. 0:01:29.145314\n",
      "192 / 500 .................................................. 0:01:29.213575\n",
      "193 / 500 .................................................. 0:01:29.210366\n",
      "194 / 500 .................................................. 0:01:29.239404\n",
      "195 / 500 .................................................. 0:01:29.213927\n",
      "196 / 500 .................................................. 0:01:29.264201\n",
      "197 / 500 .................................................. 0:01:29.227647\n",
      "198 / 500 .................................................. 0:01:27.622366\n",
      "199 / 500 .................................................. 0:01:29.242989\n",
      "200 / 500 .................................................. 0:01:29.327959\n",
      "201 / 500 .................................................. 0:01:29.340148\n",
      "202 / 500 .................................................. 0:01:29.368873\n",
      "203 / 500 .................................................. 0:01:29.400102\n",
      "204 / 500 .................................................. 0:01:29.367655\n",
      "205 / 500 .................................................. 0:01:29.294455\n",
      "206 / 500 .................................................. 0:01:29.401753\n",
      "207 / 500 .................................................. 0:01:29.380837\n",
      "208 / 500 .................................................. 0:01:29.327215\n",
      "209 / 500 .................................................. 0:01:29.345573\n",
      "210 / 500 .................................................. 0:01:29.351936\n",
      "211 / 500 .................................................. 0:01:29.274630\n",
      "212 / 500 .................................................. 0:01:29.307538\n",
      "213 / 500 .................................................. 0:01:29.299194\n",
      "214 / 500 .................................................. 0:01:29.258275\n",
      "215 / 500 .................................................. 0:01:29.260551\n",
      "216 / 500 .................................................. 0:01:29.288556\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "217 / 500 .................................................. 0:01:29.283050\n",
      "218 / 500 .................................................. 0:01:29.267942\n",
      "219 / 500 .................................................. 0:01:29.236104\n",
      "220 / 500 .................................................. 0:01:29.339491\n",
      "221 / 500 .................................................. 0:01:29.303076\n",
      "222 / 500 .................................................. 0:01:29.292002\n",
      "223 / 500 .................................................. 0:01:29.386147\n",
      "224 / 500 .................................................. 0:01:29.345570\n",
      "225 / 500 .................................................. 0:01:29.303094\n",
      "226 / 500 .................................................. 0:01:29.352727\n",
      "227 / 500 .................................................. 0:01:29.399909\n",
      "228 / 500 .................................................. 0:01:29.314526\n",
      "229 / 500 .................................................. 0:01:29.354569\n",
      "230 / 500 .................................................. 0:01:29.369909\n",
      "231 / 500 .................................................. 0:01:29.300445\n",
      "232 / 500 .................................................. 0:01:29.322538\n",
      "233 / 500 .................................................. 0:01:29.304046\n",
      "234 / 500 .................................................. 0:01:29.312901\n",
      "235 / 500 .................................................. 0:01:29.246834\n",
      "236 / 500 .................................................. 0:01:29.316472\n",
      "237 / 500 .................................................. 0:01:29.320418\n",
      "238 / 500 .................................................. 0:01:29.324453\n",
      "239 / 500 .................................................. 0:01:29.345535\n",
      "240 / 500 .................................................. 0:01:29.227846\n",
      "241 / 500 .................................................. 0:01:29.260723\n",
      "242 / 500 .................................................. 0:01:29.223342\n",
      "243 / 500 .................................................. 0:01:29.236894\n",
      "244 / 500 .................................................. 0:01:29.186312\n",
      "245 / 500 .................................................. 0:01:29.223405\n",
      "246 / 500 .................................................. 0:01:29.222177\n",
      "247 / 500 .................................................. 0:01:29.232753\n",
      "248 / 500 .................................................. 0:01:29.178040\n",
      "249 / 500 .................................................. 0:01:29.189717\n",
      "250 / 500 .................................................. 0:01:29.293353\n",
      "251 / 500 .................................................. 0:01:29.267097\n",
      "252 / 500 .................................................. 0:01:29.311062\n",
      "253 / 500 .................................................. 0:01:29.342833\n",
      "254 / 500 .................................................. 0:01:29.308097\n",
      "255 / 500 .................................................. 0:01:29.272052\n",
      "256 / 500 .................................................. 0:01:29.302788\n",
      "257 / 500 .................................................. 0:01:29.298758\n",
      "258 / 500 .................................................. 0:01:29.339742\n",
      "259 / 500 .................................................. 0:01:29.267198\n",
      "260 / 500 .................................................. 0:01:27.563043\n",
      "261 / 500 .................................................. 0:01:27.573848\n",
      "262 / 500 .................................................. 0:01:27.517535\n",
      "263 / 500 .................................................. 0:01:29.250453\n",
      "264 / 500 .................................................. 0:01:27.635219\n",
      "265 / 500 .................................................. 0:01:27.542995\n",
      "266 / 500 .................................................. 0:01:27.557443\n",
      "267 / 500 .................................................. 0:01:29.194596\n",
      "268 / 500 .................................................. 0:01:27.486810\n",
      "269 / 500 .................................................. 0:01:27.576657\n",
      "270 / 500 .................................................. 0:01:29.320985\n",
      "271 / 500 .................................................. 0:01:29.321164\n",
      "272 / 500 .................................................. 0:01:29.256303\n",
      "273 / 500 .................................................. 0:01:29.257792\n",
      "274 / 500 .................................................. 0:01:29.243413\n",
      "275 / 500 .................................................. 0:01:25.863163\n",
      "276 / 500 .................................................. 0:01:29.310716\n",
      "277 / 500 .................................................. 0:01:29.259753\n",
      "278 / 500 .................................................. 0:01:29.241048\n",
      "279 / 500 .................................................. 0:01:29.175836\n",
      "280 / 500 .................................................. 0:01:29.384074\n",
      "281 / 500 .................................................. 0:01:29.311895\n",
      "282 / 500 .................................................. 0:01:29.325466\n",
      "283 / 500 .................................................. 0:01:29.328740\n",
      "284 / 500 .................................................. 0:01:29.322017\n",
      "285 / 500 .................................................. 0:01:29.330248\n",
      "286 / 500 .................................................. 0:01:29.307551\n",
      "287 / 500 .................................................. 0:01:29.271178\n",
      "288 / 500 .................................................. 0:01:29.284269\n",
      "289 / 500 .................................................. 0:01:29.319438\n",
      "290 / 500 .................................................. 0:01:29.314952\n",
      "291 / 500 .................................................. 0:01:29.244089\n",
      "292 / 500 .................................................. 0:01:29.244502\n",
      "293 / 500 .................................................. 0:01:29.248988\n",
      "294 / 500 .................................................. 0:01:29.238110\n",
      "295 / 500 .................................................. 0:01:29.185845\n",
      "296 / 500 .................................................. 0:01:29.245222\n",
      "297 / 500 .................................................. 0:01:29.261594\n",
      "298 / 500 .................................................. 0:01:29.230911\n",
      "299 / 500 .................................................. 0:01:29.256073\n",
      "300 / 500 .................................................. 0:01:29.217887\n",
      "301 / 500 .................................................. 0:01:29.212577\n",
      "302 / 500 .................................................. 0:01:29.252750\n",
      "303 / 500 .................................................. 0:01:29.239440\n",
      "304 / 500 .................................................. 0:01:29.169651\n",
      "305 / 500 .................................................. 0:01:29.185114\n",
      "306 / 500 .................................................. 0:01:29.279152\n",
      "307 / 500 .................................................. 0:01:29.219308\n",
      "308 / 500 .................................................. 0:01:29.243046\n",
      "309 / 500 .................................................. 0:01:29.253693\n",
      "310 / 500 .................................................. 0:01:29.300793\n",
      "311 / 500 .................................................. 0:01:29.289700\n",
      "312 / 500 .................................................. 0:01:29.282394\n",
      "313 / 500 .................................................. 0:01:29.278067\n",
      "314 / 500 .................................................. 0:01:29.319170\n",
      "315 / 500 .................................................. 0:01:29.307819\n",
      "316 / 500 .................................................. 0:01:29.342382\n",
      "317 / 500 .................................................. 0:01:29.312018\n",
      "318 / 500 .................................................. 0:01:29.348257\n",
      "319 / 500 .................................................. 0:01:29.312360\n",
      "320 / 500 .................................................. 0:01:29.351458\n",
      "321 / 500 .................................................. 0:01:29.398470\n",
      "322 / 500 .................................................. 0:01:29.359452\n",
      "323 / 500 .................................................. 0:01:29.394396\n",
      "324 / 500 .................................................. 0:01:29.303149\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "325 / 500 .................................................. 0:01:29.344109\n",
      "326 / 500 .................................................. 0:01:29.301312\n",
      "327 / 500 .................................................. 0:01:29.301086\n",
      "328 / 500 .................................................. 0:01:29.316883\n",
      "329 / 500 .................................................. 0:01:29.308441\n",
      "330 / 500 .................................................. 0:01:29.284104\n",
      "331 / 500 .................................................. 0:01:29.326629\n",
      "332 / 500 .................................................. 0:01:29.258847\n",
      "333 / 500 .................................................. 0:01:29.319286\n",
      "334 / 500 .................................................. 0:01:29.292448\n",
      "335 / 500 .................................................. 0:01:29.359710\n",
      "336 / 500 .................................................. 0:01:29.329391\n",
      "337 / 500 .................................................. 0:01:29.306317\n",
      "338 / 500 .................................................. 0:01:29.334338\n",
      "339 / 500 .................................................. 0:01:29.247117\n",
      "340 / 500 .................................................. 0:01:29.322743\n",
      "341 / 500 .................................................. 0:01:29.301092\n",
      "342 / 500 .................................................. 0:01:29.268474\n",
      "343 / 500 .................................................. 0:01:29.284623\n",
      "344 / 500 .................................................. 0:01:29.314226\n",
      "345 / 500 .................................................. 0:01:29.290758\n",
      "346 / 500 .................................................. 0:01:29.288761\n",
      "347 / 500 .................................................. 0:01:29.296042\n",
      "348 / 500 .................................................. 0:01:29.282794\n",
      "349 / 500 .................................................. 0:01:29.310809\n",
      "350 / 500 .................................................. 0:01:29.354275\n",
      "351 / 500 .................................................. 0:01:29.366044\n",
      "352 / 500 .................................................. 0:01:29.370009\n",
      "353 / 500 .................................................. 0:01:29.330839\n",
      "354 / 500 .................................................. 0:01:29.390903\n",
      "355 / 500 .................................................. 0:01:29.363490\n",
      "356 / 500 .................................................. 0:01:29.321908\n",
      "357 / 500 .................................................. 0:01:29.327229\n",
      "358 / 500 .................................................. 0:01:29.353005\n",
      "359 / 500 .................................................. 0:01:29.294711\n",
      "360 / 500 .................................................. 0:01:29.173945\n",
      "361 / 500 .................................................. 0:01:29.180068\n",
      "362 / 500 .................................................. 0:01:29.142864\n",
      "363 / 500 .................................................. 0:01:29.188484\n",
      "364 / 500 .................................................. 0:01:29.206523\n",
      "365 / 500 .................................................. 0:01:27.561126\n",
      "366 / 500 .................................................. 0:01:29.205199\n",
      "367 / 500 .................................................. 0:01:29.214065\n",
      "368 / 500 .................................................. 0:01:29.204250\n",
      "369 / 500 .................................................. 0:01:29.138843\n",
      "370 / 500 .................................................. 0:01:29.322548\n",
      "371 / 500 .................................................. 0:01:29.362303\n",
      "372 / 500 .................................................. 0:01:29.364270\n",
      "373 / 500 .................................................. 0:01:29.261262\n",
      "374 / 500 .................................................. 0:01:29.318731\n",
      "375 / 500 .................................................. 0:01:29.288769\n",
      "376 / 500 .................................................. 0:01:29.342162\n",
      "377 / 500 .................................................. 0:01:29.344104\n",
      "378 / 500 .................................................. 0:01:29.330373\n",
      "379 / 500 .................................................. 0:01:29.320237\n",
      "380 / 500 .................................................. 0:01:27.638689\n",
      "381 / 500 .................................................. 0:01:29.203159\n",
      "382 / 500 .................................................. 0:01:29.224186\n",
      "383 / 500 .................................................. 0:01:29.216281\n",
      "384 / 500 .................................................. 0:01:29.257356\n",
      "385 / 500 .................................................. 0:01:27.597768\n",
      "386 / 500 .................................................. 0:01:29.175791\n",
      "387 / 500 .................................................. 0:01:29.231160\n",
      "388 / 500 .................................................. 0:01:29.124025\n",
      "389 / 500 .................................................. 0:01:29.186450\n",
      "390 / 500 .................................................. 0:01:29.267552\n",
      "391 / 500 .................................................. 0:01:29.323006\n",
      "392 / 500 .................................................. 0:01:29.307764\n",
      "393 / 500 .................................................. 0:01:29.297739\n",
      "394 / 500 .................................................. 0:01:29.356474\n",
      "395 / 500 .................................................. 0:01:29.318663\n",
      "396 / 500 .................................................. 0:01:29.347881\n",
      "397 / 500 .................................................. 0:01:29.279016\n",
      "398 / 500 .................................................. 0:01:29.318094\n",
      "399 / 500 .................................................. 0:01:29.307927\n",
      "400 / 500 .................................................. 0:01:29.246609\n",
      "401 / 500 .................................................. 0:01:29.189196\n",
      "402 / 500 .................................................. 0:01:29.260609\n",
      "403 / 500 .................................................. 0:01:29.207177\n",
      "404 / 500 .................................................. 0:01:29.244198\n",
      "405 / 500 .................................................. 0:01:29.223137\n",
      "406 / 500 .................................................. 0:01:29.247506\n",
      "407 / 500 .................................................. 0:01:29.221299\n",
      "408 / 500 .................................................. 0:01:29.221892\n",
      "409 / 500 .................................................. 0:01:29.237495\n",
      "410 / 500 .................................................. 0:01:27.577317\n",
      "411 / 500 .................................................. 0:01:29.184525\n",
      "412 / 500 .................................................. 0:01:29.274179\n",
      "413 / 500 .................................................. 0:01:29.242207\n",
      "414 / 500 .................................................. 0:01:29.277844\n",
      "415 / 500 .................................................. 0:01:29.239298\n",
      "416 / 500 .................................................. 0:01:29.229594\n",
      "417 / 500 .................................................. 0:01:29.178642\n",
      "418 / 500 .................................................. 0:01:29.232881\n",
      "419 / 500 .................................................. 0:01:29.242483\n",
      "420 / 500 .................................................. 0:01:29.289688\n",
      "421 / 500 .................................................. 0:01:29.342292\n",
      "422 / 500 .................................................. 0:01:29.338981\n",
      "423 / 500 .................................................. 0:01:29.297820\n",
      "424 / 500 .................................................. 0:01:29.349288\n",
      "425 / 500 .................................................. 0:01:29.286795\n",
      "426 / 500 .................................................. 0:01:29.307077\n",
      "427 / 500 .................................................. 0:01:29.291014\n",
      "428 / 500 .................................................. 0:01:29.295856\n",
      "429 / 500 .................................................. 0:01:29.308290\n",
      "430 / 500 .................................................. 0:01:29.155027\n",
      "431 / 500 .................................................. 0:01:29.189869\n",
      "432 / 500 .................................................. 0:01:29.211507\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "433 / 500 .................................................. 0:01:27.590447\n",
      "434 / 500 .................................................. 0:01:29.202817\n",
      "435 / 500 .................................................. 0:01:29.217068\n",
      "436 / 500 .................................................. 0:01:29.159146\n",
      "437 / 500 .................................................. 0:01:29.212964\n",
      "438 / 500 .................................................. 0:01:29.202807\n",
      "439 / 500 .................................................. 0:01:29.190238\n",
      "440 / 500 .................................................. 0:01:29.321917\n",
      "441 / 500 .................................................. 0:01:29.324443\n",
      "442 / 500 .................................................. 0:01:29.339277\n",
      "443 / 500 .................................................. 0:01:29.297215\n",
      "444 / 500 .................................................. 0:01:29.357958\n",
      "445 / 500 .................................................. 0:01:29.317098\n",
      "446 / 500 .................................................. 0:01:29.358255\n",
      "447 / 500 .................................................. 0:01:29.318267\n",
      "448 / 500 .................................................. 0:01:29.359730\n",
      "449 / 500 .................................................. 0:01:29.379333\n",
      "450 / 500 .................................................. 0:01:29.238592\n",
      "451 / 500 .................................................. 0:01:29.290573\n",
      "452 / 500 .................................................. 0:01:29.263526\n",
      "453 / 500 .................................................. 0:01:29.278508\n",
      "454 / 500 .................................................. 0:01:29.315881\n",
      "455 / 500 .................................................. 0:01:29.246069\n",
      "456 / 500 .................................................. 0:01:29.223428\n",
      "457 / 500 .................................................. 0:01:29.261961\n",
      "458 / 500 .................................................. 0:01:29.254578\n",
      "459 / 500 .................................................. 0:01:29.220020\n",
      "460 / 500 .................................................. 0:01:29.311067\n",
      "461 / 500 .................................................. 0:01:29.239864\n",
      "462 / 500 .................................................. 0:01:29.262583\n",
      "463 / 500 .................................................. 0:01:29.231929\n",
      "464 / 500 .................................................. 0:01:29.267904\n",
      "465 / 500 .................................................. 0:01:29.538600\n",
      "466 / 500 .................................................. 0:01:29.251325\n",
      "467 / 500 .................................................. 0:01:29.284328\n",
      "468 / 500 .................................................. 0:01:29.276368\n",
      "469 / 500 .................................................. 0:01:29.247795\n",
      "470 / 500 .................................................. 0:01:29.286557\n",
      "471 / 500 .................................................. 0:01:29.301285\n",
      "472 / 500 .................................................. 0:01:29.350276\n",
      "473 / 500 .................................................. 0:01:29.365295\n",
      "474 / 500 .................................................. 0:01:29.309347\n",
      "475 / 500 .................................................. 0:01:29.239152\n",
      "476 / 500 .................................................. 0:01:29.287929\n",
      "477 / 500 .................................................. 0:01:29.332753\n",
      "478 / 500 .................................................. 0:01:29.335129\n",
      "479 / 500 .................................................. 0:01:29.332404\n",
      "480 / 500 .................................................. 0:01:29.151002\n",
      "481 / 500 .................................................. 0:01:27.592865\n",
      "482 / 500 .................................................. 0:01:27.529685\n",
      "483 / 500 .................................................. 0:01:27.602626\n",
      "484 / 500 .................................................. 0:01:29.196841\n",
      "485 / 500 .................................................. 0:01:27.607677\n",
      "486 / 500 .................................................. 0:01:27.557596\n",
      "487 / 500 .................................................. 0:01:27.551886\n",
      "488 / 500 .................................................. 0:01:27.575954\n",
      "489 / 500 .................................................. 0:01:27.604644\n",
      "490 / 500 .................................................. 0:01:29.236734\n",
      "491 / 500 .................................................. 0:01:29.286255\n",
      "492 / 500 .................................................. 0:01:29.448514\n",
      "493 / 500 .................................................. 0:01:29.236132\n",
      "494 / 500 .................................................. 0:01:29.252972\n",
      "495 / 500 .................................................. 0:01:29.261562\n",
      "496 / 500 .................................................. 0:01:29.253702\n",
      "497 / 500 .................................................. 0:01:29.218957\n",
      "498 / 500 .................................................. 0:01:29.291095\n",
      "499 / 500 .................................................. 0:01:29.183988\n",
      "CPU times: user 10h 55min 8s, sys: 20.7 s, total: 10h 55min 28s\n",
      "Wall time: 12h 29min 40s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from random import shuffle\n",
    "from datetime import datetime\n",
    "\n",
    "def get_predictions(author_models, test_texts, test_labels):\n",
    "    \"\"\"Evaluate each text for each author_model and append first metric to predictions\"\"\"\n",
    "    indicies = list(range(len(test_texts)))\n",
    "\n",
    "    test_texts = np.array(test_texts)\n",
    "    test_labels = np.array(test_labels)\n",
    "\n",
    "    test_texts = test_texts[indicies]\n",
    "    test_labels = test_labels[indicies]\n",
    "\n",
    "    predictions = []\n",
    "    for i, text in enumerate(test_texts):\n",
    "        t1 = datetime.now()\n",
    "        print(\"{} / {}\".format(i, len(test_texts)), end=\" \")\n",
    "        X, y = vectorize(clean_text(text, charset))\n",
    "\n",
    "        losses = []\n",
    "        for am in author_models:\n",
    "            print(\".\", end=\"\")\n",
    "            model = am[0]\n",
    "            label = am[1]\n",
    "            loss = model.evaluate(X, y, verbose=0)\n",
    "            losses.append((loss, label))\n",
    "        print(\" {}\".format(datetime.now() - t1))\n",
    "        predictions.append(losses)\n",
    "    return predictions\n",
    "    \n",
    "\n",
    "predictions_long = get_predictions(author_models, longer_test_texts, longer_test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(predictions_long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X\n"
     ]
    }
   ],
   "source": [
    "print(\"X\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_is = []\n",
    "for pred in predictions_long:\n",
    "    pred_i = [p[0] for p in pred]\n",
    "    pred_is.append(pred_i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_labs = [np.argmin(pred) for pred in pred_is]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.90000000000000002"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "accuracy_score(longer_test_labels, pred_labs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_author_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_author_model.evaluatei"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "len(train_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "len(train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import FeatureUnion\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "word_vec = TfidfVectorizer(min_df=5, ngram_range=(1,2), lowercase=False)\n",
    "char_vec = TfidfVectorizer(min_df=5, ngram_range=(2,3), lowercase=False)\n",
    "\n",
    "fu = FeatureUnion([\n",
    "    ('word', word_vec),\n",
    "    ('char', char_vec)\n",
    "])\n",
    "\n",
    "\n",
    "X_train = fu.fit_transform(train_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test = fu.transform(test_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "svm = LinearSVC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
       "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
       "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
       "     verbose=0)"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm.fit(X_train, train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "preds = svm.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.44600000000000001"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(test_labels, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test_longer = fu.transform(longer_test_texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "preds = svm.predict(X_test_longer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.88400000000000001"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(longer_test_labels, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16400, 100)"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X, y = vectorize(clean_text(train_texts[3], charset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_author_model.layers[:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 14725 samples, validate on 1637 samples\n",
      "Epoch 1/13\n",
      "14725/14725 [==============================] - 18s - loss: 3.1623 - val_loss: 3.9301\n",
      "Epoch 2/13\n",
      "14725/14725 [==============================] - 15s - loss: 2.4849 - val_loss: 3.2724\n",
      "Epoch 3/13\n",
      "14725/14725 [==============================] - 15s - loss: 2.2303 - val_loss: 2.7735\n",
      "Epoch 4/13\n",
      "14725/14725 [==============================] - 15s - loss: 2.0664 - val_loss: 2.3324\n",
      "Epoch 5/13\n",
      "14725/14725 [==============================] - 15s - loss: 1.9487 - val_loss: 2.0202\n",
      "Epoch 6/13\n",
      "14725/14725 [==============================] - 15s - loss: 1.8679 - val_loss: 1.9499\n",
      "Epoch 7/13\n",
      "14725/14725 [==============================] - 15s - loss: 1.7817 - val_loss: 1.9287\n",
      "Epoch 8/13\n",
      "14725/14725 [==============================] - 15s - loss: 1.6960 - val_loss: 1.9165\n",
      "Epoch 9/13\n",
      "14725/14725 [==============================] - 15s - loss: 1.6509 - val_loss: 1.9007\n",
      "Epoch 10/13\n",
      "14725/14725 [==============================] - 15s - loss: 1.5811 - val_loss: 1.8963\n",
      "Epoch 11/13\n",
      "14725/14725 [==============================] - 15s - loss: 1.5102 - val_loss: 1.8982\n",
      "Epoch 12/13\n",
      "14725/14725 [==============================] - 15s - loss: 1.4347 - val_loss: 1.9258\n",
      "Epoch 13/13\n",
      "14725/14725 [==============================] - 15s - loss: 1.3818 - val_loss: 1.9303\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"\\nLSTM, BatchNorm\\nTrain on 14929 samples, validate on 1659 samples\\nEpoch 1/5\\n14929/14929 [==============================] - 75s - loss: 3.0403 - val_loss: 3.4757\\nEpoch 2/5\\n14929/14929 [==============================] - 60s - loss: 2.3156 - val_loss: 2.9687\\n\\nLSTM\\nTrain on 14929 samples, validate on 1659 samples\\nEpoch 1/5\\n14929/14929 [==============================] - 75s - loss: 3.1259 - val_loss: 2.7865\\nEpoch 2/5\\n14929/14929 [==============================] - 60s - loss: 2.6150 - val_loss: 2.3894\\n\\nCNN(5), LSTM  # faster, needs more epochs\\nTrain on 14929 samples, validate on 1659 samples\\nEpoch 1/5\\n14929/14929 [==============================] - 42s - loss: 3.1579 - val_loss: 2.9987\\nEpoch 2/5\\n14929/14929 [==============================] - 26s - loss: 2.8874 - val_loss: 2.6994\\nEpoch 3/5\\n14929/14929 [==============================] - 26s - loss: 2.6220 - val_loss: 2.4879\\nEpoch 4/5\\n14929/14929 [==============================] - 26s - loss: 2.4309 - val_loss: 2.3942\\nEpoch 5/5\\n14929/14929 [==============================] - 26s - loss: 2.2950 - val_loss: 2.2902\\n\\nCNN(5), CNN(3), LSTM doesn't drop below 3.0 in 5 epochs\\n\\n\\nEmbedding, BatchNorm, GRU, BatchNorm\\nTrain on 14725 samples, validate on 1637 samples\\nEpoch 1/20\\n14725/14725 [==============================] - 16s - loss: 2.9240 - val_loss: 3.9516\\nEpoch 2/20\\n14725/14725 [==============================] - 15s - loss: 2.2447 - val_loss: 3.3667\\nEpoch 3/20\\n14725/14725 [==============================] - 15s - loss: 2.0054 - val_loss: 2.8011\\nEpoch 4/20\\n14725/14725 [==============================] - 15s - loss: 1.8388 - val_loss: 2.3477\\nEpoch 5/20\\n14725/14725 [==============================] - 15s - loss: 1.7122 - val_loss: 2.0196\\nEpoch 6/20\\n14725/14725 [==============================] - 15s - loss: 1.6069 - val_loss: 1.9417\\nEpoch 7/20\\n14725/14725 [==============================] - 15s - loss: 1.5044 - val_loss: 1.9541\\nEpoch 8/20\\n14725/14725 [==============================] - 15s - loss: 1.3987 - val_loss: 1.9512\\nEpoch 9/20\\n14725/14725 [==============================] - 15s - loss: 1.2940 - val_loss: 1.9921\\nEpoch 10/20\\n14725/14725 [==============================] - 15s - loss: 1.1850 - val_loss: 2.0424\\n\\nTrain on 14725 samples, validate on 1637 samples\\nEpoch 1/20\\n14725/14725 [==============================] - 17s - loss: 3.2577 - val_loss: 3.8892\\nEpoch 2/20\\n14725/14725 [==============================] - 15s - loss: 2.5444 - val_loss: 3.2364\\nEpoch 3/20\\n14725/14725 [==============================] - 15s - loss: 2.2915 - val_loss: 2.7205\\nEpoch 4/20\\n14725/14725 [==============================] - 15s - loss: 2.1258 - val_loss: 2.3477\\nEpoch 5/20\\n14725/14725 [==============================] - 15s - loss: 2.0219 - val_loss: 2.0588\\nEpoch 6/20\\n14725/14725 [==============================] - 15s - loss: 1.9386 - val_loss: 1.9478\\nEpoch 7/20\\n14725/14725 [==============================] - 15s - loss: 1.8762 - val_loss: 1.9152\\nEpoch 8/20\\n14725/14725 [==============================] - 15s - loss: 1.7994 - val_loss: 1.9076\\nEpoch 9/20\\n14725/14725 [==============================] - 15s - loss: 1.7224 - val_loss: 1.8829\\nEpoch 10/20\\n14725/14725 [==============================] - 15s - loss: 1.6783 - val_loss: 1.9007\\nEpoch 11/20\\n14725/14725 [==============================] - 15s - loss: 1.6154 - val_loss: 1.8910\\nEpoch 12/20\\n14725/14725 [==============================] - 15s - loss: 1.5485 - val_loss: 1.8920\\nEpoch 13/20\\n14725/14725 [==============================] - 15s - loss: 1.4844 - val_loss: 1.9198\\nEpoch 14/20\\n14725/14725 [==============================] - 15s - loss: 1.4291 - val_loss: 1.9193\\nEpoch 15/20\\n14725/14725 [==============================] - 15s - loss: 1.3670 - val_loss: 1.9295\\nEpoch 16/20\\n14725/14725 [==============================] - 15s - loss: 1.3028 - val_loss: 1.9752\\nEpoch 17/20\\n14725/14725 [==============================] - 15s - loss: 1.2321 - val_loss: 1.9969\\nEpoch 18/20\\n14725/14725 [==============================] - 15s - loss: 1.2159 - val_loss: 2.0431\\nEpoch 19/20\\n14725/14725 [==============================] - 15s - loss: 1.1391 - val_loss: 2.0709\\nEpoch 20/20\\n14725/14725 [==============================] - 15s - loss: 1.1321 - val_loss: 2.1172\\n\\n\\nmodel = Sequential()\\nmodel.add(Embedding(input_dim=len(charset), output_dim=100))\\nmodel.add(BatchNormalization())\\nmodel.add(GRU(256))\\nmodel.add(Dropout(0.5))\\nmodel.add(BatchNormalization())\\nmodel.add(Dense(y.shape[1], activation='softmax'))\\nmodel.compile(loss='categorical_crossentropy', optimizer='adam')\\nmodel.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\\nTrain on 14725 samples, validate on 1637 samples\\nEpoch 1/20\\n14725/14725 [==============================] - 17s - loss: 3.2577 - val_loss: 3.8892\\nEpoch 2/20\\n14725/14725 [==============================] - 15s - loss: 2.5444 - val_loss: 3.2364\\nEpoch 3/20\\n14725/14725 [==============================] - 15s - loss: 2.2915 - val_loss: 2.7205\\nEpoch 4/20\\n14725/14725 [==============================] - 15s - loss: 2.1258 - val_loss: 2.3477\\nEpoch 5/20\\n14725/14725 [==============================] - 15s - loss: 2.0219 - val_loss: 2.0588\\nEpoch 6/20\\n14725/14725 [==============================] - 15s - loss: 1.9386 - val_loss: 1.9478\\nEpoch 7/20\\n14725/14725 [==============================] - 15s - loss: 1.8762 - val_loss: 1.9152\\nEpoch 8/20\\n14725/14725 [==============================] - 15s - loss: 1.7994 - val_loss: 1.9076\\nEpoch 9/20\\n14725/14725 [==============================] - 15s - loss: 1.7224 - val_loss: 1.8829\\nEpoch 10/20\\n14725/14725 [==============================] - 15s - loss: 1.6783 - val_loss: 1.9007\\nEpoch 11/20\\n14725/14725 [==============================] - 15s - loss: 1.6154 - val_loss: 1.8910\\nEpoch 12/20\\n14725/14725 [==============================] - 15s - loss: 1.5485 - val_loss: 1.8920\\nEpoch 13/20\\n14725/14725 [==============================] - 15s - loss: 1.4844 - val_loss: 1.9198\\nEpoch 14/20\\n14725/14725 [==============================] - 15s - loss: 1.4291 - val_loss: 1.9193\\nEpoch 15/20\\n14725/14725 [==============================] - 15s - loss: 1.3670 - val_loss: 1.9295\\nEpoch 16/20\\n14725/14725 [==============================] - 15s - loss: 1.3028 - val_loss: 1.9752\\nEpoch 17/20\\n14725/14725 [==============================] - 15s - loss: 1.2321 - val_loss: 1.9969\\nEpoch 18/20\\n14725/14725 [==============================] - 15s - loss: 1.2159 - val_loss: 2.0431\\nEpoch 19/20\\n14725/14725 [==============================] - 15s - loss: 1.1391 - val_loss: 2.0709\\nEpoch 20/20\\n14725/14725 [==============================] - 15s - loss: 1.1321 - val_loss: 2.1172\\n\\n\\nmodel = Sequential()\\nmodel.add(Embedding(input_dim=len(charset), output_dim=100))\\nmodel.add(GRU(256))\\nmodel.add(BatchNormalization())\\nmodel.add(Dropout(0.5))\\nmodel.add(Dense(y.shape[1], activation='softmax'))\\nmodel.compile(loss='categorical_crossentropy', optimizer='adam')\\nmodel.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\\nTrain on 14725 samples, validate on 1637 samples\\nEpoch 1/20\\n14725/14725 [==============================] - 16s - loss: 3.1731 - val_loss: 3.5648\\nEpoch 2/20\\n14725/14725 [==============================] - 14s - loss: 2.4964 - val_loss: 2.9875\\nEpoch 3/20\\n14725/14725 [==============================] - 14s - loss: 2.3446 - val_loss: 2.7695\\nEpoch 4/20\\n14725/14725 [==============================] - 14s - loss: 2.2928 - val_loss: 2.6010\\nEpoch 5/20\\n14725/14725 [==============================] - 14s - loss: 2.2642 - val_loss: 2.3900\\nEpoch 6/20\\n14725/14725 [==============================] - 14s - loss: 2.2373 - val_loss: 2.5023\\nEpoch 7/20\\n14725/14725 [==============================] - 14s - loss: 2.2186 - val_loss: 2.3780\\nEpoch 8/20\\n14725/14725 [==============================] - 14s - loss: 2.2029 - val_loss: 2.4928\\nEpoch 9/20\\n14725/14725 [==============================] - 14s - loss: 2.1852 - val_loss: 2.3480\\nEpoch 10/20\\n14725/14725 [==============================] - 14s - loss: 2.1745 - val_loss: 2.4801\\nEpoch 11/20\\n14725/14725 [==============================] - 14s - loss: 2.1563 - val_loss: 2.3951\\nEpoch 12/20\\n14725/14725 [==============================] - 14s - loss: 2.1391 - val_loss: 2.4133\\nEpoch 13/20\\n14725/14725 [==============================] - 14s - loss: 2.1192 - val_loss: 2.5896\\nEpoch 14/20\\n14725/14725 [==============================] - 14s - loss: 2.1020 - val_loss: 2.2692\\nEpoch 15/20\\n14725/14725 [==============================] - 14s - loss: 2.0770 - val_loss: 2.2179\\nEpoch 16/20\\n14725/14725 [==============================] - 14s - loss: 2.0643 - val_loss: 2.2822\\nEpoch 17/20\\n 6784/14725 [============>.................] - ETA: 7s - loss: 2.0302\\n\\n\\nmodel = Sequential()\\nmodel.add(Embedding(input_dim=len(charset), output_dim=100))\\nmodel.add(Dropout(0.2))\\nmodel.add(BatchNormalization())\\nmodel.add(GRU(256))\\nmodel.add(Dropout(0.2))\\nmodel.add(BatchNormalization())\\nmodel.add(Dense(y.shape[1], activation='softmax'))\\nmodel.compile(loss='categorical_crossentropy', optimizer='adam')\\nmodel.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\\nTrain on 14725 samples, validate on 1637 samples\\nEpoch 1/20\\n14725/14725 [==============================] - 17s - loss: 3.1528 - val_loss: 3.9042\\nEpoch 2/20\\n14725/14725 [==============================] - 15s - loss: 2.4658 - val_loss: 3.2093\\nEpoch 3/20\\n14725/14725 [==============================] - 15s - loss: 2.2001 - val_loss: 2.6764\\nEpoch 4/20\\n14725/14725 [==============================] - 15s - loss: 2.0358 - val_loss: 2.2919\\nEpoch 5/20\\n14725/14725 [==============================] - 15s - loss: 1.9497 - val_loss: 2.0060\\nEpoch 6/20\\n14725/14725 [==============================] - 15s - loss: 1.8588 - val_loss: 1.9313\\nEpoch 7/20\\n14725/14725 [==============================] - 15s - loss: 1.7883 - val_loss: 1.9153\\nEpoch 8/20\\n14725/14725 [==============================] - 15s - loss: 1.7034 - val_loss: 1.9145\\nEpoch 9/20\\n14725/14725 [==============================] - 15s - loss: 1.6382 - val_loss: 1.8979\\nEpoch 10/20\\n14725/14725 [==============================] - 15s - loss: 1.5827 - val_loss: 1.8864\\nEpoch 11/20\\n14725/14725 [==============================] - 15s - loss: 1.5093 - val_loss: 1.8967\\nEpoch 12/20\\n14725/14725 [==============================] - 15s - loss: 1.4472 - val_loss: 1.9040\\nEpoch 13/20\\n14725/14725 [==============================] - 15s - loss: 1.3809 - val_loss: 1.9227\\nEpoch 14/20\\n14725/14725 [==============================] - 15s - loss: 1.3225 - val_loss: 1.9469\\nEpoch 15/20\\n14725/14725 [==============================] - 15s - loss: 1.2516 - val_loss: 1.9862\\nEpoch 16/20\\n14725/14725 [==============================] - 15s - loss: 1.2094 - val_loss: 1.9963\\nEpoch 17/20\\n14725/14725 [==============================] - 15s - loss: 1.1658 - val_loss: 2.0331\\nEpoch 18/20\\n14725/14725 [==============================] - 15s - loss: 1.0851 - val_loss: 2.0452\\nEpoch 19/20\\n14725/14725 [==============================] - 15s - loss: 1.0394 - val_loss: 2.0810\\nEpoch 20/20\\n14725/14725 [==============================] - 15s - loss: 0.9903 - val_loss: 2.1283\\n\\n\\nmodel = Sequential()\\nmodel.add(Embedding(input_dim=len(charset), output_dim=100))\\nmodel.add(Dropout(0.3))\\nmodel.add(BatchNormalization())\\nmodel.add(GRU(256))\\nmodel.add(Dropout(0.3))\\nmodel.add(BatchNormalization())\\nmodel.add(Dense(y.shape[1], activation='softmax'))\\nmodel.compile(loss='categorical_crossentropy', optimizer='adam')\\nmodel.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\\nTrain on 14725 samples, validate on 1637 samples\\nEpoch 1/20\\n14725/14725 [==============================] - 17s - loss: 3.2991 - val_loss: 3.8902\\nEpoch 2/20\\n14725/14725 [==============================] - 15s - loss: 2.5672 - val_loss: 3.1627\\nEpoch 3/20\\n14725/14725 [==============================] - 15s - loss: 2.2731 - val_loss: 2.6340\\nEpoch 4/20\\n14725/14725 [==============================] - 15s - loss: 2.1316 - val_loss: 2.2594\\nEpoch 5/20\\n14725/14725 [==============================] - 15s - loss: 2.0249 - val_loss: 2.0159\\nEpoch 6/20\\n14725/14725 [==============================] - 15s - loss: 1.9571 - val_loss: 1.9456\\nEpoch 7/20\\n14725/14725 [==============================] - 15s - loss: 1.8789 - val_loss: 1.9213\\nEpoch 8/20\\n14725/14725 [==============================] - 15s - loss: 1.8233 - val_loss: 1.8924\\nEpoch 9/20\\n14725/14725 [==============================] - 15s - loss: 1.7575 - val_loss: 1.8987\\n\\n\\n\\nmodel = Sequential()\\nmodel.add(Embedding(input_dim=len(charset), output_dim=100))\\nmodel.add(Dropout(0.5))\\nmodel.add(BatchNormalization())\\nmodel.add(GRU(256))\\nmodel.add(Dropout(0.5))\\nmodel.add(BatchNormalization())\\nmodel.add(Dense(y.shape[1], activation='softmax'))\\nmodel.compile(loss='categorical_crossentropy', optimizer='adam')\\nmodel.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\\nTrain on 14725 samples, validate on 1637 samples\\nEpoch 1/20\\n14725/14725 [==============================] - 17s - loss: 3.6036 - val_loss: 3.7924\\nEpoch 2/20\\n14725/14725 [==============================] - 15s - loss: 2.7765 - val_loss: 3.0022\\nEpoch 3/20\\n14725/14725 [==============================] - 15s - loss: 2.4773 - val_loss: 2.5697\\nEpoch 4/20\\n14725/14725 [==============================] - 15s - loss: 2.3218 - val_loss: 2.2606\\nEpoch 5/20\\n14725/14725 [==============================] - 15s - loss: 2.2328 - val_loss: 2.0832\\nEpoch 6/20\\n14725/14725 [==============================] - 15s - loss: 2.1748 - val_loss: 2.0248\\nEpoch 7/20\\n14725/14725 [==============================] - 15s - loss: 2.1174 - val_loss: 1.9865\\nEpoch 8/20\\n14725/14725 [==============================] - 15s - loss: 2.0617 - val_loss: 1.9640\\nEpoch 9/20\\n14725/14725 [==============================] - 15s - loss: 2.0206 - val_loss: 1.9461\\nEpoch 10/20\\n14725/14725 [==============================] - 15s - loss: 1.9758 - val_loss: 1.9334\\nEpoch 11/20\\n14725/14725 [==============================] - 15s - loss: 1.9546 - val_loss: 1.9148\\nEpoch 12/20\\n14725/14725 [==============================] - 15s - loss: 1.9045 - val_loss: 1.9121\\nEpoch 13/20\\n14725/14725 [==============================] - 15s - loss: 1.8757 - val_loss: 1.8888\\nEpoch 14/20\\n14725/14725 [==============================] - 15s - loss: 1.8437 - val_loss: 1.8874\\nEpoch 15/20\\n14725/14725 [==============================] - 15s - loss: 1.8145 - val_loss: 1.8822\\nEpoch 16/20\\n14725/14725 [==============================] - 15s - loss: 1.7805 - val_loss: 1.8785\\nEpoch 17/20\\n14725/14725 [==============================] - 15s - loss: 1.7558 - val_loss: 1.8868\\nEpoch 18/20\\n14725/14725 [==============================] - 15s - loss: 1.7218 - val_loss: 1.8670\\nEpoch 19/20\\n14725/14725 [==============================] - 15s - loss: 1.7032 - val_loss: 1.8759\\nEpoch 20/20\\n14725/14725 [==============================] - 15s - loss: 1.6832 - val_loss: 1.8834\\n\\n\\nmodel = Sequential()\\nmodel.add(Embedding(input_dim=len(charset), output_dim=100))\\nmodel.add(Dropout(0.5))\\nmodel.add(BatchNormalization())\\nmodel.add(GRU(256, return_sequences=True))\\nmodel.add(Dropout(0.5))\\nmodel.add(BatchNormalization())\\nmodel.add(GRU(256))\\nmodel.add(Dropout(0.5))\\nmodel.add(BatchNormalization())\\nmodel.add(Dense(y.shape[1], activation='softmax'))\\nmodel.compile(loss='categorical_crossentropy', optimizer='adam')\\nmodel.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\\nTrain on 14725 samples, validate on 1637 samples\\nEpoch 1/20\\n14725/14725 [==============================] - 33s - loss: 3.7814 - val_loss: 3.7282\\nEpoch 2/20\\n14725/14725 [==============================] - 29s - loss: 2.8453 - val_loss: 2.8542\\nEpoch 3/20\\n14725/14725 [==============================] - 29s - loss: 2.5178 - val_loss: 2.4434\\nEpoch 4/20\\n14725/14725 [==============================] - 29s - loss: 2.3762 - val_loss: 2.1894\\nEpoch 5/20\\n14725/14725 [==============================] - 29s - loss: 2.2896 - val_loss: 2.0862\\nEpoch 6/20\\n14725/14725 [==============================] - 29s - loss: 2.2254 - val_loss: 2.0516\\nEpoch 7/20\\n14725/14725 [==============================] - 29s - loss: 2.1565 - val_loss: 2.0133\\nEpoch 8/20\\n14725/14725 [==============================] - 29s - loss: 2.1132 - val_loss: 1.9992\\nEpoch 9/20\\n14725/14725 [==============================] - 29s - loss: 2.0798 - val_loss: 1.9881\\nEpoch 10/20\\n14725/14725 [==============================] - 29s - loss: 2.0509 - val_loss: 1.9784\\nEpoch 11/20\\n14725/14725 [==============================] - 29s - loss: 2.0198 - val_loss: 1.9618\\nEpoch 12/20\\n14725/14725 [==============================] - 29s - loss: 1.9822 - val_loss: 1.9383\\nEpoch 13/20\\n14725/14725 [==============================] - 29s - loss: 1.9437 - val_loss: 1.9300\\nEpoch 14/20\\n14725/14725 [==============================] - 29s - loss: 1.9198 - val_loss: 1.9163\\nEpoch 15/20\\n14725/14725 [==============================] - 29s - loss: 1.8989 - val_loss: 1.9160\\nEpoch 16/20\\n14725/14725 [==============================] - 29s - loss: 1.8866 - val_loss: 1.9085\\nEpoch 17/20\\n14725/14725 [==============================] - 29s - loss: 1.8493 - val_loss: 1.8965\\nEpoch 18/20\\n14725/14725 [==============================] - 29s - loss: 1.8248 - val_loss: 1.8878\\nEpoch 19/20\\n14725/14725 [==============================] - 29s - loss: 1.8037 - val_loss: 1.8870\\nEpoch 20/20\\n14725/14725 [==============================] - 29s - loss: 1.7724 - val_loss: 1.8862\\n\\n\\nmodel = Sequential()\\nmodel.add(Embedding(input_dim=len(charset), output_dim=100))\\nmodel.add(Dropout(0.2))\\nmodel.add(BatchNormalization())\\nmodel.add(GRU(256, return_sequences=True))\\nmodel.add(Dropout(0.2))\\nmodel.add(BatchNormalization())\\nmodel.add(GRU(256))\\nmodel.add(Dropout(0.2))\\nmodel.add(BatchNormalization())\\nmodel.add(Dense(y.shape[1], activation='softmax'))\\nmodel.compile(loss='categorical_crossentropy', optimizer='adam')\\nmodel.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\\nTrain on 14725 samples, validate on 1637 samples\\nEpoch 1/20\\n14725/14725 [==============================] - 32s - loss: 3.2809 - val_loss: 3.7595\\nEpoch 2/20\\n14725/14725 [==============================] - 29s - loss: 2.4379 - val_loss: 2.9869\\nEpoch 3/20\\n14725/14725 [==============================] - 29s - loss: 2.1504 - val_loss: 2.5361\\nEpoch 4/20\\n14725/14725 [==============================] - 29s - loss: 1.9887 - val_loss: 2.1294\\nEpoch 5/20\\n14725/14725 [==============================] - 29s - loss: 1.8984 - val_loss: 1.9727\\nEpoch 6/20\\n14725/14725 [==============================] - 29s - loss: 1.7892 - val_loss: 1.9264\\nEpoch 7/20\\n14725/14725 [==============================] - 29s - loss: 1.7172 - val_loss: 1.9100\\nEpoch 8/20\\n14725/14725 [==============================] - 29s - loss: 1.6361 - val_loss: 1.9124\\nEpoch 9/20\\n14725/14725 [==============================] - 29s - loss: 1.5621 - val_loss: 1.9122\\nEpoch 10/20\\n14725/14725 [==============================] - 29s - loss: 1.4863 - val_loss: 1.9045\\nEpoch 11/20\\n14725/14725 [==============================] - 29s - loss: 1.4150 - val_loss: 1.9278\\nEpoch 12/20\\n14725/14725 [==============================] - 29s - loss: 1.3691 - val_loss: 1.9181\\nEpoch 13/20\\n14725/14725 [==============================] - 29s - loss: 1.2970 - val_loss: 1.9414\\nEpoch 14/20\\n 1536/14725 [==>...........................] - ETA: 25s - loss: 1.1475\\n \\n \\n \\nmodel = Sequential()\\nmodel.add(Embedding(input_dim=len(charset), output_dim=100))\\nmodel.add(BatchNormalization())\\nmodel.add(GRU(256))\\nmodel.add(Dropout(0.4))\\nmodel.add(BatchNormalization())\\nmodel.add(Dense(y.shape[1], activation='softmax'))\\nmodel.compile(loss='categorical_crossentropy', optimizer='adam')\\nmodel.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\\n Train on 14725 samples, validate on 1637 samples\\nEpoch 1/20\\n14725/14725 [==============================] - 17s - loss: 3.1787 - val_loss: 3.9282\\nEpoch 2/20\\n14725/14725 [==============================] - 15s - loss: 2.5070 - val_loss: 3.2479\\nEpoch 3/20\\n14725/14725 [==============================] - 15s - loss: 2.2191 - val_loss: 2.7522\\nEpoch 4/20\\n14725/14725 [==============================] - 15s - loss: 2.0637 - val_loss: 2.3331\\nEpoch 5/20\\n14725/14725 [==============================] - 15s - loss: 1.9539 - val_loss: 2.0326\\nEpoch 6/20\\n14725/14725 [==============================] - 15s - loss: 1.8622 - val_loss: 1.9440\\nEpoch 7/20\\n14725/14725 [==============================] - 15s - loss: 1.7821 - val_loss: 1.9166\\nEpoch 8/20\\n14725/14725 [==============================] - 15s - loss: 1.7169 - val_loss: 1.8996\\nEpoch 9/20\\n14725/14725 [==============================] - 15s - loss: 1.6561 - val_loss: 1.8849\\nEpoch 10/20\\n14725/14725 [==============================] - 15s - loss: 1.5910 - val_loss: 1.9032\\nEpoch 11/20\\n14725/14725 [==============================] - 15s - loss: 1.5082 - val_loss: 1.8878\\nEpoch 12/20\\n14725/14725 [==============================] - 15s - loss: 1.4513 - val_loss: 1.9252\\nEpoch 13/20\\n 7552/14725 [==============>...............] - ETA: 7s - loss: 1.3534\\n \\n \\nmodel = Sequential()\\nmodel.add(Embedding(input_dim=len(charset), output_dim=100))\\nmodel.add(BatchNormalization())\\nmodel.add(GRU(512))\\nmodel.add(Dropout(0.4))\\nmodel.add(BatchNormalization())\\nmodel.add(Dense(y.shape[1], activation='softmax'))\\nmodel.compile(loss='categorical_crossentropy', optimizer='adam')\\nmodel.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\\n Train on 14725 samples, validate on 1637 samples\\nEpoch 1/20\\n14725/14725 [==============================] - 23s - loss: 3.1173 - val_loss: 3.8415\\nEpoch 2/20\\n14725/14725 [==============================] - 20s - loss: 2.4202 - val_loss: 3.1512\\nEpoch 3/20\\n14725/14725 [==============================] - 20s - loss: 2.1302 - val_loss: 2.7191\\nEpoch 4/20\\n14725/14725 [==============================] - 20s - loss: 1.9435 - val_loss: 2.3341\\nEpoch 5/20\\n14725/14725 [==============================] - 20s - loss: 1.7966 - val_loss: 1.9877\\nEpoch 6/20\\n14725/14725 [==============================] - 20s - loss: 1.6621 - val_loss: 1.9349\\nEpoch 7/20\\n14725/14725 [==============================] - 20s - loss: 1.5190 - val_loss: 1.9632\\nEpoch 8/20\\n14725/14725 [==============================] - 20s - loss: 1.3925 - val_loss: 1.9735\\n\""
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import LSTM, Input, Embedding, Conv1D, MaxPooling1D, BatchNormalization, GRU\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.utils import np_utils\n",
    "\n",
    "# load ascii text and covert to lowercase\n",
    "\n",
    "# cnn = Dropout(0.2)(embedded)\n",
    "# cnn = Conv1D(128, 5, activation='relu')(cnn)\n",
    "# cnn = MaxPooling1D(pool_size=4)(cnn)\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=len(charset), output_dim=100))\n",
    "model.add(Dropout(0.1))\n",
    "model.add(BatchNormalization())\n",
    "model.add(GRU(256))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "model.fit(X, y, epochs=13, batch_size=128, validation_split=0.1)\n",
    "\n",
    "\"\"\"\n",
    "LSTM, BatchNorm\n",
    "Train on 14929 samples, validate on 1659 samples\n",
    "Epoch 1/5\n",
    "14929/14929 [==============================] - 75s - loss: 3.0403 - val_loss: 3.4757\n",
    "Epoch 2/5\n",
    "14929/14929 [==============================] - 60s - loss: 2.3156 - val_loss: 2.9687\n",
    "\n",
    "LSTM\n",
    "Train on 14929 samples, validate on 1659 samples\n",
    "Epoch 1/5\n",
    "14929/14929 [==============================] - 75s - loss: 3.1259 - val_loss: 2.7865\n",
    "Epoch 2/5\n",
    "14929/14929 [==============================] - 60s - loss: 2.6150 - val_loss: 2.3894\n",
    "\n",
    "CNN(5), LSTM  # faster, needs more epochs\n",
    "Train on 14929 samples, validate on 1659 samples\n",
    "Epoch 1/5\n",
    "14929/14929 [==============================] - 42s - loss: 3.1579 - val_loss: 2.9987\n",
    "Epoch 2/5\n",
    "14929/14929 [==============================] - 26s - loss: 2.8874 - val_loss: 2.6994\n",
    "Epoch 3/5\n",
    "14929/14929 [==============================] - 26s - loss: 2.6220 - val_loss: 2.4879\n",
    "Epoch 4/5\n",
    "14929/14929 [==============================] - 26s - loss: 2.4309 - val_loss: 2.3942\n",
    "Epoch 5/5\n",
    "14929/14929 [==============================] - 26s - loss: 2.2950 - val_loss: 2.2902\n",
    "\n",
    "CNN(5), CNN(3), LSTM doesn't drop below 3.0 in 5 epochs\n",
    "\n",
    "\n",
    "Embedding, BatchNorm, GRU, BatchNorm\n",
    "Train on 14725 samples, validate on 1637 samples\n",
    "Epoch 1/20\n",
    "14725/14725 [==============================] - 16s - loss: 2.9240 - val_loss: 3.9516\n",
    "Epoch 2/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.2447 - val_loss: 3.3667\n",
    "Epoch 3/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.0054 - val_loss: 2.8011\n",
    "Epoch 4/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.8388 - val_loss: 2.3477\n",
    "Epoch 5/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.7122 - val_loss: 2.0196\n",
    "Epoch 6/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.6069 - val_loss: 1.9417\n",
    "Epoch 7/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.5044 - val_loss: 1.9541\n",
    "Epoch 8/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.3987 - val_loss: 1.9512\n",
    "Epoch 9/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.2940 - val_loss: 1.9921\n",
    "Epoch 10/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.1850 - val_loss: 2.0424\n",
    "\n",
    "Train on 14725 samples, validate on 1637 samples\n",
    "Epoch 1/20\n",
    "14725/14725 [==============================] - 17s - loss: 3.2577 - val_loss: 3.8892\n",
    "Epoch 2/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.5444 - val_loss: 3.2364\n",
    "Epoch 3/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.2915 - val_loss: 2.7205\n",
    "Epoch 4/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.1258 - val_loss: 2.3477\n",
    "Epoch 5/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.0219 - val_loss: 2.0588\n",
    "Epoch 6/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.9386 - val_loss: 1.9478\n",
    "Epoch 7/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.8762 - val_loss: 1.9152\n",
    "Epoch 8/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.7994 - val_loss: 1.9076\n",
    "Epoch 9/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.7224 - val_loss: 1.8829\n",
    "Epoch 10/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.6783 - val_loss: 1.9007\n",
    "Epoch 11/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.6154 - val_loss: 1.8910\n",
    "Epoch 12/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.5485 - val_loss: 1.8920\n",
    "Epoch 13/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.4844 - val_loss: 1.9198\n",
    "Epoch 14/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.4291 - val_loss: 1.9193\n",
    "Epoch 15/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.3670 - val_loss: 1.9295\n",
    "Epoch 16/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.3028 - val_loss: 1.9752\n",
    "Epoch 17/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.2321 - val_loss: 1.9969\n",
    "Epoch 18/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.2159 - val_loss: 2.0431\n",
    "Epoch 19/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.1391 - val_loss: 2.0709\n",
    "Epoch 20/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.1321 - val_loss: 2.1172\n",
    "\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=len(charset), output_dim=100))\n",
    "model.add(BatchNormalization())\n",
    "model.add(GRU(256))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "model.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\n",
    "Train on 14725 samples, validate on 1637 samples\n",
    "Epoch 1/20\n",
    "14725/14725 [==============================] - 17s - loss: 3.2577 - val_loss: 3.8892\n",
    "Epoch 2/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.5444 - val_loss: 3.2364\n",
    "Epoch 3/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.2915 - val_loss: 2.7205\n",
    "Epoch 4/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.1258 - val_loss: 2.3477\n",
    "Epoch 5/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.0219 - val_loss: 2.0588\n",
    "Epoch 6/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.9386 - val_loss: 1.9478\n",
    "Epoch 7/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.8762 - val_loss: 1.9152\n",
    "Epoch 8/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.7994 - val_loss: 1.9076\n",
    "Epoch 9/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.7224 - val_loss: 1.8829\n",
    "Epoch 10/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.6783 - val_loss: 1.9007\n",
    "Epoch 11/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.6154 - val_loss: 1.8910\n",
    "Epoch 12/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.5485 - val_loss: 1.8920\n",
    "Epoch 13/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.4844 - val_loss: 1.9198\n",
    "Epoch 14/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.4291 - val_loss: 1.9193\n",
    "Epoch 15/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.3670 - val_loss: 1.9295\n",
    "Epoch 16/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.3028 - val_loss: 1.9752\n",
    "Epoch 17/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.2321 - val_loss: 1.9969\n",
    "Epoch 18/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.2159 - val_loss: 2.0431\n",
    "Epoch 19/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.1391 - val_loss: 2.0709\n",
    "Epoch 20/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.1321 - val_loss: 2.1172\n",
    "\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=len(charset), output_dim=100))\n",
    "model.add(GRU(256))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "model.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\n",
    "Train on 14725 samples, validate on 1637 samples\n",
    "Epoch 1/20\n",
    "14725/14725 [==============================] - 16s - loss: 3.1731 - val_loss: 3.5648\n",
    "Epoch 2/20\n",
    "14725/14725 [==============================] - 14s - loss: 2.4964 - val_loss: 2.9875\n",
    "Epoch 3/20\n",
    "14725/14725 [==============================] - 14s - loss: 2.3446 - val_loss: 2.7695\n",
    "Epoch 4/20\n",
    "14725/14725 [==============================] - 14s - loss: 2.2928 - val_loss: 2.6010\n",
    "Epoch 5/20\n",
    "14725/14725 [==============================] - 14s - loss: 2.2642 - val_loss: 2.3900\n",
    "Epoch 6/20\n",
    "14725/14725 [==============================] - 14s - loss: 2.2373 - val_loss: 2.5023\n",
    "Epoch 7/20\n",
    "14725/14725 [==============================] - 14s - loss: 2.2186 - val_loss: 2.3780\n",
    "Epoch 8/20\n",
    "14725/14725 [==============================] - 14s - loss: 2.2029 - val_loss: 2.4928\n",
    "Epoch 9/20\n",
    "14725/14725 [==============================] - 14s - loss: 2.1852 - val_loss: 2.3480\n",
    "Epoch 10/20\n",
    "14725/14725 [==============================] - 14s - loss: 2.1745 - val_loss: 2.4801\n",
    "Epoch 11/20\n",
    "14725/14725 [==============================] - 14s - loss: 2.1563 - val_loss: 2.3951\n",
    "Epoch 12/20\n",
    "14725/14725 [==============================] - 14s - loss: 2.1391 - val_loss: 2.4133\n",
    "Epoch 13/20\n",
    "14725/14725 [==============================] - 14s - loss: 2.1192 - val_loss: 2.5896\n",
    "Epoch 14/20\n",
    "14725/14725 [==============================] - 14s - loss: 2.1020 - val_loss: 2.2692\n",
    "Epoch 15/20\n",
    "14725/14725 [==============================] - 14s - loss: 2.0770 - val_loss: 2.2179\n",
    "Epoch 16/20\n",
    "14725/14725 [==============================] - 14s - loss: 2.0643 - val_loss: 2.2822\n",
    "Epoch 17/20\n",
    " 6784/14725 [============>.................] - ETA: 7s - loss: 2.0302\n",
    "\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=len(charset), output_dim=100))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(BatchNormalization())\n",
    "model.add(GRU(256))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "model.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\n",
    "Train on 14725 samples, validate on 1637 samples\n",
    "Epoch 1/20\n",
    "14725/14725 [==============================] - 17s - loss: 3.1528 - val_loss: 3.9042\n",
    "Epoch 2/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.4658 - val_loss: 3.2093\n",
    "Epoch 3/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.2001 - val_loss: 2.6764\n",
    "Epoch 4/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.0358 - val_loss: 2.2919\n",
    "Epoch 5/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.9497 - val_loss: 2.0060\n",
    "Epoch 6/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.8588 - val_loss: 1.9313\n",
    "Epoch 7/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.7883 - val_loss: 1.9153\n",
    "Epoch 8/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.7034 - val_loss: 1.9145\n",
    "Epoch 9/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.6382 - val_loss: 1.8979\n",
    "Epoch 10/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.5827 - val_loss: 1.8864\n",
    "Epoch 11/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.5093 - val_loss: 1.8967\n",
    "Epoch 12/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.4472 - val_loss: 1.9040\n",
    "Epoch 13/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.3809 - val_loss: 1.9227\n",
    "Epoch 14/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.3225 - val_loss: 1.9469\n",
    "Epoch 15/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.2516 - val_loss: 1.9862\n",
    "Epoch 16/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.2094 - val_loss: 1.9963\n",
    "Epoch 17/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.1658 - val_loss: 2.0331\n",
    "Epoch 18/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.0851 - val_loss: 2.0452\n",
    "Epoch 19/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.0394 - val_loss: 2.0810\n",
    "Epoch 20/20\n",
    "14725/14725 [==============================] - 15s - loss: 0.9903 - val_loss: 2.1283\n",
    "\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=len(charset), output_dim=100))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(BatchNormalization())\n",
    "model.add(GRU(256))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "model.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\n",
    "Train on 14725 samples, validate on 1637 samples\n",
    "Epoch 1/20\n",
    "14725/14725 [==============================] - 17s - loss: 3.2991 - val_loss: 3.8902\n",
    "Epoch 2/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.5672 - val_loss: 3.1627\n",
    "Epoch 3/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.2731 - val_loss: 2.6340\n",
    "Epoch 4/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.1316 - val_loss: 2.2594\n",
    "Epoch 5/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.0249 - val_loss: 2.0159\n",
    "Epoch 6/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.9571 - val_loss: 1.9456\n",
    "Epoch 7/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.8789 - val_loss: 1.9213\n",
    "Epoch 8/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.8233 - val_loss: 1.8924\n",
    "Epoch 9/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.7575 - val_loss: 1.8987\n",
    "\n",
    "\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=len(charset), output_dim=100))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(BatchNormalization())\n",
    "model.add(GRU(256))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "model.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\n",
    "Train on 14725 samples, validate on 1637 samples\n",
    "Epoch 1/20\n",
    "14725/14725 [==============================] - 17s - loss: 3.6036 - val_loss: 3.7924\n",
    "Epoch 2/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.7765 - val_loss: 3.0022\n",
    "Epoch 3/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.4773 - val_loss: 2.5697\n",
    "Epoch 4/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.3218 - val_loss: 2.2606\n",
    "Epoch 5/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.2328 - val_loss: 2.0832\n",
    "Epoch 6/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.1748 - val_loss: 2.0248\n",
    "Epoch 7/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.1174 - val_loss: 1.9865\n",
    "Epoch 8/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.0617 - val_loss: 1.9640\n",
    "Epoch 9/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.0206 - val_loss: 1.9461\n",
    "Epoch 10/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.9758 - val_loss: 1.9334\n",
    "Epoch 11/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.9546 - val_loss: 1.9148\n",
    "Epoch 12/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.9045 - val_loss: 1.9121\n",
    "Epoch 13/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.8757 - val_loss: 1.8888\n",
    "Epoch 14/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.8437 - val_loss: 1.8874\n",
    "Epoch 15/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.8145 - val_loss: 1.8822\n",
    "Epoch 16/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.7805 - val_loss: 1.8785\n",
    "Epoch 17/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.7558 - val_loss: 1.8868\n",
    "Epoch 18/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.7218 - val_loss: 1.8670\n",
    "Epoch 19/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.7032 - val_loss: 1.8759\n",
    "Epoch 20/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.6832 - val_loss: 1.8834\n",
    "\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=len(charset), output_dim=100))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(BatchNormalization())\n",
    "model.add(GRU(256, return_sequences=True))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(BatchNormalization())\n",
    "model.add(GRU(256))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "model.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\n",
    "Train on 14725 samples, validate on 1637 samples\n",
    "Epoch 1/20\n",
    "14725/14725 [==============================] - 33s - loss: 3.7814 - val_loss: 3.7282\n",
    "Epoch 2/20\n",
    "14725/14725 [==============================] - 29s - loss: 2.8453 - val_loss: 2.8542\n",
    "Epoch 3/20\n",
    "14725/14725 [==============================] - 29s - loss: 2.5178 - val_loss: 2.4434\n",
    "Epoch 4/20\n",
    "14725/14725 [==============================] - 29s - loss: 2.3762 - val_loss: 2.1894\n",
    "Epoch 5/20\n",
    "14725/14725 [==============================] - 29s - loss: 2.2896 - val_loss: 2.0862\n",
    "Epoch 6/20\n",
    "14725/14725 [==============================] - 29s - loss: 2.2254 - val_loss: 2.0516\n",
    "Epoch 7/20\n",
    "14725/14725 [==============================] - 29s - loss: 2.1565 - val_loss: 2.0133\n",
    "Epoch 8/20\n",
    "14725/14725 [==============================] - 29s - loss: 2.1132 - val_loss: 1.9992\n",
    "Epoch 9/20\n",
    "14725/14725 [==============================] - 29s - loss: 2.0798 - val_loss: 1.9881\n",
    "Epoch 10/20\n",
    "14725/14725 [==============================] - 29s - loss: 2.0509 - val_loss: 1.9784\n",
    "Epoch 11/20\n",
    "14725/14725 [==============================] - 29s - loss: 2.0198 - val_loss: 1.9618\n",
    "Epoch 12/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.9822 - val_loss: 1.9383\n",
    "Epoch 13/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.9437 - val_loss: 1.9300\n",
    "Epoch 14/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.9198 - val_loss: 1.9163\n",
    "Epoch 15/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.8989 - val_loss: 1.9160\n",
    "Epoch 16/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.8866 - val_loss: 1.9085\n",
    "Epoch 17/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.8493 - val_loss: 1.8965\n",
    "Epoch 18/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.8248 - val_loss: 1.8878\n",
    "Epoch 19/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.8037 - val_loss: 1.8870\n",
    "Epoch 20/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.7724 - val_loss: 1.8862\n",
    "\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=len(charset), output_dim=100))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(BatchNormalization())\n",
    "model.add(GRU(256, return_sequences=True))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(BatchNormalization())\n",
    "model.add(GRU(256))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "model.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\n",
    "Train on 14725 samples, validate on 1637 samples\n",
    "Epoch 1/20\n",
    "14725/14725 [==============================] - 32s - loss: 3.2809 - val_loss: 3.7595\n",
    "Epoch 2/20\n",
    "14725/14725 [==============================] - 29s - loss: 2.4379 - val_loss: 2.9869\n",
    "Epoch 3/20\n",
    "14725/14725 [==============================] - 29s - loss: 2.1504 - val_loss: 2.5361\n",
    "Epoch 4/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.9887 - val_loss: 2.1294\n",
    "Epoch 5/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.8984 - val_loss: 1.9727\n",
    "Epoch 6/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.7892 - val_loss: 1.9264\n",
    "Epoch 7/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.7172 - val_loss: 1.9100\n",
    "Epoch 8/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.6361 - val_loss: 1.9124\n",
    "Epoch 9/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.5621 - val_loss: 1.9122\n",
    "Epoch 10/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.4863 - val_loss: 1.9045\n",
    "Epoch 11/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.4150 - val_loss: 1.9278\n",
    "Epoch 12/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.3691 - val_loss: 1.9181\n",
    "Epoch 13/20\n",
    "14725/14725 [==============================] - 29s - loss: 1.2970 - val_loss: 1.9414\n",
    "Epoch 14/20\n",
    " 1536/14725 [==>...........................] - ETA: 25s - loss: 1.1475\n",
    " \n",
    " \n",
    " \n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=len(charset), output_dim=100))\n",
    "model.add(BatchNormalization())\n",
    "model.add(GRU(256))\n",
    "model.add(Dropout(0.4))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "model.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\n",
    " Train on 14725 samples, validate on 1637 samples\n",
    "Epoch 1/20\n",
    "14725/14725 [==============================] - 17s - loss: 3.1787 - val_loss: 3.9282\n",
    "Epoch 2/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.5070 - val_loss: 3.2479\n",
    "Epoch 3/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.2191 - val_loss: 2.7522\n",
    "Epoch 4/20\n",
    "14725/14725 [==============================] - 15s - loss: 2.0637 - val_loss: 2.3331\n",
    "Epoch 5/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.9539 - val_loss: 2.0326\n",
    "Epoch 6/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.8622 - val_loss: 1.9440\n",
    "Epoch 7/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.7821 - val_loss: 1.9166\n",
    "Epoch 8/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.7169 - val_loss: 1.8996\n",
    "Epoch 9/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.6561 - val_loss: 1.8849\n",
    "Epoch 10/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.5910 - val_loss: 1.9032\n",
    "Epoch 11/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.5082 - val_loss: 1.8878\n",
    "Epoch 12/20\n",
    "14725/14725 [==============================] - 15s - loss: 1.4513 - val_loss: 1.9252\n",
    "Epoch 13/20\n",
    " 7552/14725 [==============>...............] - ETA: 7s - loss: 1.3534\n",
    " \n",
    " \n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=len(charset), output_dim=100))\n",
    "model.add(BatchNormalization())\n",
    "model.add(GRU(512))\n",
    "model.add(Dropout(0.4))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "model.fit(X, y, epochs=20, batch_size=128, validation_split=0.1)\n",
    " Train on 14725 samples, validate on 1637 samples\n",
    "Epoch 1/20\n",
    "14725/14725 [==============================] - 23s - loss: 3.1173 - val_loss: 3.8415\n",
    "Epoch 2/20\n",
    "14725/14725 [==============================] - 20s - loss: 2.4202 - val_loss: 3.1512\n",
    "Epoch 3/20\n",
    "14725/14725 [==============================] - 20s - loss: 2.1302 - val_loss: 2.7191\n",
    "Epoch 4/20\n",
    "14725/14725 [==============================] - 20s - loss: 1.9435 - val_loss: 2.3341\n",
    "Epoch 5/20\n",
    "14725/14725 [==============================] - 20s - loss: 1.7966 - val_loss: 1.9877\n",
    "Epoch 6/20\n",
    "14725/14725 [==============================] - 20s - loss: 1.6621 - val_loss: 1.9349\n",
    "Epoch 7/20\n",
    "14725/14725 [==============================] - 20s - loss: 1.5190 - val_loss: 1.9632\n",
    "Epoch 8/20\n",
    "14725/14725 [==============================] - 20s - loss: 1.3925 - val_loss: 1.9735\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.fit(X, y, epochs=5, batch_size=128, validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_author_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----- Generating with seed: \"oes it really matter what it says this is some test text does it really matter what it says this is \"\n",
      "oes it really matter what it says this is some test text does it really matter what it says this is "
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "object of type 'numpy.float64' has no len()",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-22-af9aa51fe5bd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mgenerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdiversity\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.7\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"this is some test text does it really matter what it says \"\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-7-16c3c1fabb4f>\u001b[0m in \u001b[0;36mgenerate\u001b[0;34m(model, diversity, text)\u001b[0m\n\u001b[1;32m     34\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m         \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 36\u001b[0;31m         \u001b[0mnext_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdiversity\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     37\u001b[0m         \u001b[0mnext_char\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mindices_char\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m         \u001b[0mgenerated\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mnext_char\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-7-16c3c1fabb4f>\u001b[0m in \u001b[0;36msample\u001b[0;34m(preds, temperature)\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0mexp_preds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexp_preds\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexp_preds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m     \u001b[0mprobas\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmultinomial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprobas\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mmtrand.pyx\u001b[0m in \u001b[0;36mmtrand.RandomState.multinomial (numpy/random/mtrand/mtrand.c:37721)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: object of type 'numpy.float64' has no len()"
     ]
    }
   ],
   "source": [
    "generate(model, diversity=0.7, text=\"this is some test text does it really matter what it says \" * 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generate(model, diversity=0.5, text=\"\"):\n",
    "    \"\"\"Generate text from a model\"\"\"\n",
    "    start_index = random.randint(0, len(text) - maxlen - 1)\n",
    "    generated = ''\n",
    "    sentence = text[start_index: start_index + maxlen]\n",
    "    generated += sentence\n",
    "    print('----- Generating with seed: \"' + sentence + '\"')\n",
    "    sys.stdout.write(generated)\n",
    "\n",
    "    for i in range(5000):\n",
    "        x = np.zeros((1, maxlen), dtype=np.int)\n",
    "        for t, char in enumerate(sentence):\n",
    "            try:\n",
    "                x[0, t] = char_indices[char]\n",
    "            except:\n",
    "                print(sentence)\n",
    "        preds = model.predict(x, verbose=0)[0]\n",
    "        next_index = sample(preds, diversity)\n",
    "        next_char = indices_char[next_index]\n",
    "        generated += next_char\n",
    "        sentence = sentence[1:] + next_char\n",
    "        sys.stdout.write(next_char)\n",
    "        sys.stdout.flush()\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----- Generating with seed: \" what it says this is some test text does it really matter what it says this is some test text does \"\n",
      " what it says this is some test text does it really matter what it says this is some test text does the chan store if the store in for Vis a singic and the places I nor for the store in find store is a gitter beat to the trist and the counders here to find. The sele the store with the storit and the chansic I was next the back the truck out. This is a git and the courder the trust and the ching store with the me to good and the next the countret to the this store is little back to the clourder the selsection for the stration's and my for probebles to have the store if friends out befould the chan net store with the park is can sele, I was not longer casteral friends. The store store is little this is this lace in findst and the inclure the selmen store in finds a befter the beet the car this store if the chan strater and closed to the counders. The strail also here in find store with the store with me and selestanding and the strorthors. So I had things a back to the mant I was not look in fried. The store in finds. The mest to the had a leng so this shore sprob food. The lent for the store for the store with the mant and the pricked out a find the chings realing and the mettreer smore with the store is later with got and the counders here for the stort and consorthing. The store store store is littely to get the onlite of the been to this shorth and I was trinks a fead a feas and my next the car the store in find store better here and class. The me of the only convenienter the the cound store. The store store is little but the only Nortar Crip and my ore also head with the things. The me so this shores and class store screet and the fings and long sometime store store is little also this is a back is this is a bittle this was small with the metr seet and my rear to was my sor the counder fead and was counders. The line for the strort and the Chrack is bak in friends and the chan in finds. The strail was come to this store is little bas in finds on the mettranter but the me store is little any the fincers and land store in findlest your for the stort office all park in things around findss. The ment store is fried was converite back the park in finds. The stores store with the counderth and check is this is a good sement and the boxt of the best thing to hand a fead and the me sell this stort of the one of the straf a feat and the chises with the only got for the stroirs and the mark is a firle a feed and the only metter slow with the stricket and my could my dere for real peating. The me something the counders to but the chan store is a Still a was a glass in finds that was conser this store shore for the strorth only regulay same but if my ere fur nite before the bands. The lane I was to gas here and the counder for the store is this is this is a bask in finds and the curont real for the store, The store in findst. The start in friends which when I gound the plation for Crome was strort and a fead all peat for your for the store and the chisies with the manity look in friends. The last in firl also but hive the was next the coure fent and the plattor for your for the store is little back to be. And the me of the stor"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-49291dfdfef3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mgenerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdiversity\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"this is some test text does it really matter what it says \"\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m30\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-24-9c3c76047201>\u001b[0m in \u001b[0;36mgenerate\u001b[0;34m(model, diversity, text)\u001b[0m\n\u001b[1;32m     15\u001b[0m             \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m         \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m         \u001b[0mnext_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdiversity\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0mnext_char\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mindices_char\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.4/site-packages/keras/models.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose)\u001b[0m\n\u001b[1;32m    900\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    901\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 902\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    903\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    904\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpredict_on_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.4/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose)\u001b[0m\n\u001b[1;32m   1583\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1584\u001b[0m         return self._predict_loop(f, ins,\n\u001b[0;32m-> 1585\u001b[0;31m                                   batch_size=batch_size, verbose=verbose)\n\u001b[0m\u001b[1;32m   1586\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1587\u001b[0m     def train_on_batch(self, x, y,\n",
      "\u001b[0;32m~/.local/lib/python3.4/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_predict_loop\u001b[0;34m(self, f, ins, batch_size, verbose)\u001b[0m\n\u001b[1;32m   1210\u001b[0m                 \u001b[0mins_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_slice_arrays\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1211\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1212\u001b[0;31m             \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1213\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1214\u001b[0m                 \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.4/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2227\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2228\u001b[0m         updated = session.run(self.outputs + [self.updates_op],\n\u001b[0;32m-> 2229\u001b[0;31m                               feed_dict=feed_dict)\n\u001b[0m\u001b[1;32m   2230\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2231\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    776\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    777\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 778\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    779\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    780\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    980\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    981\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 982\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    983\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    984\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1030\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1031\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1032\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1033\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1034\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1037\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1040\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.4/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1019\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1020\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1021\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1022\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1023\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "generate(model, diversity=0.5, text=\"this is some test text does it really matter what it says \" * 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}