GRU_CNN_multitask.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/prabod/anaconda2/lib/python2.7/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using Theano backend.\n",
      "Using cuDNN version 7005 on context None\n",
      "Mapped name None to device cuda: GeForce GTX 950M (0000:01:00.0)\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division\n",
    "import csv\n",
    "import random as rn\n",
    "import os\n",
    "import sys\n",
    "os.environ['PYTHONHASHSEED'] = '0'\n",
    "# os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"   # see issue #152\n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"\n",
    "# import numpy\n",
    "np.random.seed(1)\n",
    "import pandas as pd\n",
    "# import tensorflow\n",
    "# tensorflow.set_random_seed(2)\n",
    "rn.seed(1)\n",
    "# session_conf = tensorflow.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)\n",
    "os.environ['KERAS_BACKEND'] = 'theano'\n",
    "from keras import backend as K\n",
    "# sess = tensorflow.Session(graph=tensorflow.get_default_graph(), config=session_conf)\n",
    "# K.set_session(sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trainData = pd.read_table(\"preprocessed/2018-E-c-En-train.txt\",sep=\"\\t\", index_col=0, quoting=csv.QUOTE_NONE)\n",
    "testData = pd.read_table(\"preprocessed/2018-E-c-En-test.txt\", sep=\"\\t\", index_col=0, quoting=csv.QUOTE_NONE)\n",
    "devData = pd.read_table(\"preprocessed/2018-E-c-En-dev.txt\", sep=\"\\t\", index_col=0, quoting=csv.QUOTE_NONE)\n",
    "gold = pd.read_table(\"2018-E-c-En-test-gold.txt\", sep=\"\\t\", index_col=0, quoting=csv.QUOTE_NONE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# trainData = trainData.drop(['anticipation','trust','surprise','pessimism'],axis=1)\n",
    "# testData = testData.drop(['anticipation','trust','surprise','pessimism'],axis=1)\n",
    "# devData = devData.drop(['anticipation','trust','surprise','pessimism'],axis=1)\n",
    "# gold = gold.drop(['anticipation','trust','surprise','pessimism'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.hist(trainData['Tweet'].str.split().apply(len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Input, Dense,TimeDistributed, Embedding, SpatialDropout1D,Concatenate, concatenate,MaxoutDense,LSTM,GaussianNoise,Average,Flatten\n",
    "from keras.layers import GRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, Conv1D, Dropout,Activation,MaxPooling1D,Merge\n",
    "from keras.preprocessing import text, sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "max_features = 20000\n",
    "maxlen = 50\n",
    "embed_size = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# trainData['Tweet'] = trainData['Tweet'].map(lambda com : clean(com))\n",
    "# devData['Tweet'] = devData['Tweet'].map(lambda com : clean(com))\n",
    "# testData['Tweet'] = testData['Tweet'].map(lambda com : clean(com))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tokenizer = text.Tokenizer(num_words=max_features,filters='')\n",
    "tokenizer.fit_on_texts(trainData['Tweet'].tolist()+devData['Tweet'].tolist()+testData['Tweet'].tolist())\n",
    "X_train = tokenizer.texts_to_sequences(trainData['Tweet'].tolist())\n",
    "X_val = tokenizer.texts_to_sequences(devData['Tweet'].tolist())\n",
    "X_test = tokenizer.texts_to_sequences(testData['Tweet'].tolist())\n",
    "x_train = sequence.pad_sequences(X_train, maxlen=maxlen)\n",
    "x_val = sequence.pad_sequences(X_val, maxlen=maxlen)\n",
    "x_test = sequence.pad_sequences(X_test, maxlen=maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_train = trainData.loc[:, trainData.columns != 'Tweet'].as_matrix()\n",
    "y_val = devData.loc[:, devData.columns != 'Tweet'].as_matrix()\n",
    "y_test = testData.loc[:, testData.columns != 'Tweet'].as_matrix()\n",
    "y_gold = gold.loc[:, testData.columns != 'Tweet'].as_matrix()\n",
    "\n",
    "y_test = np.array(y_test.tolist())\n",
    "# y_test = y_test[:,10]\n",
    "\n",
    "y_train = np.array(y_train.tolist())\n",
    "# y_train = y_train[:,10]\n",
    "\n",
    "y_val = np.array(y_val.tolist())\n",
    "# y_val = y_val[:,10]\n",
    "\n",
    "y_gold = np.array(y_gold.tolist())\n",
    "# y_gold = y_gold[:,10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.word_index[',']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from gensim.models import KeyedVectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EMBEDDING_FILE = '/media/prabod/37727493640E51BE/GoogleNews-vectors-negative300.bin.gz'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EMBEDDING_FILE = 'embeddings/w2v.vec'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EMBEDDING_FILE = '/media/prabod/37727493640E51BE/word2vec_twitter_model/word2vec_twitter_model.bin'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EMBEDDING_FILE = 'glove.twitter.27B.200d.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EMBEDDING_FILE = 'wiki-news-300d-1M.vec'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EMBEDDING_FILE = 'datastories.twitter.300d.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from word2vecReader import Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embeddings_index=Word2Vec.load_word2vec_format(EMBEDDING_FILE, binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embeddings_index['dog'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')\n",
    "embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_index = tokenizer.word_index\n",
    "nb_words = max(max_features, len(word_index))\n",
    "embedding_matrix = np.zeros((nb_words, embed_size))\n",
    "without_embed = []\n",
    "n=0\n",
    "for word, i in word_index.items():\n",
    "    if i >= max_features: break\n",
    "    try:\n",
    "        embedding_vector = embeddings_index[word]\n",
    "        if embedding_vector is not None: \n",
    "            embedding_matrix[i] = embedding_vector\n",
    "    except:\n",
    "            n+=1\n",
    "            without_embed.append(word)\n",
    "print \"Words without embeddings %d\" % n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Words without embeddings 566\n"
     ]
    }
   ],
   "source": [
    "word_index = tokenizer.word_index\n",
    "nb_words = max(max_features, len(word_index))\n",
    "embedding_matrix = np.zeros((nb_words, embed_size))\n",
    "without_embed = []\n",
    "n=0\n",
    "for word, i in word_index.items():\n",
    "    if i >= max_features: break\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None: \n",
    "        embedding_matrix[i] = embedding_vector\n",
    "    else:\n",
    "        n+=1\n",
    "        without_embed.append(word)\n",
    "print \"Words without embeddings %d\" % n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run the below to save the matrix as a pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import cPickle as pickle\n",
    "with open('preprocessed/embedding_matrix_300_punc.p','wb') as f:\n",
    "    pickle.dump(embedding_matrix,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print without_embed\n",
    "# print word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.1585244267198405, 0.060942173479561317, 0.16213858424725822, 0.07739282153539381, 0.1543494516450648, 0.04361914257228315, 0.12362911266201396, 0.04953888334995015, 0.1251246261216351, 0.022495014955134597, 0.022245762711864406]\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "ass = trainData.drop('Tweet',1).sum()\n",
    "ass = pd.DataFrame(ass,columns=['s'])\n",
    "ass['w'] = (ass['s']/ass['s'].sum())\n",
    "weights = ass['w'].tolist()\n",
    "print weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "1/np.round(-np.log(weights)*1.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import keras.backend as K\n",
    "def weighted_multitask_loss(y_true, y_pred):\n",
    "    # Avoid divide by 0\n",
    "    W = np.array(weights,dtype=np.float32)\n",
    "    y_pred = K.clip(y_pred, K.epsilon(), 1 - K.epsilon())\n",
    "    # Multi-task loss\n",
    "    return K.mean(K.sum((- y_true * -K.log(W)*K.log(y_pred) - (1.0 - y_true) *K.log(1.0 - y_pred)), axis=1))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import keras.backend as K\n",
    "def fbased(y_true, y_pred):\n",
    "    y_pred = K.clip(y_pred, K.epsilon(), 1 - K.epsilon())\n",
    "    return K.mean(1 - (2 * y_pred * y_true)/(K.sum(y_true) + K.sum(y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def binary_crossentropy_weighted(y_true, y_pred):\n",
    "    class_weights = 2\n",
    "    y_pred = K.clip(y_pred, K.epsilon(), 1.0 - K.epsilon())\n",
    "    tmp =class_weights*((-y_true * K.log(y_pred))) -(1.0 - y_true) * K.log(1.0 - y_pred)\n",
    "    loss = K.mean(tmp,axis=-1)\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def identity_loss(y_true, y_pred):\n",
    "\n",
    "    return K.mean(y_pred - 0 * y_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import keras.backend.tensorflow_backend as tfb\n",
    "\n",
    "POS_WEIGHT = 2  # multiplier for positive targets, needs to be tuned\n",
    "\n",
    "def weighted_binary_crossentropy(target, output):\n",
    "    \"\"\"\n",
    "    Weighted binary crossentropy between an output tensor \n",
    "    and a target tensor. POS_WEIGHT is used as a multiplier \n",
    "    for the positive targets.\n",
    "\n",
    "    Combination of the following functions:\n",
    "    * keras.losses.binary_crossentropy\n",
    "    * keras.backend.tensorflow_backend.binary_crossentropy\n",
    "    * tf.nn.weighted_cross_entropy_with_logits\n",
    "    \"\"\"\n",
    "    # transform back to logits\n",
    "    _epsilon = tfb._to_tensor(tfb.epsilon(), output.dtype.base_dtype)\n",
    "    output = tf.clip_by_value(output, _epsilon, 1 - _epsilon)\n",
    "    output = tf.log(output / (1 - output))\n",
    "    # compute weighted loss\n",
    "    loss = tf.nn.weighted_cross_entropy_with_logits(targets=target,\n",
    "                                                    logits=output,\n",
    "                                                    pos_weight=POS_WEIGHT)\n",
    "    return tf.reduce_mean(loss, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import keras.backend.tensorflow_backend as tfb\n",
    "\n",
    "POS_WEIGHT = 2  # multiplier for positive targets, needs to be tuned\n",
    "\n",
    "def loss1(target, output):\n",
    "    _epsilon = tfb._to_tensor(tfb.epsilon(), output.dtype.base_dtype)\n",
    "    output = tf.clip_by_value(output, _epsilon, 1 - _epsilon)\n",
    "    output = tf.log(output / (1 - output))\n",
    "    return tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=output, labels=target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras import backend as K, initializers, regularizers, constraints\n",
    "from keras.engine.topology import Layer\n",
    "def dot_product(x, kernel):\n",
    "    \"\"\"\n",
    "    Wrapper for dot product operation, in order to be compatible with both\n",
    "    Theano and Tensorflow\n",
    "    Args:\n",
    "        x (): input\n",
    "        kernel (): weights\n",
    "    Returns:\n",
    "    \"\"\"\n",
    "    if K.backend() == 'tensorflow':\n",
    "        return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1)\n",
    "    else:\n",
    "        return K.dot(x, kernel)\n",
    "    \n",
    "\n",
    "class AttentionWithContext(Layer):\n",
    "    \"\"\"\n",
    "    Attention operation, with a context/query vector, for temporal data.\n",
    "    Supports Masking.\n",
    "    Follows the work of Yang et al. [https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf]\n",
    "    \"Hierarchical Attention Networks for Document Classification\"\n",
    "    by using a context vector to assist the attention\n",
    "    # Input shape\n",
    "        3D tensor with shape: `(samples, steps, features)`.\n",
    "    # Output shape\n",
    "        2D tensor with shape: `(samples, features)`.\n",
    "    How to use:\n",
    "    Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.\n",
    "    The dimensions are inferred based on the output shape of the RNN.\n",
    "    Note: The layer has been tested with Keras 2.0.6\n",
    "    Example:\n",
    "        model.add(LSTM(64, return_sequences=True))\n",
    "        model.add(AttentionWithContext())\n",
    "        # next add a Dense layer (for classification/regression) or whatever...\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self,\n",
    "                 W_regularizer=None, u_regularizer=None, b_regularizer=None,\n",
    "                 W_constraint=None, u_constraint=None, b_constraint=None,\n",
    "                 bias=True, **kwargs):\n",
    "\n",
    "        self.supports_masking = True\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(W_regularizer)\n",
    "        self.u_regularizer = regularizers.get(u_regularizer)\n",
    "        self.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.u_constraint = constraints.get(u_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        super(AttentionWithContext, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1], input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((input_shape[-1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "\n",
    "        self.u = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_u'.format(self.name),\n",
    "                                 regularizer=self.u_regularizer,\n",
    "                                 constraint=self.u_constraint)\n",
    "\n",
    "        super(AttentionWithContext, self).build(input_shape)\n",
    "\n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        # do not pass the mask to the next layers\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        uit = dot_product(x, self.W)\n",
    "\n",
    "        if self.bias:\n",
    "            uit += self.b\n",
    "\n",
    "        uit = K.tanh(uit)\n",
    "        ait = dot_product(uit, self.u)\n",
    "\n",
    "        a = K.exp(ait)\n",
    "\n",
    "        # apply mask after the exp. will be re-normalized next\n",
    "        if mask is not None:\n",
    "            # Cast the mask to floatX to avoid float64 upcasting in theano\n",
    "            a *= K.cast(mask, K.floatx())\n",
    "\n",
    "        # in some cases especially in the early stages of training the sum may be almost zero\n",
    "        # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.\n",
    "        # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())\n",
    "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "\n",
    "        a = K.expand_dims(a)\n",
    "        weighted_input = x * a\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return input_shape[0], input_shape[-1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from attdeep import AttentionWeightedAverage\n",
    "from attlayer import Attention\n",
    "from keras.layers import Dropout, SpatialDropout1D, LSTM, Activation,GaussianNoise,Convolution1D,BatchNormalization\n",
    "from keras.regularizers import L1L2\n",
    "from keras.optimizers import RMSprop,Adam\n",
    "from performance import dice_coef,dice_coef_loss,\\\n",
    "jaccard_distance,jaccard_coef_loss,jaccard_coef_loss_1,jaccard_coef,jaccard_coef_int,fmeasure,recall,precision,\\\n",
    "multitask_loss\n",
    "def gru_model():\n",
    "    model_input = Input(shape=(maxlen,), dtype='int32')\n",
    "    embed_reg = L1L2(l2=1e-8)\n",
    "    embed = Embedding(input_dim=max_features,\n",
    "                  output_dim=256,\n",
    "                  mask_zero=True,\n",
    "                  trainable=False,\n",
    "                  input_length=maxlen,\n",
    "                  embeddings_regularizer=embed_reg,\n",
    "                  name='embedding')\n",
    "    x = embed(model_input)\n",
    "#     x = BatchNormalization()(x)\n",
    "    x = Activation('tanh')(x)\n",
    "    x = GaussianNoise(0.2)(x)\n",
    "    embed_drop = SpatialDropout1D(0.4, name='embed_drop')\n",
    "    x = embed_drop(x)\n",
    "    lstm_0_output = Bidirectional(LSTM(64, return_sequences=True), name=\"bi_lstm_0\")(x)\n",
    "    lstm_1_output = Bidirectional(LSTM(64, return_sequences=True), name=\"bi_lstm_1\")(lstm_0_output)\n",
    "    x = concatenate([lstm_1_output, lstm_0_output, x])\n",
    "    x = GaussianNoise(0.3)(x)\n",
    "    x = AttentionWeightedAverage(name='attlayer')(x)\n",
    "#     x = Attention(name='attlayer')(x)\n",
    "    x = Dropout(0.2)(x)\n",
    "    outputs = [Dense(11, activation='sigmoid', name='softmax')(x)]\n",
    "    model = Model(inputs=[model_input], outputs=outputs, name=\"DeepMoji\")\n",
    "    model.compile(loss=weighted_multitask_loss,\n",
    "              optimizer=Adam(clipnorm=1,lr=0.001),\n",
    "              metrics=['accuracy',fmeasure,recall,precision])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from performance import dice_coef,dice_coef_loss,\\\n",
    "jaccard_distance,jaccard_coef_loss,jaccard_coef_loss_1,jaccard_coef,jaccard_coef_int,fmeasure,recall,precision,\\\n",
    "multitask_loss\n",
    "def get_model():\n",
    "    sequence_input = Input(shape=(maxlen, ))\n",
    "    x = Embedding(max_features, embed_size,weights=[embedding_matrix],trainable = True)(sequence_input)\n",
    "#     x = SpatialDropout1D(0.2)(x)\n",
    "    x = Bidirectional(GRU(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(x)\n",
    "    x = Conv1D(64, kernel_size = 2, padding = \"valid\", kernel_initializer = \"glorot_uniform\")(x)\n",
    "    avg_pool = GlobalAveragePooling1D()(x)\n",
    "    max_pool = GlobalMaxPooling1D()(x)\n",
    "    x = concatenate([avg_pool, max_pool]) \n",
    "    x = Dense(128, activation='relu')(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    preds = Dense(11, activation=\"sigmoid\")(x)\n",
    "    \n",
    "    model = Model(inputs=sequence_input, outputs=preds)\n",
    "    model.compile(loss=weighted_binary_crossentropy,\n",
    "                  optimizer='adam',\n",
    "                  metrics=['accuracy',jaccard_coef,dice_coef,fmeasure,recall,precision])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from performance import dice_coef,dice_coef_loss,\\\n",
    "jaccard_distance,jaccard_coef_loss,jaccard_coef_loss_1,jaccard_coef,jaccard_coef_int,fmeasure,recall,precision,\\\n",
    "multitask_loss\n",
    "from attdeep import AttentionWeightedAverage\n",
    "from attlayer import Attention,AttLayer\n",
    "from keras.regularizers import l2, L1L2\n",
    "from keras.optimizers import RMSprop,Adam\n",
    "def get_model_2():\n",
    "    model_input = Input(shape=(maxlen,), dtype='int32')\n",
    "    embed_reg = L1L2(l2=1e-8)\n",
    "    embed = Embedding(max_features, embed_size,\n",
    "                      mask_zero=True,\n",
    "                      weights=[embedding_matrix],\n",
    "                      trainable=False,\n",
    "                      name='embedding')\n",
    "    x = embed(model_input)\n",
    "#     x = Activation('tanh')(x)\n",
    "    x = GaussianNoise(0.3)(x)\n",
    "#     embed_drop = SpatialDropout1D(0.2, name='embed_drop')\n",
    "#     x = embed_drop(x)\n",
    "    lstm_0_output = Bidirectional(LSTM(128, return_sequences=True), name=\"bi_lstm_0\")(x)\n",
    "    lstm_1_output = Bidirectional(LSTM(128, return_sequences=True), name=\"bi_lstm_1\")(lstm_0_output)\n",
    "    x = concatenate([lstm_1_output, lstm_0_output])\n",
    "    x = GaussianNoise(0.3)(x)\n",
    "    x = TimeDistributed(Dense(256))(x)\n",
    "#     x = AttentionWeightedAverage(name='attlayer')(x)\n",
    "    x = AttentionWithContext(name='attlayer')(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    outputs = [Dense(11, activation='sigmoid', name='softmax')(x)]\n",
    "    model = Model(inputs=[model_input], outputs=outputs, name=\"DeepMoji\")\n",
    "    model.compile(loss=weighted_binary_crossentropy,\n",
    "              optimizer=Adam(clipnorm=1, lr=0.001),\n",
    "              metrics=['accuracy',fmeasure,recall,precision])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from performance import dice_coef,dice_coef_loss,\\\n",
    "jaccard_distance,jaccard_coef_loss,jaccard_coef_loss_1,jaccard_coef,jaccard_coef_int,fmeasure,recall,precision,\\\n",
    "multitask_loss\n",
    "from attdeep import AttentionWeightedAverage\n",
    "from attlayer import Attention\n",
    "from keras.regularizers import l2, L1L2\n",
    "from keras.optimizers import RMSprop,Adam\n",
    "from keras.layers import Dropout, SpatialDropout1D, LSTM, Activation,Convolution1D,MaxPooling1D\n",
    "def get_model_3():\n",
    "    filter_sizes = (2,4,5,8)\n",
    "    dropout_prob = [0.4,0.5]\n",
    "\n",
    "    graph_in = Input(shape=(maxlen, embed_size))\n",
    "    convs = []\n",
    "    avgs = []\n",
    "    for fsz in filter_sizes:\n",
    "        conv = Convolution1D(nb_filter=32,\n",
    "                             filter_length=fsz,\n",
    "                             border_mode='valid',\n",
    "                             activation='relu',\n",
    "                             subsample_length=1)(graph_in)\n",
    "        pool = MaxPooling1D(pool_length=maxlen-fsz+1)(conv)\n",
    "        flattenMax = Flatten()(pool)\n",
    "        convs.append(flattenMax)\n",
    "\n",
    "    if len(filter_sizes)>1:\n",
    "        out = Merge(mode='concat')(convs)\n",
    "    else:\n",
    "        out = convs[0]\n",
    "\n",
    "    graph = Model(input=graph_in, output=out, name=\"graphModel\")\n",
    "    \n",
    "    model_input = Input(shape=(maxlen,), dtype='int32')\n",
    "    embed = Embedding(max_features, embed_size,\n",
    "#                       mask_zero=True,\n",
    "                      weights=[embedding_matrix],\n",
    "                      trainable=False,\n",
    "                      name='embedding')\n",
    "    x = embed(model_input)\n",
    "    x = graph(x)\n",
    "    x = Dense(128)(x)\n",
    "#     x = MaxPooling1D(pool_length=pool_length)(x)\n",
    "    x = LSTM(70)(x)\n",
    "    outputs = [Dense(11, activation='sigmoid', name='softmax')(x)]\n",
    "    model = Model(inputs=[model_input], outputs=outputs, name=\"DeepMoji\")\n",
    "    model.compile(loss=weighted_multitask_loss,\n",
    "              optimizer=Adam(clipnorm=1, lr=0.001),\n",
    "              metrics=['accuracy',fmeasure,recall,precision])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from attdeep import AttentionWeightedAverage\n",
    "from keras.regularizers import l2, L1L2\n",
    "from keras.optimizers import RMSprop,Adam\n",
    "def get_cnn_model_v3():    # added filter\n",
    "    sequence_input = Input(shape=(maxlen, ))\n",
    "    x = Embedding(max_features, embed_size,weights=[embedding_matrix],trainable = False)(sequence_input)\n",
    "    drop = SpatialDropout1D(0.2)(x)\n",
    "#     x = Conv1D(128, kernel_size = 2, padding = \"valid\", kernel_initializer = \"glorot_uniform\")(x)\n",
    "#     x = GlobalMaxPooling1D()(x)\n",
    "    d = Dense(128, activation='relu')(x)\n",
    "    x = concatenate([d,drop])\n",
    "    x = GaussianNoise(0.3)(x)\n",
    "#     x = Bidirectional(LSTM(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(x)\n",
    "    x = Bidirectional(GRU(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(x)\n",
    "    x = Conv1D(64, kernel_size = 2, padding = \"valid\", kernel_initializer = \"glorot_uniform\",kernel_regularizer=L1L2(l2=1e-4))(x)\n",
    "    avg_pool = GlobalAveragePooling1D()(x)\n",
    "    max_pool = GlobalMaxPooling1D()(x)\n",
    "    x = concatenate([avg_pool, max_pool]) \n",
    "    x = GaussianNoise(0.3)(x)\n",
    "#     x = Dense(64, activation='relu',activity_regularizer=L1L2(l2=1e-4))(x)\n",
    "    x = Dense(128, activation='relu',activity_regularizer=L1L2(l2=1e-4),kernel_regularizer=L1L2(l2=1e-4))(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x = GaussianNoise(0.3)(x)\n",
    "#     x = MaxoutDense(11)(x)\n",
    "    preds = Dense(11, activation=\"sigmoid\",activity_regularizer=L1L2(l2=1e-4),kernel_regularizer=L1L2(l2=1e-4))(x)\n",
    "    \n",
    "    model = Model(inputs=sequence_input, outputs=preds)\n",
    "    model.compile(loss=weighted_multitask_loss,\n",
    "                  optimizer=Adam(clipnorm=1, lr=0.001),\n",
    "                  metrics=['acc',fmeasure,recall,precision])\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from attdeep import AttentionWeightedAverage\n",
    "from keras.regularizers import l2, L1L2\n",
    "from attlayer import Attention\n",
    "from keras.layers import Dropout, SpatialDropout1D, LSTM, Activation,GaussianNoise,Convolution1D\n",
    "from keras.optimizers import RMSprop,Adam\n",
    "def model_all():    # added filter\n",
    "    \n",
    "    filter_sizes = (2,4,5,8)\n",
    "    dropout_prob = [0.4,0.5]\n",
    "\n",
    "    graph_in = Input(shape=(maxlen, embed_size))\n",
    "    convs = []\n",
    "    avgs = []\n",
    "    for fsz in filter_sizes:\n",
    "        conv = Convolution1D(nb_filter=32,\n",
    "                             filter_length=fsz,\n",
    "                             border_mode='valid',\n",
    "                             activation='relu',\n",
    "                             subsample_length=1)(graph_in)\n",
    "        pool = MaxPooling1D(pool_length=maxlen-fsz+1)(conv)\n",
    "        flattenMax = Flatten()(pool)\n",
    "        convs.append(flattenMax)\n",
    "\n",
    "    if len(filter_sizes)>1:\n",
    "        out = Merge(mode='concat')(convs)\n",
    "    else:\n",
    "        out = convs[0]\n",
    "\n",
    "    graph = Model(input=graph_in, output=out, name=\"graphModel\")\n",
    "    \n",
    "    sequence_input = Input(shape=(maxlen, ))\n",
    "    embedding = Embedding(max_features, embed_size,weights=[embedding_matrix],trainable = False)(sequence_input)\n",
    "    drop = SpatialDropout1D(0.2)(embedding)\n",
    "#     d = Dense(128, activation='relu')(embedding)\n",
    "#     x = concatenate([d,drop])\n",
    "    g_noise = GaussianNoise(0.3)(embedding)\n",
    "#     x = Bidirectional(LSTM(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(x)\n",
    "    bi_lstm = Bidirectional(GRU(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(g_noise)\n",
    "#     lstm_0_output = Bidirectional(GRU(35, return_sequences=True), name=\"bi_lstm_0\")(g_noise)\n",
    "#     lstm_1_output = Bidirectional(GRU(35, return_sequences=True), name=\"bi_lstm_1\")(lstm_0_output)\n",
    "#     bi_lstm = concatenate([lstm_1_output, lstm_0_output,drop])\n",
    "    conv = Conv1D(64, kernel_size = 2, padding = \"valid\", kernel_initializer = \"glorot_uniform\")(bi_lstm)\n",
    "    avg_pool = GlobalAveragePooling1D()(conv)\n",
    "    max_pool = GlobalMaxPooling1D()(conv)\n",
    "    atten = Attention()(bi_lstm)\n",
    "    time_dist = TimeDistributed(Dense(256))(bi_lstm)\n",
    "    time_atten = AttentionWithContext()(time_dist)\n",
    "    conv_set = graph(embedding)\n",
    "#     conc_all = concatenate([avg_pool, max_pool,atten,time_atten])\n",
    "    pool = concatenate([avg_pool, max_pool])\n",
    "    x1 = Dense(11, activation='sigmoid',activity_regularizer=L1L2(l2=1e-4),kernel_regularizer=L1L2(l2=1e-4))(pool)\n",
    "    x2 = Dense(11, activation='sigmoid',activity_regularizer=L1L2(l2=1e-4),kernel_regularizer=L1L2(l2=1e-4))(atten)\n",
    "#     x3 = Dense(11, activation='sigmoid',activity_regularizer=L1L2(l2=1e-4),kernel_regularizer=L1L2(l2=1e-4))(time_dist)\n",
    "    x4 = Dense(11, activation='sigmoid',activity_regularizer=L1L2(l2=1e-4),kernel_regularizer=L1L2(l2=1e-4))(time_atten)\n",
    "    x5 = Dense(11, activation='sigmoid',activity_regularizer=L1L2(l2=1e-4),kernel_regularizer=L1L2(l2=1e-4))(conv_set)\n",
    "    \n",
    "    conc_all = concatenate([x1,x2,x4,x5])\n",
    "    x = GaussianNoise(0.3)(conc_all)\n",
    "#     x = Dense(64, activation='relu',activity_regularizer=L1L2(l2=1e-4))(x)\n",
    "#     x = Dropout(0.5)(x)\n",
    "#     x = GaussianNoise(0.3)(x)\n",
    "#     x = MaxoutDense(11)(x)\n",
    "    preds = Dense(11, activation=\"sigmoid\",activity_regularizer=L1L2(l2=1e-4),kernel_regularizer=L1L2(l2=1e-4))(x)\n",
    "    \n",
    "    model = Model(inputs=sequence_input, outputs=preds)\n",
    "    model.compile(loss=weighted_multitask_loss,\n",
    "                  optimizer=Adam(lr=0.001),\n",
    "                  metrics=['acc',fmeasure,recall,precision])\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from attdeep import AttentionWeightedAverage\n",
    "from keras.regularizers import l2, L1L2\n",
    "from keras.optimizers import RMSprop,Adam\n",
    "def get_cnn_model_dual_embed():    # added filter\n",
    "    sequence_input = Input(shape=(maxlen, ))\n",
    "    embed_reg = L1L2(l2=1e-6)\n",
    "    x1 = Embedding(max_features, embed_size,weights=[embedding_matrix],trainable = False)(sequence_input)\n",
    "    x2 = Embedding(max_features, embed_size,weights=[embedding_matrix_dup],trainable = True,embeddings_regularizer=embed_reg)(sequence_input)\n",
    "    x = Concatenate(axis=1)([x1,x2])\n",
    "    x = GaussianNoise(0.2)(x)\n",
    "    embed_drop = SpatialDropout1D(0.2, name='embed_drop')\n",
    "    x = embed_drop(x)\n",
    "    lstm_0_output = Bidirectional(LSTM(70, return_sequences=True), name=\"bi_lstm_0\")(x)\n",
    "    lstm_1_output = Bidirectional(LSTM(70, return_sequences=True), name=\"bi_lstm_1\")(lstm_0_output)\n",
    "    x = concatenate([lstm_1_output, lstm_0_output, x])\n",
    "    x = GaussianNoise(0.2)(x)\n",
    "    x = AttentionWeightedAverage(name='attlayer')(x)\n",
    "#     x = Attention(name='attlayer')(x)\n",
    "    x = Dropout(0.2)(x)\n",
    "    outputs = [Dense(11, activation='sigmoid', name='softmax')(x)]\n",
    "    model = Model(inputs=sequence_input, outputs=outputs, name=\"DeepMoji\")\n",
    "    model.compile(loss=weighted_multitask_loss,\n",
    "              optimizer=Adam(clipnorm=1, lr=0.001),\n",
    "              metrics=['accuracy',fmeasure,recall,precision])\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from attdeep import AttentionWeightedAverage\n",
    "from keras.regularizers import l2, L1L2\n",
    "from keras.layers import Dropout, SpatialDropout1D, LSTM, Activation,GaussianNoise,Convolution1D\n",
    "def get_cnn_model_v4():    # added filter\n",
    "#     embedded_sequences = Bidirectional(GRU(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(embedded_sequences)\n",
    "    filter_sizes = (2,4,5,8)\n",
    "    dropout_prob = [0.4,0.5]\n",
    "\n",
    "    graph_in = Input(shape=(maxlen, embed_size))\n",
    "    convs = []\n",
    "    avgs = []\n",
    "    for fsz in filter_sizes:\n",
    "        conv = Convolution1D(nb_filter=32,\n",
    "                             filter_length=fsz,\n",
    "                             border_mode='valid',\n",
    "                             activation='relu',\n",
    "                             subsample_length=1)(graph_in)\n",
    "        pool = MaxPooling1D(pool_length=maxlen-fsz+1)(conv)\n",
    "        flattenMax = Flatten()(pool)\n",
    "        convs.append(flattenMax)\n",
    "\n",
    "    if len(filter_sizes)>1:\n",
    "        out = Merge(mode='concat')(convs)\n",
    "    else:\n",
    "        out = convs[0]\n",
    "\n",
    "    graph = Model(input=graph_in, output=out, name=\"graphModel\")\n",
    "\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(input_dim=max_features, #size of vocabulary\n",
    "                     output_dim = embed_size,\n",
    "                     input_length = maxlen,\n",
    "#                      weights=[embedding_matrix],\n",
    "                     trainable=False))\n",
    "    model.add(Dropout(dropout_prob[0]))\n",
    "    model.add(Bidirectional(GRU(100, return_sequences=True,dropout=0.1,recurrent_dropout=0.1)))\n",
    "    model.add(graph)\n",
    "    model.add(Dense(128))\n",
    "    model.add(Dropout(dropout_prob[1]))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dense(11))\n",
    "    model.add(Activation('sigmoid'))\n",
    "    model.compile(loss=weighted_multitask_loss,\n",
    "                  optimizer=Adam(clipnorm=1, lr=0.001),\n",
    "                  metrics=['acc',fmeasure,recall,precision])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = get_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_tra, X_val, y_tra, y_val = x_train,x_val, y_train,y_val "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.callbacks import Callback\n",
    "from sklearn.metrics import roc_auc_score,jaccard_similarity_score\n",
    "class RocAucEvaluation(Callback):\n",
    "    def __init__(self, validation_data=(), interval=1):\n",
    "        super(Callback, self).__init__()\n",
    "\n",
    "        self.interval = interval\n",
    "        self.X_val, self.y_val = validation_data\n",
    "#     def on_batch_end(self, batch, logs={}):\n",
    "#         y_pred = self.model.predict(self.X_val, verbose=0)\n",
    "#         score = roc_auc_score(self.y_val, y_pred)\n",
    "#         jac_score = jaccard_similarity_score(self.y_val, y_pred.round(), normalize=False)\n",
    "#         logs['roc_auc_val'] = score\n",
    "#         logs['jac_val'] = jac_score\n",
    "    \n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        if epoch % self.interval == 0:\n",
    "            y_pred = self.model.predict(self.X_val, verbose=0)\n",
    "            score = roc_auc_score(self.y_val, y_pred)\n",
    "            jac_score = jaccard_similarity_score(self.y_val, y_pred.round())\n",
    "            logs['roc_auc_val'] = score\n",
    "            logs['jac_val'] = jac_score\n",
    "            print(\"\\n ROC-AUC - epoch: %d - score: %.6f \\n\" % (epoch+1, score))\n",
    "            print(\"\\n JAC-SIM - epoch: %d - score: %.6f \\n\" % (epoch+1, jac_score))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.callbacks import TensorBoard, ReduceLROnPlateau, ModelCheckpoint, EarlyStopping\n",
    "RocAuc = RocAucEvaluation(validation_data=(X_val, y_val), interval=1)\n",
    "reduce_lr = ReduceLROnPlateau(monitor='val_loss', verbose=1,factor=0.2,patience=3, min_lr=0.0001)\n",
    "checkpointer = ModelCheckpoint(filepath='models/weights_exp_105_preprocessed.hdf5', monitor='jac_val',verbose=1,mode='max', save_best_only=True)\n",
    "earlyStoper = EarlyStopping(monitor='jac_val', min_delta=0, patience=7, verbose=1, mode='max')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.callbacks import TensorBoard, ReduceLROnPlateau, ModelCheckpoint, EarlyStopping\n",
    "RocAuc = RocAucEvaluation(validation_data=(X_val, y_val), interval=1)\n",
    "reduce_lr = ReduceLROnPlateau(monitor='val_loss', verbose=1,factor=0.2,patience=3, min_lr=0.0001)\n",
    "checkpointer = ModelCheckpoint(filepath='models/weights_exp_105_preprocessed.hdf5', monitor='val_loss',verbose=1,mode='min', save_best_only=True)\n",
    "earlyStoper = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=1, mode='min')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from performance import dice_coef,dice_coef_loss,\\\n",
    "jaccard_distance,jaccard_coef_loss,jaccard_coef_loss_1,jaccard_coef,jaccard_coef_int,fmeasure,recall,precision,\\\n",
    "multitask_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "def get_class_weights(y, smooth_factor=0):\n",
    "    \"\"\"\n",
    "    Returns the weights for each class based on the frequencies of the samples\n",
    "    :param smooth_factor: factor that smooths extremely uneven weights\n",
    "    :param y: list of true labels (the labels must be hashable)\n",
    "    :return: dictionary with the weight for each class\n",
    "    \"\"\"\n",
    "    counter = Counter(y)\n",
    "\n",
    "    if smooth_factor > 0:\n",
    "        p = max(counter.values()) * smooth_factor\n",
    "        for k in counter.keys():\n",
    "            counter[k] += p\n",
    "\n",
    "    majority = max(counter.values())\n",
    "\n",
    "    return {cls: float(majority / count) for cls, count in counter.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class_weights=get_class_weights(y_train.tolist(),0)\n",
    "print class_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "epochs = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "OUT_FILE='Prediction/E-C_en_pred_exp_105.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# model = get_cnn_model_dual_embed()\n",
    "from sklearn.cross_validation import KFold\n",
    "n_folds = 10\n",
    "# data, labels, header_info = load_data()\n",
    "skf = KFold(len(X_tra),10, random_state=None, shuffle=True)\n",
    "\n",
    "for i, (train, test) in enumerate(skf):\n",
    "    print \"Running Fold\", i+1, \"/\", n_folds\n",
    "    model = None # Clearing the NN.\n",
    "    model = get_cnn_model_v3()\n",
    "    train_and_evaluate_model(model, X_tra[train], y_tra[train], X_tra[test], y_tra[test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "import itertools\n",
    "def train_and_evaluate_model(model, X_tra, y_tra, X_val, y_val):\n",
    "    hist = model.fit(X_tra, y_tra, batch_size=batch_size,shuffle=False, epochs=epochs,validation_data=(X_val, y_val),\n",
    "                     callbacks=[RocAuc,earlyStoper,checkpointer,reduce_lr])\n",
    "\n",
    "    tData = pd.read_table(\"datasets/2018-E-c-En-test.txt\",sep=\"\\t\", index_col=0, quoting=csv.QUOTE_NONE)\n",
    "    model = load_model('models/weights_exp_105_preprocessed.hdf5',custom_objects={'AttentionWeightedAverage': AttentionWeightedAverage,\n",
    "                                                                                 'AttentionWithContext':AttentionWithContext,\n",
    "                                                                                 'dice_coef_loss':dice_coef_loss,\n",
    "                                                                                'dice_coef': dice_coef,\n",
    "                                                                                'jaccard_distance':jaccard_distance,\n",
    "                                                                                'fmeasure':fmeasure,\n",
    "                                                                                'recall':recall,\n",
    "                                                                                'precision':precision,\n",
    "                                                                                'jaccard_coef':jaccard_coef,\n",
    "                                                                                'jaccard_coef_loss':jaccard_coef_loss,\n",
    "                                                                                'weighted_binary_crossentropy':weighted_binary_crossentropy,\n",
    "                                                                                'multitask_loss':multitask_loss,\n",
    "                                                                                'weighted_multitask_loss':weighted_multitask_loss})\n",
    "    y_pred = model.predict(x_test, batch_size=32)\n",
    "    boool = y_pred >= 0.5\n",
    "    boool = boool.astype(int)\n",
    "    conf_mat = confusion_matrix(y_gold,boool,np.array(['anger','anticipation','disgust','fear','joy','love','optimism','pessimism','sadness','surprise','trust']))\n",
    "    attr_others=['anger','anticipation','disgust','fear','joy','love','optimism','pessimism','sadness','surprise','trust']\n",
    "    plot_correlation_matrix(boool,attr_others)\n",
    "    classes = ['anger','anticipation','disgust','fear','joy','love','optimism','pessimism','sadness','surprise','trust']\n",
    "    print(jaccard_similarity_score(y_gold, boool))\n",
    "    print(classification_report(y_gold, boool))\n",
    "# tData.loc[:,['anger','anticipation','disgust','fear','joy','love','optimism','pessimism','sadness','surprise','trust']] = boool\n",
    "# tData.to_csv(OUT_FILE,header=True, index=True,sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = get_cnn_model_v3()\n",
    "model.summary()\n",
    "train_and_evaluate_model(model, X_tra, y_tra, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "model = get_cnn_model_v3()\n",
    "model.summary()\n",
    "train_and_evaluate_model(model, X_tra, y_tra, X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pred_data = pd.read_table(\"Prediction/E-C_en_pred_exp_47.txt\",sep=\"\\t\", index_col=0, quoting=csv.QUOTE_NONE)\n",
    "pd.set_option('display.max_colwidth', -1)\n",
    "gold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "attr_others=['anger','anticipation','disgust','fear','joy','love','optimism','pessimism','sadness','surprise','trust']\n",
    "plot_correlation_matrix(y_train,attr_others)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "score,acc,f,re,p = model.evaluate(x_test, y_test)\n",
    "\n",
    "print(score,acc,f,re,p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.utils import plot_model\n",
    "plot_model(model, show_shapes=True,to_file='model_98.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hist.history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "histG = modelG.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),\n",
    "                 callbacks=[reduce_lr,earlyStoper,checkpointer,RocAuc])\n",
    "\n",
    "\n",
    "y_predG = model.predict(x_test, batch_size=1024)\n",
    "booolG = y_predG >= 0.5\n",
    "booolG = booolG.astype(int)\n",
    "testData.loc[:,['anger','anticipation','disgust','fear','joy','love','optimism','pessimism','sadness','surprise','trust']] = boool\n",
    "testData.to_csv(OUT_FILE,header=True, index=True,sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(jaccard_similarity_score(y_gold, boool))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       anger       0.76      0.80      0.78      1101\n",
      "anticipation       0.38      0.21      0.27       425\n",
      "     disgust       0.68      0.82      0.74      1099\n",
      "        fear       0.73      0.75      0.74       485\n",
      "         joy       0.81      0.88      0.84      1442\n",
      "        love       0.58      0.68      0.63       516\n",
      "    optimism       0.63      0.82      0.72      1143\n",
      "   pessimism       0.42      0.30      0.35       375\n",
      "     sadness       0.70      0.71      0.71       960\n",
      "    surprise       0.63      0.10      0.17       170\n",
      "       trust       0.29      0.03      0.06       153\n",
      "\n",
      " avg / total       0.67      0.71      0.68      7869\n",
      "\n",
      "0.5878965209894943\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "from sklearn.metrics import roc_auc_score,jaccard_similarity_score\n",
    "from attdeep import AttentionWeightedAverage\n",
    "OUT_FILE='Prediction/E-C_en_pred_exp_177.txt'\n",
    "from keras.models import load_model\n",
    "modelG = load_model('models/weights_exp_177_preprocessed.hdf5',custom_objects={'AttentionWeightedAverage': AttentionWeightedAverage,\n",
    "                                                                                # 'Attention':Attention,\n",
    "#                                                                                 'AttentionM':AttentionM,\n",
    "#                                                                                 'AttentionMC':AttentionMC,\n",
    "                                                                                'binary_crossentropy_weighted':binary_crossentropy_weighted,\n",
    "#                                                                                  'AttentionWithContext':AttentionWithContext,\n",
    "                                                                                 'dice_coef_loss':dice_coef_loss,\n",
    "                                                                                'dice_coef': dice_coef,\n",
    "                                                                                'jaccard_distance':jaccard_distance,\n",
    "                                                                                'fmeasure':fmeasure,\n",
    "                                                                                'recall':recall,\n",
    "                                                                                'precision':precision,\n",
    "                                                                                'jaccard_coef':jaccard_coef,\n",
    "                                                                                'jaccard_coef_loss':jaccard_coef_loss,\n",
    "#                                                                                 'weighted_binary_crossentropy':weighted_binary_crossentropy,\n",
    "                                                                                'multitask_loss':multitask_loss,\n",
    "                                                                                'weighted_multitask_loss':weighted_multitask_loss})\n",
    "\n",
    "testData = pd.read_table(\"datasets/2018-E-c-En-test.txt\",sep=\"\\t\", index_col=0, quoting=csv.QUOTE_NONE)\n",
    "y_pred = modelG.predict(x_test, batch_size=32)\n",
    "boool = y_pred >= 0.5\n",
    "boool = boool.astype(int)\n",
    "classes = ['anger','anticipation','disgust','fear','joy','love','optimism','pessimism','sadness','surprise','trust']\n",
    "print(classification_report(y_gold, boool, target_names=classes))\n",
    "print(jaccard_similarity_score(y_gold, boool))\n",
    "testData.loc[:,['anger','anticipation','disgust','fear','joy','love','optimism','pessimism','sadness','surprise','trust']] = boool\n",
    "testData.to_csv(OUT_FILE,header=True, index=True,sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = model.predict(x_test, batch_size=32)\n",
    "boool = y_pred >= 0.5\n",
    "boool = boool.astype(int)\n",
    "classes = ['anger','anticipation','disgust','fear','joy','love','optimism','pessimism','sadness','surprise','trust']\n",
    "print(classification_report(y_gold, boool, target_names=classes))\n",
    "print(jaccard_similarity_score(y_gold, boool))\n",
    "testData.loc[:,['anger','anticipation','disgust','fear','joy','love','optimism','pessimism','sadness','surprise','trust']] = boool\n",
    "testData.to_csv(OUT_FILE,header=True, index=True,sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "p_train=modelG.predict(x_train, batch_size=32)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Multi-label accuracy (Jaccard index) between E-C_en_pred_exp_175.txt and 2018-E-c-En-test-gold.txt:\t0.5878965209894943\n",
      "Micro-averaged F1 score between E-C_en_pred_exp_175.txt and 2018-E-c-En-test-gold.txt:\t0.7005424278321591\n",
      "Macro-averaged F1 score between E-C_en_pred_exp_175.txt and 2018-E-c-En-test-gold.txt:\t0.5465999633788241\n"
     ]
    }
   ],
   "source": [
    "%run eval/evaluate.py 3 'Prediction/E-C_en_pred_exp_175.txt' 'Prediction/2018-E-c-En-test-gold.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%run modelCorrelation.py 'Prediction/E-C_en_pred_exp_56.txt' 'Prediction/E-C_en_pred_exp_47.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.plot(y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nb_classes = 11\n",
    "\n",
    "vocab = tokenizer.word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from performance import fmeasure,recall,precision\n",
    "from attdeep import AttentionWeightedAverage\n",
    "from performance import multitask_loss, jaccard_distance,dice_coef_loss, dice_coef,jaccard_coef,dice_coef,fmeasure,recall,precision,jaccard_coef_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "model = load_model('models/weights_exp_38_preprocessed.hdf5',custom_objects={'AttentionWeightedAverage': AttentionWeightedAverage,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'dice_coef_loss':dice_coef_loss,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'dice_coef': dice_coef,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'jaccard_distance':jaccard_distance,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'fmeasure':fmeasure,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'recall':recall,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'precision':precision,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'jaccard_coef':jaccard_coef,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'jaccard_coef_loss':jaccard_coef_loss,\n",
    "                                                            'weighted_multitask_loss':weighted_multitask_loss})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import example_helper\n",
    "from deepmoji.finetuning import (\n",
    "    load_benchmark,\n",
    "    finetune)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print y_va"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_tr, X_va, y_tr, y_va = train_test_split(x_train, y_train, train_size=0.95, random_state=233)\n",
    "lmodel.summary()\n",
    "lmodel, acc = finetune(lmodel, [x_train,x_val,x_test], [y_train, y_val,y_gold], nb_classes,\n",
    "                      16, method='chain-thaw')\n",
    "print('Acc: {}'.format(acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_predG = lmodel.predict(x_test, batch_size=1024)\n",
    "booolG = y_predG >= 0.5\n",
    "booolG = booolG.astype(int)\n",
    "testData.loc[:,['anger','anticipation','disgust','fear','joy','love','optimism','pessimism','sadness','surprise','trust']] = booolG\n",
    "testData.to_csv(OUT_FILE,header=True, index=True,sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gold.to_csv('Prediction/gold.txt',header=True, index=True,sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from sklearn.metrics import confusion_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def confusion_matrix(yt, yp, classes):\n",
    "    instcount = yt.shape[0]\n",
    "    n_classes = classes.shape[0]\n",
    "    mtx = np.zeros((n_classes, 4))\n",
    "    for i in range(instcount):\n",
    "        for c in range(n_classes):\n",
    "            mtx[c,0] += 1 if yt[i,c]==1 and yp[i,c]==1 else 0\n",
    "            mtx[c,1] += 1 if yt[i,c]==1 and yp[i,c]==0 else 0\n",
    "            mtx[c,2] += 1 if yt[i,c]==0 and yp[i,c]==0 else 0\n",
    "            mtx[c,3] += 1 if yt[i,c]==0 and yp[i,c]==1 else 0\n",
    "    mtx = [[m0/(m0+m1), m1/(m0+m1), m2/(m2+m3), m3/(m2+m3)] for m0,m1,m2,m3 in mtx]\n",
    "    plt.figure(num=None, figsize=(2, 10), dpi=100, facecolor='w', edgecolor='k')\n",
    "    plt.imshow(mtx, interpolation='nearest',cmap='Blues')\n",
    "    plt.title(\"title\")\n",
    "    tick_marks = np.arange(n_classes)\n",
    "    plt.xticks(np.arange(4), ['1 - 1','1 - 0','0 - 0','0 - 1'])\n",
    "    plt.yticks(tick_marks, classes)\n",
    "    for i, j in itertools.product(range(n_classes), range(4)):\n",
    "        plt.text(j, i, round(mtx[i][j],2), horizontalalignment=\"center\")\n",
    "\n",
    "    #plt.tight_layout()\n",
    "    plt.ylabel('labels')\n",
    "    plt.xlabel('Predicted')\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import itertools\n",
    "# y_pred = model.predict(x_test, batch_size=32)\n",
    "boool = y_new >= [0.5       , 0.3, 0.5       , 0.4, 0.5       ,\n",
    "       0.4, 0.5       , 0.4, 0.5       , 0.25      ,\n",
    "       0.25      ]\n",
    "# boool = y_new >=0.5\n",
    "boool = boool.astype(int)\n",
    "conf_mat = confusion_matrix(y_gold,boool,np.array(['anger','anticipation','disgust','fear','joy','love','optimism','pessimism','sadness','surprise','trust']))\n",
    "attr_others=['anger','anticipation','disgust','fear','joy','love','optimism','pessimism','sadness','surprise','trust']\n",
    "plot_correlation_matrix(boool,attr_others)\n",
    "print(classification_report(y_gold, boool, target_names=classes))\n",
    "print(jaccard_similarity_score(y_gold, boool))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "pd.scatter_matrix(trainData.loc[:,['anger','anticipation','disgust','fear','joy','love','optimism','pessimism','sadness','surprise','trust']], alpha = 0.2, figsize = (15,15), diagonal = 'kde')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "ass = trainData.drop('Tweet',1).sum()\n",
    "ass = pd.DataFrame(ass,columns=['s'])\n",
    "ass['w'] = 1- (ass['s']/ass['s'].sum())\n",
    "weights = ass['w'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "   \n",
    "def plot_correlation_matrix(df,col):\n",
    "    \"\"\"Takes a pandas dataframe as input\"\"\"\n",
    "    df = pd.DataFrame(df,columns=col)\n",
    "    fig, ax = plt.subplots(nrows=1, ncols=1,figsize=(10, 10))\n",
    "    \n",
    "    cax = ax.matshow(df.corr(),cmap=plt.get_cmap('seismic'))\n",
    "\n",
    "    ticks = list(range(len(df.columns)))\n",
    "    ax.set_xticks(ticks)\n",
    "    ax.set_yticks(ticks)\n",
    "\n",
    "    ax.set_xticklabels(df.columns, rotation=20, horizontalalignment='left')\n",
    "    ax.set_yticklabels(df.columns)\n",
    "    fig.colorbar(cax)\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "attr_others=['anger','anticipation','disgust','fear','joy','love','optimism','pessimism','sadness','surprise','trust']\n",
    "plot_correlation_matrix(y_train,attr_others)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "mo1 = load_model('models/weights_exp_38_preprocessed.hdf5',custom_objects={'AttentionWeightedAverage': AttentionWeightedAverage,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'dice_coef_loss':dice_coef_loss,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'dice_coef': dice_coef,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'jaccard_distance':jaccard_distance,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'fmeasure':fmeasure,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'recall':recall,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'precision':precision,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'jaccard_coef':jaccard_coef,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'jaccard_coef_loss':jaccard_coef_loss,\n",
    "                                                            'weighted_multitask_loss':weighted_multitask_loss})\n",
    "mo2 = load_model('models/weights_exp_47_preprocessed.hdf5',custom_objects={'AttentionWeightedAverage': AttentionWeightedAverage,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'dice_coef_loss':dice_coef_loss,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'dice_coef': dice_coef,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'jaccard_distance':jaccard_distance,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'fmeasure':fmeasure,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'recall':recall,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'precision':precision,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'jaccard_coef':jaccard_coef,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'jaccard_coef_loss':jaccard_coef_loss,\n",
    "                                                            'weighted_multitask_loss':weighted_multitask_loss})\n",
    "mo3 = load_model('models/weights_exp_46_preprocessed.hdf5',custom_objects={'AttentionWeightedAverage': AttentionWeightedAverage,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'dice_coef_loss':dice_coef_loss,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'dice_coef': dice_coef,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'jaccard_distance':jaccard_distance,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'fmeasure':fmeasure,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'recall':recall,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'precision':precision,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'jaccard_coef':jaccard_coef,\n",
    "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t'jaccard_coef_loss':jaccard_coef_loss,\n",
    "                                                            'weighted_multitask_loss':weighted_multitask_loss})\n",
    "models = [mo1,mo2,mo3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def ensemble(models, model_input):\n",
    "    \n",
    "    outputs = [model.outputs[0] for model in models]\n",
    "    y = Average()(outputs)\n",
    "    \n",
    "    model = Model(model_input, y, name='ensemble')\n",
    "    \n",
    "    return model\n",
    "model_input = Input(shape=(maxlen, ))\n",
    "ensemble_model = ensemble(models, model_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred1 = pd.read_table(\"Prediction/E-C_en_pred_exp_47.txt\", sep=\"\\t\", index_col=0, quoting=csv.QUOTE_NONE)\n",
    "pred2 = pd.read_table(\"Prediction/E-C_en_pred_exp_49.txt\", sep=\"\\t\", index_col=0, quoting=csv.QUOTE_NONE)\n",
    "pred3 = pd.read_table(\"Prediction/E-C_en_pred_exp_34.txt\", sep=\"\\t\", index_col=0, quoting=csv.QUOTE_NONE)\n",
    "\n",
    "y_pred1 = pred1.loc[:, pred1.columns != 'Tweet'].as_matrix()\n",
    "y_pred1 = np.array(y_pred1.tolist())\n",
    "\n",
    "y_pred2 = pred2.loc[:, pred2.columns != 'Tweet'].as_matrix()\n",
    "y_pred2 = np.array(y_pred2.tolist())\n",
    "\n",
    "y_pred3 = pred3.loc[:, pred3.columns != 'Tweet'].as_matrix()\n",
    "y_pred3 = np.array(y_pred3.tolist())\n",
    "\n",
    "\n",
    "avg_pred = (y_pred3+ y_pred1+ y_pred2)/3.0\n",
    "\n",
    "tData = pd.read_table(\"datasets/2018-E-c-En-test.txt\",sep=\"\\t\", index_col=0, quoting=csv.QUOTE_NONE)\n",
    "boool = avg_pred >= 0.5\n",
    "boool = boool.astype(int)\n",
    "tData.loc[:,['anger','anticipation','disgust','fear','joy','love','optimism','pessimism','sadness','surprise','trust']] = boool\n",
    "tData.to_csv('ensemble_5.txt',header=True, index=True,sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "def reliability_curve(y_true, y_score, bins=10, normalize=False):\n",
    "    \"\"\"Compute reliability curve\n",
    "\n",
    "    Reliability curves allow checking if the predicted probabilities of a\n",
    "    binary classifier are well calibrated. This function returns two arrays\n",
    "    which encode a mapping from predicted probability to empirical probability.\n",
    "    For this, the predicted probabilities are partitioned into equally sized\n",
    "    bins and the mean predicted probability and the mean empirical probabilties\n",
    "    in the bins are computed. For perfectly calibrated predictions, both\n",
    "    quantities whould be approximately equal (for sufficiently many test\n",
    "    samples).\n",
    "\n",
    "    Note: this implementation is restricted to binary classification.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "\n",
    "    y_true : array, shape = [n_samples]\n",
    "        True binary labels (0 or 1).\n",
    "\n",
    "    y_score : array, shape = [n_samples]\n",
    "        Target scores, can either be probability estimates of the positive\n",
    "        class or confidence values. If normalize is False, y_score must be in\n",
    "        the interval [0, 1]\n",
    "\n",
    "    bins : int, optional, default=10\n",
    "        The number of bins into which the y_scores are partitioned.\n",
    "        Note: n_samples should be considerably larger than bins such that\n",
    "              there is sufficient data in each bin to get a reliable estimate\n",
    "              of the reliability\n",
    "\n",
    "    normalize : bool, optional, default=False\n",
    "        Whether y_score needs to be normalized into the bin [0, 1]. If True,\n",
    "        the smallest value in y_score is mapped onto 0 and the largest one\n",
    "        onto 1.\n",
    "\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    y_score_bin_mean : array, shape = [bins]\n",
    "        The mean predicted y_score in the respective bins.\n",
    "\n",
    "    empirical_prob_pos : array, shape = [bins]\n",
    "        The empirical probability (frequency) of the positive class (+1) in the\n",
    "        respective bins.\n",
    "\n",
    "\n",
    "    References\n",
    "    ----------\n",
    "    .. [1] `Predicting Good Probabilities with Supervised Learning\n",
    "            <http://machinelearning.wustl.edu/mlpapers/paper_files/icml2005_Niculescu-MizilC05.pdf>`_\n",
    "\n",
    "    \"\"\"\n",
    "    if normalize:  # Normalize scores into bin [0, 1]\n",
    "        y_score = (y_score - y_score.min()) / (y_score.max() - y_score.min())\n",
    "\n",
    "    bin_width = 1.0 / bins\n",
    "    bin_centers = np.linspace(0, 1.0 - bin_width, bins) + bin_width / 2\n",
    "\n",
    "    y_score_bin_mean = np.empty(bins)\n",
    "    empirical_prob_pos = np.empty(bins)\n",
    "    for i, threshold in enumerate(bin_centers):\n",
    "        # determine all samples where y_score falls into the i-th bin\n",
    "        bin_idx = np.logical_and(threshold - bin_width / 2 < y_score,\n",
    "                                 y_score <= threshold + bin_width / 2)\n",
    "        # Store mean y_score and mean empirical probability of positive class\n",
    "        y_score_bin_mean[i] = y_score[bin_idx].mean()\n",
    "        empirical_prob_pos[i] = y_true[bin_idx].mean()\n",
    "    return y_score_bin_mean, empirical_prob_pos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "reliability_scores = {}\n",
    "for i in range(11):\n",
    "    reliability_scores[i]=reliability_curve(y_gold[:,i],y_pred[:,i])\n",
    "    \n",
    "for method, (y_score_bin_mean, empirical_prob_pos) in reliability_scores.items():\n",
    "    plt.figure(method, figsize=(8, 8))\n",
    "    plt.subplot2grid((3, 1), (0, 0), rowspan=2)\n",
    "    plt.plot([0.0, 1.0], [0.0, 1.0], 'k', label=\"Perfect\")\n",
    "    scores_not_nan = np.logical_not(np.isnan(empirical_prob_pos))\n",
    "    plt.plot(y_score_bin_mean[scores_not_nan],\n",
    "             empirical_prob_pos[scores_not_nan], label=method)\n",
    "plt.ylabel(\"Empirical probability\")\n",
    "plt.legend(loc=0)\n",
    "\n",
    "# plt.subplot2grid((3, 1), (2, 0))\n",
    "# for i in range(11):\n",
    "#     y_score_ = y_pred[:,i]\n",
    "#     y_score_ = (y_score_ - y_score_.min()) / (y_score_.max() - y_score_.min())\n",
    "#     plt.hist(y_score_, range=(0, 1), bins=10, label=i,\n",
    "#              histtype=\"step\", lw=2)\n",
    "plt.xlabel(\"Predicted Probability\")\n",
    "plt.ylabel(\"Count\")\n",
    "plt.legend(loc='upper center', ncol=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.isotonic import IsotonicRegression as IR\n",
    "y_new = np.zeros(y_pred.shape)\n",
    "# for t in range(11):\n",
    "#     ir = IR( out_of_bounds = 'clip' )\n",
    "#     ir.fit( p_train[:,t], y_train[:,t] )\n",
    "#     y_new[:,t] = ir.transform( y_pred[:,t] )\n",
    "\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression as LR\n",
    "for t in range(11):\n",
    "    lr = LR()                                                       \n",
    "    lr.fit( p_train[:,t].reshape( -1, 1 ),  y_train[:,t] )     # LR needs X to be 2-dimensional\n",
    "    y_new[:,t] = lr.predict_proba( y_pred[:,t].reshape( -1, 1 ))[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# y_new = np.zeros(y_pred.shape)\n",
    "y_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.plot(p_calibrated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print p_calibrated\n",
    "reliability_scoress = {}\n",
    "for i in range(1):\n",
    "    reliability_scoress[i]=reliability_curve(y_gold[:,10],p_calibrated)\n",
    "    \n",
    "for method, (y_score_bin_mean, empirical_prob_pos) in reliability_scoress.items():\n",
    "    plt.figure(method, figsize=(8, 8))\n",
    "    plt.subplot2grid((3, 1), (0, 0), rowspan=2)\n",
    "    plt.plot([0.0, 1.0], [0.0, 1.0], 'k', label=\"Perfect\")\n",
    "    scores_not_nan = np.logical_not(np.isnan(empirical_prob_pos))\n",
    "    plt.plot(y_score_bin_mean[scores_not_nan],\n",
    "             empirical_prob_pos[scores_not_nan], label=method)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}