03_Tweets_Sentiment_Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "import re\n",
    "import numpy as np\n",
    "import nltk\n",
    "import itertools\n",
    "from sklearn.metrics import precision_recall_curve, auc, confusion_matrix\n",
    "from sklearn.metrics import classification_report, f1_score, fbeta_score, precision_score, recall_score\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from joblib import dump, load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class StemmedTfidfVectorizer(TfidfVectorizer):\n",
    "    def build_analyzer(self):\n",
    "        analyzer = super(TfidfVectorizer, self).build_analyzer()\n",
    "        return lambda doc: english_stemmer.stemWords(analyzer(doc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import Stemmer\n",
    "english_stemmer = Stemmer.Stemmer('en')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tweet Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Topic</th>\n",
       "      <th>Sentiment</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>apple</td>\n",
       "      <td>positive</td>\n",
       "      <td>Now all @Apple has to do is get swype on the i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>apple</td>\n",
       "      <td>positive</td>\n",
       "      <td>@Apple will be adding more carrier support to ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>apple</td>\n",
       "      <td>positive</td>\n",
       "      <td>Hilarious @youtube video - guy does a duet wit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>apple</td>\n",
       "      <td>positive</td>\n",
       "      <td>@RIM you made it too easy for me to switch to ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>apple</td>\n",
       "      <td>positive</td>\n",
       "      <td>I just realized that the reason I got into twi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Topic Sentiment                                               text\n",
       "0  apple  positive  Now all @Apple has to do is get swype on the i...\n",
       "1  apple  positive  @Apple will be adding more carrier support to ...\n",
       "2  apple  positive  Hilarious @youtube video - guy does a duet wit...\n",
       "3  apple  positive  @RIM you made it too easy for me to switch to ...\n",
       "4  apple  positive  I just realized that the reason I got into twi..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SANTweets = pd.read_csv(\"dataset/SandersAnalyticsTweets.csv\")\n",
    "SANTweets[\"text\"] = SANTweets[\"TweetText\"]\n",
    "SANTweets = SANTweets[[\"Topic\", \"Sentiment\", \"text\"]]\n",
    "SANTweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5113, 3)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SANTweets.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sentiment\n",
       "irrelevant    1689\n",
       "negative       572\n",
       "neutral       2333\n",
       "positive       519\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SANTweets.groupby(\"Sentiment\").size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Topic\n",
       "apple        1142\n",
       "google       1317\n",
       "microsoft    1364\n",
       "twitter      1290\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SANTweets.groupby(\"Topic\").size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate_sent(s):\n",
    "    if s == \"irrelevant\" or s == \"neutral\":\n",
    "        s = 0\n",
    "    if s == \"positive\":\n",
    "        s = 1\n",
    "    if s == \"negative\":\n",
    "        s = -1\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "SANTweets[\"sent\"] = SANTweets[\"Sentiment\"].apply(translate_sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Topic</th>\n",
       "      <th>Sentiment</th>\n",
       "      <th>text</th>\n",
       "      <th>sent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>apple</td>\n",
       "      <td>positive</td>\n",
       "      <td>Now all @Apple has to do is get swype on the i...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>apple</td>\n",
       "      <td>positive</td>\n",
       "      <td>@Apple will be adding more carrier support to ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>apple</td>\n",
       "      <td>positive</td>\n",
       "      <td>Hilarious @youtube video - guy does a duet wit...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>apple</td>\n",
       "      <td>positive</td>\n",
       "      <td>@RIM you made it too easy for me to switch to ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>apple</td>\n",
       "      <td>positive</td>\n",
       "      <td>I just realized that the reason I got into twi...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Topic Sentiment                                               text  sent\n",
       "0  apple  positive  Now all @Apple has to do is get swype on the i...     1\n",
       "1  apple  positive  @Apple will be adding more carrier support to ...     1\n",
       "2  apple  positive  Hilarious @youtube video - guy does a duet wit...     1\n",
       "3  apple  positive  @RIM you made it too easy for me to switch to ...     1\n",
       "4  apple  positive  I just realized that the reason I got into twi...     1"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SANTweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lang</th>\n",
       "      <th>user_screen_name</th>\n",
       "      <th>text</th>\n",
       "      <th>sent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>en</td>\n",
       "      <td>mrsshinde</td>\n",
       "      <td>RT @mrsshinde: @SamsungMobile @Moto @oneplus @...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>en</td>\n",
       "      <td>olutobi_og</td>\n",
       "      <td>@SamsungMobile kindly include play next in the...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>en</td>\n",
       "      <td>themobileindian</td>\n",
       "      <td>@SamsungMobile has started rolling out the And...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>en</td>\n",
       "      <td>sobakhani</td>\n",
       "      <td>@SamsungMobile how to find lost Samsung note 1...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>en</td>\n",
       "      <td>Imchetan_p</td>\n",
       "      <td>@SamsungMobile @SamsungIndia @Samsung I must s...</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  lang user_screen_name                                               text  \\\n",
       "0   en        mrsshinde  RT @mrsshinde: @SamsungMobile @Moto @oneplus @...   \n",
       "1   en       olutobi_og  @SamsungMobile kindly include play next in the...   \n",
       "2   en  themobileindian  @SamsungMobile has started rolling out the And...   \n",
       "3   en        sobakhani  @SamsungMobile how to find lost Samsung note 1...   \n",
       "4   en       Imchetan_p  @SamsungMobile @SamsungIndia @Samsung I must s...   \n",
       "\n",
       "   sent  \n",
       "0     0  \n",
       "1     0  \n",
       "2     0  \n",
       "3     0  \n",
       "4    -1  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SMTweets = pd.read_csv(\"dataset/SamsungTweetsSent.csv\")\n",
    "SMTweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sent\n",
       "-1    132\n",
       " 0    138\n",
       " 1     19\n",
       "dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SMTweets.groupby(\"sent\").size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lang</th>\n",
       "      <th>text</th>\n",
       "      <th>username</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>en</td>\n",
       "      <td>@Razer debuted an incredibly compact all-in-on...</td>\n",
       "      <td>techthelead</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>en</td>\n",
       "      <td>In keynote address, @Delta Unveils New #OOH Pa...</td>\n",
       "      <td>YourOAAA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>en</td>\n",
       "      <td>👍 We are ready for Day 2 at #CES2020. Discover...</td>\n",
       "      <td>Sio_db</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>en</td>\n",
       "      <td>GO-&amp;gt; #CES2020 #France #USA !\\n#BusinessFran...</td>\n",
       "      <td>dillardmarg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>en</td>\n",
       "      <td>The industry's smallest and lightest 4K60P pro...</td>\n",
       "      <td>HoldanBlog</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  lang                                               text     username\n",
       "0   en  @Razer debuted an incredibly compact all-in-on...  techthelead\n",
       "1   en  In keynote address, @Delta Unveils New #OOH Pa...     YourOAAA\n",
       "2   en  👍 We are ready for Day 2 at #CES2020. Discover...       Sio_db\n",
       "3   en  GO-&gt; #CES2020 #France #USA !\\n#BusinessFran...  dillardmarg\n",
       "4   en  The industry's smallest and lightest 4K60P pro...   HoldanBlog"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CESTweets = pd.read_csv(\"dataset/ces2020_tweets_full_text.csv\")\n",
    "CESTweets.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cleaning and Preprocessing Tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dictionaries and preprocessing function from previous notebook:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "emoticon_repl = {\n",
    "    # positive emoticons\n",
    "    r\":-?d+\": \" good \", r\":[- ]?\\)+\": \" good \", r\";-?\\)+\": \" good \",\n",
    "    r\"\\(+-?:\": \" good \", r\"=\\)+\" : \" good \", r\"\\b<3\\b\" : \" good \",    \n",
    "    # negative emoticons\n",
    "    r\"[\\s\\r\\t\\n]+:/+\": \" bad \", r\":\\\\+\": \" bad \", r\"[\\s\\r\\t\\n]+\\)-?:\": \" bad \",\n",
    "    r\":-?\\(+\": \" bad \", r\"[\\s\\t\\r\\n]+d+-?:\": \" bad \"\n",
    "}\n",
    "\n",
    "contracted_repl = {\n",
    "    # casi particolari\n",
    "    r\"won\\'t\" : \"will not\", r\"won\\'\" : \"will not\", r\"can\\'t\": \"can not\", r\"shan\\'t\": \"shall not\",\n",
    "    r\"shan\\'\": \"shall not\", r\"ain\\'t\": \"is not\", r\"ain\\'\": \"is not\",\n",
    "    # casi generali\n",
    "    r\"n\\'t\": \" not\", r\"\\'t\": \" not\", r\"n\\'\": \" not\", r\"\\'s\": \" is\", r\"\\'ve\": \" have\", \n",
    "    r\"\\'re\": \" are\", \n",
    "    r\"\\'ll\": \" will\", r\"\\'d\": \" would\",\n",
    "}\n",
    "\n",
    "with open('dataset/slang_subset_manual.json', 'r') as fid:\n",
    "    slang_repl = json.load(fid)\n",
    "    \n",
    "def preprocess(sent, translate_slang = True):\n",
    "    \n",
    "    sent = sent.lower()\n",
    "    sent = re.sub(r'^<div id=\"video.*>&nbsp;', '', sent) # Video-review part\n",
    "    sent = re.sub('https?://[A-Za-z0-9./]+', '', sent) # URLs\n",
    "    \n",
    "    for k in emoticon_repl:\n",
    "        sent = re.sub(k, emoticon_repl[k], sent)\n",
    "\n",
    "    if translate_slang:\n",
    "        for k in slang_repl:\n",
    "            sent = re.sub(r\"\\b\"+k+r\"\\b\", slang_repl[k], sent)\n",
    "        \n",
    "    for k in contracted_repl:\n",
    "        sent = re.sub(k, contracted_repl[k], sent)\n",
    "    \n",
    "    sent = re.sub('[/]+', ' ', sent) # word1/word2 to word1 word2\n",
    "    sent = re.sub('[^A-Za-z0-9-_ ]+', '', sent)\n",
    "    sent = re.sub('\\b\\d+\\b', '', sent)\n",
    "    \n",
    "    return sent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function for preprocessing tweets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_tweets(tweets_df):\n",
    "    from spellchecker import SpellChecker\n",
    "    spell = SpellChecker(distance=1)\n",
    "    \n",
    "    tweets_df[\"textPreprocessed\"] = tweets_df.text\n",
    "    tweets_df[\"textPreprocessed\"] =  tweets_df[\"textPreprocessed\"].str.replace(\"@\\w+\", \"\") # remove AT's\n",
    "    tweets_df[\"textPreprocessed\"] = tweets_df[\"textPreprocessed\"].str.replace(\"^(RT)+\", \"\") # Remove RT at beginning of retweets\n",
    "    \n",
    "    # Add stuff probably\n",
    "    \n",
    "    tweets_df[\"textPreprocessed\"] = tweets_df[\"textPreprocessed\"].apply(preprocess)\n",
    "    tweets_df[\"textPreprocessed\"] = tweets_df[\"textPreprocessed\"].apply(\n",
    "        lambda x : \" \".join([spell.correction(el) for el in x.split()]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Applying this function to the datasets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lang</th>\n",
       "      <th>text</th>\n",
       "      <th>username</th>\n",
       "      <th>textPreprocessed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>en</td>\n",
       "      <td>@Razer debuted an incredibly compact all-in-on...</td>\n",
       "      <td>techthelead</td>\n",
       "      <td>debuted an incredibly compact all-in-one syste...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>en</td>\n",
       "      <td>In keynote address, @Delta Unveils New #OOH Pa...</td>\n",
       "      <td>YourOAAA</td>\n",
       "      <td>in keynote address unveils new ooh parallel re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>en</td>\n",
       "      <td>👍 We are ready for Day 2 at #CES2020. Discover...</td>\n",
       "      <td>Sio_db</td>\n",
       "      <td>we are ready for day 2 at ces2020 discover our...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>en</td>\n",
       "      <td>GO-&amp;gt; #CES2020 #France #USA !\\n#BusinessFran...</td>\n",
       "      <td>dillardmarg</td>\n",
       "      <td>go-go ces2020 france usa businessfrance ice in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>en</td>\n",
       "      <td>The industry's smallest and lightest 4K60P pro...</td>\n",
       "      <td>HoldanBlog</td>\n",
       "      <td>the industry is smallest and lightest 4k60p pr...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  lang                                               text     username  \\\n",
       "0   en  @Razer debuted an incredibly compact all-in-on...  techthelead   \n",
       "1   en  In keynote address, @Delta Unveils New #OOH Pa...     YourOAAA   \n",
       "2   en  👍 We are ready for Day 2 at #CES2020. Discover...       Sio_db   \n",
       "3   en  GO-&gt; #CES2020 #France #USA !\\n#BusinessFran...  dillardmarg   \n",
       "4   en  The industry's smallest and lightest 4K60P pro...   HoldanBlog   \n",
       "\n",
       "                                    textPreprocessed  \n",
       "0  debuted an incredibly compact all-in-one syste...  \n",
       "1  in keynote address unveils new ooh parallel re...  \n",
       "2  we are ready for day 2 at ces2020 discover our...  \n",
       "3  go-go ces2020 france usa businessfrance ice in...  \n",
       "4  the industry is smallest and lightest 4k60p pr...  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess_tweets(CESTweets)\n",
    "CESTweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lang</th>\n",
       "      <th>user_screen_name</th>\n",
       "      <th>text</th>\n",
       "      <th>sent</th>\n",
       "      <th>textPreprocessed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>en</td>\n",
       "      <td>mrsshinde</td>\n",
       "      <td>RT @mrsshinde: @SamsungMobile @Moto @oneplus @...</td>\n",
       "      <td>0</td>\n",
       "      <td>we must work to save safeguard humans from mob...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>en</td>\n",
       "      <td>olutobi_og</td>\n",
       "      <td>@SamsungMobile kindly include play next in the...</td>\n",
       "      <td>0</td>\n",
       "      <td>kindly include play next in the next samsungmu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>en</td>\n",
       "      <td>themobileindian</td>\n",
       "      <td>@SamsungMobile has started rolling out the And...</td>\n",
       "      <td>0</td>\n",
       "      <td>has started rolling out the android 10 update ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>en</td>\n",
       "      <td>sobakhani</td>\n",
       "      <td>@SamsungMobile how to find lost Samsung note 1...</td>\n",
       "      <td>0</td>\n",
       "      <td>how to find lost samsung note 10 plus in pakistan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>en</td>\n",
       "      <td>Imchetan_p</td>\n",
       "      <td>@SamsungMobile @SamsungIndia @Samsung I must s...</td>\n",
       "      <td>-1</td>\n",
       "      <td>i must say that your sales services really suc...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  lang user_screen_name                                               text  \\\n",
       "0   en        mrsshinde  RT @mrsshinde: @SamsungMobile @Moto @oneplus @...   \n",
       "1   en       olutobi_og  @SamsungMobile kindly include play next in the...   \n",
       "2   en  themobileindian  @SamsungMobile has started rolling out the And...   \n",
       "3   en        sobakhani  @SamsungMobile how to find lost Samsung note 1...   \n",
       "4   en       Imchetan_p  @SamsungMobile @SamsungIndia @Samsung I must s...   \n",
       "\n",
       "   sent                                   textPreprocessed  \n",
       "0     0  we must work to save safeguard humans from mob...  \n",
       "1     0  kindly include play next in the next samsungmu...  \n",
       "2     0  has started rolling out the android 10 update ...  \n",
       "3     0  how to find lost samsung note 10 plus in pakistan  \n",
       "4    -1  i must say that your sales services really suc...  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess_tweets(SMTweets)\n",
    "SMTweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Topic</th>\n",
       "      <th>Sentiment</th>\n",
       "      <th>text</th>\n",
       "      <th>sent</th>\n",
       "      <th>textPreprocessed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>apple</td>\n",
       "      <td>positive</td>\n",
       "      <td>Now all @Apple has to do is get swype on the i...</td>\n",
       "      <td>1</td>\n",
       "      <td>now all has to do is get swipe on the phone an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>apple</td>\n",
       "      <td>positive</td>\n",
       "      <td>@Apple will be adding more carrier support to ...</td>\n",
       "      <td>1</td>\n",
       "      <td>will be adding more carrier support to the pho...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>apple</td>\n",
       "      <td>positive</td>\n",
       "      <td>Hilarious @youtube video - guy does a duet wit...</td>\n",
       "      <td>1</td>\n",
       "      <td>hilarious video - guy does a duet with is sir ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>apple</td>\n",
       "      <td>positive</td>\n",
       "      <td>@RIM you made it too easy for me to switch to ...</td>\n",
       "      <td>1</td>\n",
       "      <td>you made it too easy for me to switch to phone...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>apple</td>\n",
       "      <td>positive</td>\n",
       "      <td>I just realized that the reason I got into twi...</td>\n",
       "      <td>1</td>\n",
       "      <td>i just realized that the reason i got into twi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Topic Sentiment                                               text  sent  \\\n",
       "0  apple  positive  Now all @Apple has to do is get swype on the i...     1   \n",
       "1  apple  positive  @Apple will be adding more carrier support to ...     1   \n",
       "2  apple  positive  Hilarious @youtube video - guy does a duet wit...     1   \n",
       "3  apple  positive  @RIM you made it too easy for me to switch to ...     1   \n",
       "4  apple  positive  I just realized that the reason I got into twi...     1   \n",
       "\n",
       "                                    textPreprocessed  \n",
       "0  now all has to do is get swipe on the phone an...  \n",
       "1  will be adding more carrier support to the pho...  \n",
       "2  hilarious video - guy does a duet with is sir ...  \n",
       "3  you made it too easy for me to switch to phone...  \n",
       "4  i just realized that the reason i got into twi...  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess_tweets(SANTweets)\n",
    "SANTweets.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sentiwordnet Sentiment Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package universal_tagset to\n",
      "[nltk_data]     C:\\Users\\gianc\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package universal_tagset is already up-to-date!\n",
      "[nltk_data] Downloading package sentiwordnet to\n",
      "[nltk_data]     C:\\Users\\gianc\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package sentiwordnet is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "from nltk.stem import WordNetLemmatizer, PorterStemmer\n",
    "from nltk.corpus import sentiwordnet as swn\n",
    "nltk.download('universal_tagset')\n",
    "nltk.download('sentiwordnet')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The SWNClassifier takes in input the pre-processed tweets and  works as follows:\n",
    "\n",
    "\n",
    "- Each tweet is split in tokens using nltk's tokenizer;\n",
    "- A *Part of Speech* tag is assigned to each token with nltk's `pos_tag` function;\n",
    "- Each tag is translated to a SentiWordNet tag;\n",
    "- Each token/tag pair is assigned the positivity and negativity score defined by SentiWordNet;\n",
    "- The positivity and negativity score of a tweet is computed as the sum of positivity and negativity scores of the constituting tokens;\n",
    "- If both positivity and negativity scores are 0, the tweet is labelled as neutral. If the positivity score is greater than the negativity score, the tweet is labelled as positive, and negative otherwise.\n",
    "\n",
    "The function returns, for a list of tweets:\n",
    "\n",
    "- Their tokens and their tags\n",
    "- Their positivity score\n",
    "- Their negativity score\n",
    "- Their sentiment score (-1 for negative, 0 for neutral, 1 for positive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SWNClassifier(X):\n",
    "    # Adapted from https://towardsdatascience.com/sentiment-analysis-on-swachh-bharat-using-twitter-216369cfa534\n",
    "    lem = WordNetLemmatizer()\n",
    "    pstem = PorterStemmer()\n",
    "    X_tagged = []\n",
    "    li_swn=[]\n",
    "    li_swn_pos=[]\n",
    "    li_swn_neg=[]\n",
    "    missing_words=[]\n",
    "    for i in range(len(X)):\n",
    "        text = X[i]\n",
    "        tokens = nltk.word_tokenize(text)\n",
    "        tagged_sent = nltk.tag.pos_tag(tokens)\n",
    "        store_it = [(word, nltk.tag.map_tag('en-ptb', 'universal', tag)) for word, tag in tagged_sent]\n",
    "        X_tagged.append(store_it)\n",
    "        #print(\"Tagged Parts of Speech:\",store_it)\n",
    "\n",
    "        pos_total=0\n",
    "        neg_total=0\n",
    "        for word,tag in store_it:\n",
    "            # print(tag)\n",
    "            if(tag=='NOUN'):\n",
    "                tag='n'\n",
    "            elif(tag=='VERB'):\n",
    "                tag='v'\n",
    "            elif(tag=='ADJ'):\n",
    "                tag='a'\n",
    "            elif(tag=='ADV'):\n",
    "                tag = 'r'\n",
    "            else:\n",
    "                tag='nothing'\n",
    "\n",
    "                \n",
    "            if(tag!='nothing'):\n",
    "                concat = word+'.'+tag+'.01'\n",
    "                try:\n",
    "                    this_word_pos=swn.senti_synset(concat).pos_score()\n",
    "                    this_word_neg=swn.senti_synset(concat).neg_score()\n",
    "                    # print(word,tag,':',this_word_pos,this_word_neg)\n",
    "                except Exception as e:\n",
    "                    wor = lem.lemmatize(word)\n",
    "                    concat = wor+'.'+tag+'.01'\n",
    "                    # Checking if there's a possiblity of lemmatized word be accepted into SWN corpus\n",
    "                    try:\n",
    "                        this_word_pos=swn.senti_synset(concat).pos_score()\n",
    "                        this_word_neg=swn.senti_synset(concat).neg_score()\n",
    "                    except Exception as e:\n",
    "                        wor = pstem.stem(word)\n",
    "                        concat = wor+'.'+tag+'.01'\n",
    "                        # Checking if there's a possiblity of lemmatized word be accepted\n",
    "                        try:\n",
    "                            this_word_pos=swn.senti_synset(concat).pos_score()\n",
    "                            this_word_neg=swn.senti_synset(concat).neg_score()\n",
    "                        except:\n",
    "                            missing_words.append(word)\n",
    "                            continue\n",
    "                pos_total+=this_word_pos\n",
    "                neg_total+=this_word_neg\n",
    "        li_swn_pos.append(pos_total)\n",
    "        li_swn_neg.append(neg_total)\n",
    "\n",
    "        if(pos_total!=0 or neg_total!=0):\n",
    "            if(pos_total>neg_total):\n",
    "                li_swn.append(1)\n",
    "            else:\n",
    "                li_swn.append(-1)\n",
    "        else:\n",
    "            li_swn.append(0)\n",
    "            \n",
    "    return X_tagged, li_swn_pos, li_swn_neg, li_swn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scores(y_test, predictions):\n",
    "\n",
    "    prec = precision_score(y_test, predictions) # Precision\n",
    "    rec = recall_score(y_test, predictions) # Recall\n",
    "    f1 = f1_score(y_test, predictions) # F1\n",
    "    f2 = fbeta_score(y_test, predictions, 2) # F2\n",
    "    cm = confusion_matrix(y_test, predictions)\n",
    "    \n",
    "    scores_strings = [\"Test Precision\",\n",
    "                      \"Test Recall\", \"F1\", \"F2\"]\n",
    "    \n",
    "    scores = [prec, rec, f1, f2]\n",
    "    \n",
    "    print((\"{:20s} {:.5f}\\n\"*4)[:-1].format(*itertools.chain(*zip(scores_strings, scores))))\n",
    "    \n",
    "    print(classification_report(y_test, predictions, digits=5))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluating the SWNClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluating the SWNClassifier on SanTweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us compare the positive/negative/neutral labels assigned by the SWNClassifier with the original labels assigned to SANTweets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 8.11 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "SANTweets_tagged, SANTweets_SWN_POS, SANTweets_SWN_NEG, SANTweets_SWN_SENT = SWNClassifier(SANTweets.textPreprocessed.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "SANTweets_SENT = SANTweets.sent.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluation on positive/negative/neutral labels:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1    0.23529   0.54545   0.32877       572\n",
      "           0    0.93148   0.37519   0.53492      4022\n",
      "           1    0.16844   0.70328   0.27178       519\n",
      "\n",
      "    accuracy                        0.42754      5113\n",
      "   macro avg    0.44507   0.54131   0.37849      5113\n",
      "weighted avg    0.77614   0.42754   0.48514      5113\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(SANTweets_SENT, SANTweets_SWN_SENT, digits=5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluation on Neutral/Sentiment labels:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Precision       0.93148\n",
      "Test Recall          0.37519\n",
      "F1                   0.53492\n",
      "F2                   0.42608\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "       False    0.28056   0.89826   0.42757      1091\n",
      "        True    0.93148   0.37519   0.53492      4022\n",
      "\n",
      "    accuracy                        0.48680      5113\n",
      "   macro avg    0.60602   0.63672   0.48125      5113\n",
      "weighted avg    0.79259   0.48680   0.51201      5113\n",
      "\n"
     ]
    }
   ],
   "source": [
    "scores(SANTweets_SENT==0, np.array(SANTweets_SWN_SENT)==0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluating the SWNClassifier on SMTweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us compare the positive/negative/neutral labels assigned by the SWNClassifier with the manual labels assigned to SMTweets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 499 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "SMTweets_tagged, SMTweets_SWN_POS, SMTweets_SWN_NEG, SMTweets_SWN_SENT = SWNClassifier(SMTweets.textPreprocessed.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMTweets_MANUAL_SENT = SMTweets.sent.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluation on positive/negative/neutral labels:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1    0.66418   0.67424   0.66917       132\n",
      "           0    0.75000   0.15217   0.25301       138\n",
      "           1    0.08661   0.57895   0.15068        19\n",
      "\n",
      "    accuracy                        0.41869       289\n",
      "   macro avg    0.50026   0.46845   0.35762       289\n",
      "weighted avg    0.66719   0.41869   0.43637       289\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(SMTweets_MANUAL_SENT, np.array(SMTweets_SWN_SENT), digits=5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluation on Neutral/Sentiment labels:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Precision       0.75000\n",
      "Test Recall          0.15217\n",
      "F1                   0.25301\n",
      "F2                   0.18103\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "       False    0.55172   0.95364   0.69903       151\n",
      "        True    0.75000   0.15217   0.25301       138\n",
      "\n",
      "    accuracy                        0.57093       289\n",
      "   macro avg    0.65086   0.55291   0.47602       289\n",
      "weighted avg    0.64640   0.57093   0.48605       289\n",
      "\n"
     ]
    }
   ],
   "source": [
    "scores(SMTweets_MANUAL_SENT == 0, np.array(SMTweets_SWN_SENT)==0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluating Multinomial Naive Bayes Classifier from previous notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load joblib files:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import zipfile\n",
    "with zipfile.ZipFile('joblib_data/tfidf_vect_nostemmer.zip', 'r') as zip_ref:\n",
    "    zip_ref.extractall('joblib_data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\gianc\\Anaconda3\\lib\\site-packages\\sklearn\\base.py:306: UserWarning: Trying to unpickle estimator TfidfTransformer from version 0.19.1 when using version 0.21.2. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n",
      "C:\\Users\\gianc\\Anaconda3\\lib\\site-packages\\sklearn\\base.py:306: UserWarning: Trying to unpickle estimator TfidfVectorizer from version 0.19.1 when using version 0.21.2. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n"
     ]
    }
   ],
   "source": [
    "vectorizer = load('joblib_data/tfidf_vect_nostemmer.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\gianc\\Anaconda3\\lib\\site-packages\\sklearn\\base.py:306: UserWarning: Trying to unpickle estimator MultinomialNB from version 0.19.1 when using version 0.21.2. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n"
     ]
    }
   ],
   "source": [
    "clf = load('joblib_data/clf_nb_nostemmer.joblib')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating MNB Classifier on Samsung Mobile Tweet Replies"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we evaluate our classifier on the SMTweets. Because our classifier only outputs positive/negative, we have to filter out neutral tweets. Hence, we take into account:\n",
    "\n",
    "- SMTweets manually labelled as positive/negative;\n",
    "- SMTweets labelled as positive/negative by the SWNClassifier."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SMTweets manually labelled as positive or negative:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMTweets_MANUAL_POS_NEG = SMTweets[SMTweets.sent != 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMTweets_MANUAL_POS_NEG_x = SMTweets_MANUAL_POS_NEG.textPreprocessed.values\n",
    "SMTweets_MANUAL_POS_NEG_y = SMTweets_MANUAL_POS_NEG.sent.values == 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Manual labels: 139 negative, 19 positive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([False,  True]), array([132,  19], dtype=int64))"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(SMTweets_MANUAL_POS_NEG_y, return_counts=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Applying the vectorizer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<151x50000 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 2393 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SMTweets_MANUAL_POS_NEG_x_vect = vectorizer.transform(SMTweets_MANUAL_POS_NEG_x)\n",
    "SMTweets_MANUAL_POS_NEG_x_vect"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predictions from the MNB Classifier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 997 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "predictions = clf.predict(SMTweets_MANUAL_POS_NEG_x_vect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([False,  True]), array([133,  18], dtype=int64))"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(predictions, return_counts = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MNB predictions against the manual labels:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Precision       0.83333\n",
      "Test Recall          0.78947\n",
      "F1                   0.81081\n",
      "F2                   0.79787\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "       False    0.96992   0.97727   0.97358       132\n",
      "        True    0.83333   0.78947   0.81081        19\n",
      "\n",
      "    accuracy                        0.95364       151\n",
      "   macro avg    0.90163   0.88337   0.89220       151\n",
      "weighted avg    0.95274   0.95364   0.95310       151\n",
      "\n"
     ]
    }
   ],
   "source": [
    "scores(SMTweets_MANUAL_POS_NEG_y, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SMTweets labelled as positive or negative by the SWNClassifier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "SMTweets_SWN_SENT = np.array(SMTweets_SWN_SENT)\n",
    "SMTweets_SWN_POS_NEG_x = SMTweets.textPreprocessed.values[SMTweets_SWN_SENT != 0]\n",
    "SMTweets_SWN_POS_NEG_y = SMTweets_SWN_SENT[SMTweets_SWN_SENT != 0] == 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SWN Labels: 134 negative, 127 positive."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([False,  True]), array([134, 127], dtype=int64))"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(SMTweets_SWN_POS_NEG_y, return_counts=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Applying the vectorizer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<261x50000 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 3739 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SMTweets_SWN_POS_NEG_x_vect = vectorizer.transform(SMTweets_SWN_POS_NEG_x)\n",
    "SMTweets_SWN_POS_NEG_x_vect"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predictions from the MNB Classifier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 0 ns\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "predictions = clf.predict(SMTweets_SWN_POS_NEG_x_vect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([False,  True]), array([209,  52], dtype=int64))"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(predictions, return_counts = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MNB predictions against the SWN labels on SMTweets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Precision       0.78846\n",
      "Test Recall          0.32283\n",
      "F1                   0.45810\n",
      "F2                   0.36607\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "       False    0.58852   0.91791   0.71720       134\n",
      "        True    0.78846   0.32283   0.45810       127\n",
      "\n",
      "    accuracy                        0.62835       261\n",
      "   macro avg    0.68849   0.62037   0.58765       261\n",
      "weighted avg    0.68581   0.62835   0.59113       261\n",
      "\n"
     ]
    }
   ],
   "source": [
    "scores(SMTweets_SWN_POS_NEG_y, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating MNB Classifier on Sanders Analytics Tweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we evaluate our classifier on the SANTweets. Because our classifier only outputs positive/negative, we have to filter out neutral tweets. Hence, we take into account:\n",
    "\n",
    "- SANTweets originally labelled as positive/negative;\n",
    "- SANTweets labelled as positive/negative by the SWNClassifier."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SANTweets labelled as positive or negative:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "SANTweets_SENT = SANTweets.sent.values\n",
    "SANTweets_POS_NEG_x = SANTweets.textPreprocessed.values[SANTweets_SENT != 0]\n",
    "SANTweets_POS_NEG_y = SANTweets_SENT[SANTweets_SENT != 0] == 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Original labels: 572 negative, 519 positive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([False,  True]), array([572, 519], dtype=int64))"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(SANTweets_POS_NEG_y, return_counts=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Applying the vectorizer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1091x50000 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 9149 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SANTweets_POS_NEG_x_vect = vectorizer.transform(SANTweets_POS_NEG_x)\n",
    "SANTweets_POS_NEG_x_vect"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predictions from the MNB Classifier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 998 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "predictions = clf.predict(SANTweets_POS_NEG_x_vect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([False,  True]), array([724, 367], dtype=int64))"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(predictions, return_counts = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MNB predictions against the original SANTweets labels:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Precision       0.78202\n",
      "Test Recall          0.55299\n",
      "F1                   0.64786\n",
      "F2                   0.58739\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "       False    0.67956   0.86014   0.75926       572\n",
      "        True    0.78202   0.55299   0.64786       519\n",
      "\n",
      "    accuracy                        0.71402      1091\n",
      "   macro avg    0.73079   0.70656   0.70356      1091\n",
      "weighted avg    0.72830   0.71402   0.70626      1091\n",
      "\n"
     ]
    }
   ],
   "source": [
    "scores(SANTweets_POS_NEG_y, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SANTweets labelled as positive or negative by the SWNClassifier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "SANTweets_SWN_SENT = np.array(SANTweets_SWN_SENT)\n",
    "SANTweets_SWN_POS_NEG_x = SANTweets.textPreprocessed.values[SANTweets_SWN_SENT != 0]\n",
    "SANTweets_SWN_POS_NEG_y = SANTweets_SWN_SENT[SANTweets_SWN_SENT != 0] == 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SWN SANTweets Labels: 1326 negative, 2167 positive."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([False,  True]), array([1326, 2167], dtype=int64))"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(SANTweets_SWN_POS_NEG_y, return_counts=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Applying the vectorizer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<3493x50000 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 27226 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SANTweets_SWN_POS_NEG_x_vect = vectorizer.transform(SANTweets_SWN_POS_NEG_x)\n",
    "SANTweets_SWN_POS_NEG_x_vect"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predictions from the MNB Classifier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 978 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "predictions = clf.predict(SANTweets_SWN_POS_NEG_x_vect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([False,  True]), array([1977, 1516], dtype=int64))"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(predictions, return_counts = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MNB predictions against the SWN labels on SANTweets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Precision       0.73879\n",
      "Test Recall          0.51684\n",
      "F1                   0.60820\n",
      "F2                   0.54988\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "       False    0.47041   0.70136   0.56312      1326\n",
      "        True    0.73879   0.51684   0.60820      2167\n",
      "\n",
      "    accuracy                        0.58689      3493\n",
      "   macro avg    0.60460   0.60910   0.58566      3493\n",
      "weighted avg    0.63691   0.58689   0.59109      3493\n",
      "\n"
     ]
    }
   ],
   "source": [
    "scores(SANTweets_SWN_POS_NEG_y, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating MNB Classifier on CESTweets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Because CESTweets do not have any label, we can only rely on the SWNClassifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "CESTweets_tagged, CESTweets_SWN_POS, CESTweets_SWN_NEG, CESTweets_SWN_SENT = SWNClassifier(CESTweets.textPreprocessed.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CESTweets labelled as positive or negative by the SWNClassifier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "CESTweets_SWN_SENT = np.array(CESTweets_SWN_SENT)\n",
    "CESTweets_SWN_POS_NEG_x = CESTweets.textPreprocessed.values[CESTweets_SWN_SENT != 0]\n",
    "CESTweets_SWN_POS_NEG_y = CESTweets_SWN_SENT[CESTweets_SWN_SENT != 0] == 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SWN CESTweets Labels: 465 negative, 1229 positive."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([False,  True]), array([ 465, 1229], dtype=int64))"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(CESTweets_SWN_POS_NEG_y, return_counts=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Applying the vectorizer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<1694x50000 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 17878 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CESTweets_SWN_POS_NEG_x_vect = vectorizer.transform(CESTweets_SWN_POS_NEG_x)\n",
    "CESTweets_SWN_POS_NEG_x_vect"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predictions from the MNB Classifier:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 998 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "predictions = clf.predict(CESTweets_SWN_POS_NEG_x_vect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([False,  True]), array([915, 779], dtype=int64))"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(predictions, return_counts = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MNB predictions against the SWN labels on SMTweets:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Precision       0.81130\n",
      "Test Recall          0.51424\n",
      "F1                   0.62948\n",
      "F2                   0.55487\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "       False    0.34754   0.68387   0.46087       465\n",
      "        True    0.81130   0.51424   0.62948      1229\n",
      "\n",
      "    accuracy                        0.56080      1694\n",
      "   macro avg    0.57942   0.59906   0.54518      1694\n",
      "weighted avg    0.68400   0.56080   0.58320      1694\n",
      "\n"
     ]
    }
   ],
   "source": [
    "scores(CESTweets_SWN_POS_NEG_y, predictions)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}