hatespeech_by_KNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "data = pd.read_csv(\"data_folder/tweet_dataset.csv\")\n",
    "tweet_list = []\n",
    "label_list = []\n",
    "for each in list(data.iloc[:,2]):\n",
    "    tweet_list.append(each.split())\n",
    "for each in list(data.iloc[:,1]):\n",
    "    label_list.append(each.split())\n",
    "dataset_len = len(tweet_list)\n",
    "train_tweet_list = tweet_list[:dataset_len*6//10]\n",
    "test_tweet_list = tweet_list[dataset_len*6//10:]\n",
    "train_label_list = label_list[:dataset_len*6//10]\n",
    "test_label_list = label_list[dataset_len*6//10:]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification using KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Self Trained word vector of 100 length\n",
    "'''\n",
    "from gensim.models import Word2Vec\n",
    "model = Word2Vec.load('model100')\n",
    "vocab = model.wv.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:10: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  # Remove the CWD from sys.path while we load stuff.\n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:19: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:24: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.82407888 0.51076923 0.44287871]\n",
      "0.7339421142238044\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['none', 'none', 'none', ..., 'sexism', 'sexism', 'none'],\n",
       "      dtype='<U6')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import f1_score\n",
    "import numpy as np\n",
    "X = []\n",
    "\n",
    "for each in train_tweet_list:\n",
    "    t = np.zeros(100)\n",
    "    for i in each[1:]:\n",
    "        t = t + model[i]\n",
    "    t = t / len(each)\n",
    "    X.append(t)\n",
    "    \n",
    "test_vector = []\n",
    "for each in test_tweet_list:\n",
    "    t = np.zeros(100)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            t = t + model[i]                    \n",
    "    t = t / len(each)\n",
    "    test_vector.append(t)\n",
    "    \n",
    "clf = KNeighborsClassifier(algorithm='auto')\n",
    "clf.fit(X,train_label_list)\n",
    "pred = clf.predict(test_vector)\n",
    "acc = accuracy_score(test_label_list,pred)\n",
    "f1 = f1_score(test_label_list,pred,average=None)\n",
    "print(f1)\n",
    "print(acc)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "vectorizer = TfidfVectorizer(['tweet_dataset.csv'],stop_words=None,analyzer='word')\n",
    "\n",
    "vectorizer1 = TfidfVectorizer(['tweet_dataset.csv'],stop_words=None,analyzer='char')\n",
    "vec = vectorizer.fit_transform(data.iloc[:,2])\n",
    "vec1 = vectorizer1.fit_transform(data.iloc[:,2])\n",
    "v = dict(zip(vectorizer.get_feature_names(),vectorizer.idf_))\n",
    "v1 = dict(zip(vectorizer1.get_feature_names(),vectorizer1.idf_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:8: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  \n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:6: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  \n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:22: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:20: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:26: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.83011862 0.54504164 0.43546695]\n",
      "0.7427642779755456\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['none', 'none', 'none', ..., 'sexism', 'sexism', 'none'],\n",
       "      dtype='<U6')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_tf = []\n",
    "for each in train_tweet_list:\n",
    "    t = np.zeros(100)\n",
    "    for i in each[1:]:\n",
    "        if len(i)==1:\n",
    "            t = t + model[i] * v1[i.lower()]\n",
    "        else:\n",
    "            t = t + model[i] * v[i.lower()]\n",
    "    t = t / len(each)\n",
    "    X_tf.append(t)\n",
    "'''\n",
    "converting test tweet dataset to word vectors\n",
    "'''\n",
    "test_vector_tf = []\n",
    "for each in test_tweet_list:\n",
    "    t = np.zeros(100)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            if len(i)==1:\n",
    "                t = t + model[i] * v1[i.lower()]\n",
    "            else:\n",
    "                t = t + model[i] * v[i.lower()]      \n",
    "    t = t / len(each)\n",
    "    test_vector_tf.append(t)\n",
    "clf = KNeighborsClassifier(algorithm='auto')\n",
    "clf.fit(X_tf,train_label_list)\n",
    "pred = clf.predict(test_vector_tf)\n",
    "acc = accuracy_score(test_label_list,pred)\n",
    "f1 = f1_score(test_label_list,pred,average=None)\n",
    "print(f1)\n",
    "print(acc)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Self trained word vector of 200 length\n",
    "'''\n",
    "\n",
    "model = Word2Vec.load('model200')\n",
    "vocab = model.wv.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:6: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  \n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:15: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  from ipykernel import kernelapp as app\n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:20: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.82494552 0.52647736 0.43794147]\n",
      "0.7354898622504257\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['none', 'none', 'none', ..., 'sexism', 'sexism', 'none'],\n",
       "      dtype='<U6')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = []\n",
    "\n",
    "for each in train_tweet_list:\n",
    "    t = np.zeros(200)\n",
    "    for i in each[1:]:\n",
    "        t = t + model[i]\n",
    "    t = t / len(each)\n",
    "    X.append(t)\n",
    "    \n",
    "test_vector = []\n",
    "for each in test_tweet_list:\n",
    "    t = np.zeros(200)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            t = t + model[i]                    \n",
    "    t = t / len(each)\n",
    "    test_vector.append(t)\n",
    "    \n",
    "clf = KNeighborsClassifier(algorithm='auto')\n",
    "clf.fit(X,train_label_list)\n",
    "pred = clf.predict(test_vector)\n",
    "acc = accuracy_score(test_label_list,pred)\n",
    "f1 = f1_score(test_label_list,pred,average=None)\n",
    "print(f1)\n",
    "print(acc)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:8: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  \n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:6: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  \n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:22: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:20: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:26: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.82801236 0.52255054 0.43302181]\n",
      "0.7387401331063302\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['none', 'none', 'none', ..., 'sexism', 'sexism', 'none'],\n",
       "      dtype='<U6')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_tf = []\n",
    "for each in train_tweet_list:\n",
    "    t = np.zeros(200)\n",
    "    for i in each[1:]:\n",
    "        if len(i)==1:\n",
    "            t = t + model[i] * v1[i.lower()]\n",
    "        else:\n",
    "            t = t + model[i] * v[i.lower()]\n",
    "    t = t / len(each)\n",
    "    X_tf.append(t)\n",
    "'''\n",
    "converting test tweet dataset to word vectors\n",
    "'''\n",
    "test_vector_tf = []\n",
    "for each in test_tweet_list:\n",
    "    t = np.zeros(200)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            if len(i)==1:\n",
    "                t = t + model[i] * v1[i.lower()]\n",
    "            else:\n",
    "                t = t + model[i] * v[i.lower()]      \n",
    "    t = t / len(each)\n",
    "    test_vector_tf.append(t)\n",
    "clf = KNeighborsClassifier(algorithm='auto')\n",
    "clf.fit(X_tf,train_label_list)\n",
    "pred = clf.predict(test_vector_tf)\n",
    "acc = accuracy_score(test_label_list,pred)\n",
    "f1 = f1_score(test_label_list,pred,average=None)\n",
    "print(f1)\n",
    "print(acc)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Self trained word vector of 300 length\n",
    "'''\n",
    "\n",
    "model = Word2Vec.load('model300')\n",
    "vocab = model.wv.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:6: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  \n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:15: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  from ipykernel import kernelapp as app\n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:20: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.82194895 0.51086957 0.42309658]\n",
      "0.7305370685652376\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['none', 'none', 'none', ..., 'sexism', 'sexism', 'none'],\n",
       "      dtype='<U6')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = []\n",
    "\n",
    "for each in train_tweet_list:\n",
    "    t = np.zeros(300)\n",
    "    for i in each[1:]:\n",
    "        t = t + model[i]\n",
    "    t = t / len(each)\n",
    "    X.append(t)\n",
    "    \n",
    "test_vector = []\n",
    "for each in test_tweet_list:\n",
    "    t = np.zeros(300)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            t = t + model[i]                    \n",
    "    t = t / len(each)\n",
    "    test_vector.append(t)\n",
    "    \n",
    "clf = KNeighborsClassifier(algorithm='auto')\n",
    "clf.fit(X,train_label_list)\n",
    "pred = clf.predict(test_vector)\n",
    "acc = accuracy_score(test_label_list,pred)\n",
    "f1 = f1_score(test_label_list,pred,average=None)\n",
    "print(f1)\n",
    "print(acc)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:8: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  \n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:6: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  \n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:22: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:20: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "C:\\Users\\HP\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:26: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.83141447 0.52655889 0.44010554]\n",
      "0.7433833771861941\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['none', 'none', 'none', ..., 'sexism', 'sexism', 'none'],\n",
       "      dtype='<U6')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_tf = []\n",
    "for each in train_tweet_list:\n",
    "    t = np.zeros(300)\n",
    "    for i in each[1:]:\n",
    "        if len(i)==1:\n",
    "            t = t + model[i] * v1[i.lower()]\n",
    "        else:\n",
    "            t = t + model[i] * v[i.lower()]\n",
    "    t = t / len(each)\n",
    "    X_tf.append(t)\n",
    "'''\n",
    "converting test tweet dataset to word vectors\n",
    "'''\n",
    "test_vector_tf = []\n",
    "for each in test_tweet_list:\n",
    "    t = np.zeros(300)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            if len(i)==1:\n",
    "                t = t + model[i] * v1[i.lower()]\n",
    "            else:\n",
    "                t = t + model[i] * v[i.lower()]      \n",
    "    t = t / len(each)\n",
    "    test_vector_tf.append(t)\n",
    "clf = KNeighborsClassifier(algorithm='auto')\n",
    "clf.fit(X_tf,train_label_list)\n",
    "pred = clf.predict(test_vector_tf)\n",
    "acc = accuracy_score(test_label_list,pred)\n",
    "f1 = f1_score(test_label_list,pred,average=None)\n",
    "print(f1)\n",
    "print(acc)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-12-dd76ff2c0c6f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mgensim\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodels\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mKeyedVectors\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mKeyedVectors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload_word2vec_format\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"GoogleNews-vectors-negative300.bin\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mbinary\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mvocab\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\gensim\\models\\keyedvectors.py\u001b[0m in \u001b[0;36mload_word2vec_format\u001b[1;34m(cls, fname, fvocab, binary, encoding, unicode_errors, limit, datatype)\u001b[0m\n\u001b[0;32m   1474\u001b[0m         return _load_word2vec_format(\n\u001b[0;32m   1475\u001b[0m             \u001b[0mcls\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfvocab\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfvocab\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbinary\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbinary\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mencoding\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0municode_errors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0municode_errors\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1476\u001b[1;33m             limit=limit, datatype=datatype)\n\u001b[0m\u001b[0;32m   1477\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1478\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget_keras_embedding\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_embeddings\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\gensim\\models\\utils_any2vec.py\u001b[0m in \u001b[0;36m_load_word2vec_format\u001b[1;34m(cls, fname, fvocab, binary, encoding, unicode_errors, limit, datatype)\u001b[0m\n\u001b[0;32m    347\u001b[0m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvector_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    348\u001b[0m         \u001b[0mresult\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvector_size\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvector_size\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 349\u001b[1;33m         \u001b[0mresult\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvectors\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mzeros\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvocab_size\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvector_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdatatype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    350\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    351\u001b[0m         \u001b[1;32mdef\u001b[0m \u001b[0madd_word\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mword\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from gensim.models import KeyedVectors\n",
    "model = KeyedVectors.load_word2vec_format(\"GoogleNews-vectors-negative300.bin\",binary=True)\n",
    "vocab = model.wv.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "\n",
    "for each in train_tweet_list:\n",
    "    t = np.zeros(300)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            t = t + model[i]\n",
    "    t = t / len(each)\n",
    "    X.append(t)\n",
    "    \n",
    "test_vector = []\n",
    "for each in test_tweet_list:\n",
    "    t = np.zeros(300)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            t = t + model[i]                    \n",
    "    t = t / len(each)\n",
    "    test_vector.append(t)\n",
    "    \n",
    "clf = KNeighborsClassifier(algorithm='auto')\n",
    "clf.fit(X,train_label_list)\n",
    "pred = clf.predict(test_vector)\n",
    "acc = accuracy_score(test_label_list,pred)\n",
    "f1 = f1_score(test_label_list,pred,average=None)\n",
    "print(f1)\n",
    "print(acc)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_tf = []\n",
    "for each in train_tweet_list:\n",
    "    t = np.zeros(300)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            if len(i)==1:\n",
    "                t = t + model[i] * v1[i.lower()]\n",
    "            else:\n",
    "                t = t + model[i] * v[i.lower()]\n",
    "    t = t / len(each)\n",
    "    X_tf.append(t)\n",
    "'''\n",
    "converting test tweet dataset to word vectors\n",
    "'''\n",
    "test_vector_tf = []\n",
    "for each in test_tweet_list:\n",
    "    t = np.zeros(300)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            if len(i)==1:\n",
    "                t = t + model[i] * v1[i.lower()]\n",
    "            else:\n",
    "                t = t + model[i] * v[i.lower()]      \n",
    "    t = t / len(each)\n",
    "    test_vector_tf.append(t)\n",
    "clf = KNeighborsClassifier(algorithm='auto')\n",
    "clf.fit(X_tf,train_label_list)\n",
    "pred = clf.predict(test_vector_tf)\n",
    "acc = accuracy_score(test_label_list,pred)\n",
    "f1 = f1_score(test_label_list,pred,average=None)\n",
    "print(f1)\n",
    "print(acc)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Model with word vector dimension 300\n",
    "'''\n",
    "model = KeyedVectors.load_word2vec_format(\"glove.300d.txt\",binary=False)\n",
    "vocab = model.wv.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "\n",
    "for each in train_tweet_list:\n",
    "    t = np.zeros(300)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            t = t + model[i]\n",
    "    t = t / len(each)\n",
    "    X.append(t)\n",
    "    \n",
    "test_vector = []\n",
    "for each in test_tweet_list:\n",
    "    t = np.zeros(300)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            t = t + model[i]                    \n",
    "    t = t / len(each)\n",
    "    test_vector.append(t)\n",
    "    \n",
    "clf = KNeighborsClassifier(algorithm='auto')\n",
    "clf.fit(X,train_label_list)\n",
    "pred = clf.predict(test_vector)\n",
    "acc = accuracy_score(test_label_list,pred)\n",
    "f1 = f1_score(test_label_list,pred,average=None)\n",
    "print(f1)\n",
    "print(acc)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_tf = []\n",
    "for each in train_tweet_list:\n",
    "    t = np.zeros(300)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            if len(i)==1:\n",
    "                t = t + model[i] * v1[i.lower()]\n",
    "            else:\n",
    "                t = t + model[i] * v[i.lower()]\n",
    "    t = t / len(each)\n",
    "    X_tf.append(t)\n",
    "'''\n",
    "converting test tweet dataset to word vectors\n",
    "'''\n",
    "test_vector_tf = []\n",
    "for each in test_tweet_list:\n",
    "    t = np.zeros(300)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            if len(i)==1:\n",
    "                t = t + model[i] * v1[i.lower()]\n",
    "            else:\n",
    "                t = t + model[i] * v[i.lower()]      \n",
    "    t = t / len(each)\n",
    "    test_vector_tf.append(t)\n",
    "clf = KNeighborsClassifier(algorithm='auto')\n",
    "clf.fit(X_tf,train_label_list)\n",
    "pred = clf.predict(test_vector_tf)\n",
    "acc = accuracy_score(test_label_list,pred)\n",
    "print(acc)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Model with word vector dimension 200\n",
    "'''\n",
    "model = KeyedVectors.load_word2vec_format(\"glove.200d.txt\",binary=False)\n",
    "vocab = model.wv.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "\n",
    "for each in train_tweet_list:\n",
    "    t = np.zeros(200)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            t = t + model[i]\n",
    "    t = t / len(each)\n",
    "    X.append(t)\n",
    "    \n",
    "test_vector = []\n",
    "for each in test_tweet_list:\n",
    "    t = np.zeros(200)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            t = t + model[i]                    \n",
    "    t = t / len(each)\n",
    "    test_vector.append(t)\n",
    "    \n",
    "clf = KNeighborsClassifier(algorithm='auto')\n",
    "clf.fit(X,train_label_list)\n",
    "pred = clf.predict(test_vector)\n",
    "acc = accuracy_score(test_label_list,pred)\n",
    "f1 = f1_score(test_label_list,pred,average=None)\n",
    "print(f1)\n",
    "print(acc)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_tf = []\n",
    "for each in train_tweet_list:\n",
    "    t = np.zeros(200)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            if len(i)==1:\n",
    "                t = t + model[i] * v1[i.lower()]\n",
    "            else:\n",
    "                t = t + model[i] * v[i.lower()]\n",
    "    t = t / len(each)\n",
    "    X_tf.append(t)\n",
    "'''\n",
    "converting test tweet dataset to word vectors\n",
    "'''\n",
    "test_vector_tf = []\n",
    "for each in test_tweet_list:\n",
    "    t = np.zeros(200)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            if len(i)==1:\n",
    "                t = t + model[i] * v1[i.lower()]\n",
    "            else:\n",
    "                t = t + model[i] * v[i.lower()]      \n",
    "    t = t / len(each)\n",
    "    test_vector_tf.append(t)\n",
    "clf = KNeighborsClassifier(algorithm='auto')\n",
    "clf.fit(X_tf,train_label_list)\n",
    "pred = clf.predict(test_vector_tf)\n",
    "acc = accuracy_score(test_label_list,pred)\n",
    "f1 = f1_score(test_label_list,pred,average=None)\n",
    "print(f1)\n",
    "print(acc)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Model with word vector dimension 100\n",
    "'''\n",
    "model = KeyedVectors.load_word2vec_format(\"glove.100d.txt\",binary=False)\n",
    "vocab = model.wv.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "\n",
    "for each in train_tweet_list:\n",
    "    t = np.zeros(100)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            t = t + model[i]\n",
    "    t = t / len(each)\n",
    "    X.append(t)\n",
    "    \n",
    "test_vector = []\n",
    "for each in test_tweet_list:\n",
    "    t = np.zeros(100)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            t = t + model[i]                    \n",
    "    t = t / len(each)\n",
    "    test_vector.append(t)\n",
    "    \n",
    "clf = KNeighborsClassifier(algorithm='auto')\n",
    "clf.fit(X,train_label_list)\n",
    "pred = clf.predict(test_vector)\n",
    "acc = accuracy_score(test_label_list,pred)\n",
    "f1 = f1_score(test_label_list,pred,average=None)\n",
    "print(f1)\n",
    "print(acc)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_tf = []\n",
    "for each in train_tweet_list:\n",
    "    t = np.zeros(100)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            if len(i)==1:\n",
    "                t = t + model[i] * v1[i.lower()]\n",
    "            else:\n",
    "                t = t + model[i] * v[i.lower()]\n",
    "    t = t / len(each)\n",
    "    X_tf.append(t)\n",
    "'''\n",
    "converting test tweet dataset to word vectors\n",
    "'''\n",
    "test_vector_tf = []\n",
    "for each in test_tweet_list:\n",
    "    t = np.zeros(100)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            if len(i)==1:\n",
    "                t = t + model[i] * v1[i.lower()]\n",
    "            else:\n",
    "                t = t + model[i] * v[i.lower()]      \n",
    "    t = t / len(each)\n",
    "    test_vector_tf.append(t)\n",
    "clf = KNeighborsClassifier(algorithm='auto')\n",
    "clf.fit(X_tf,train_label_list)\n",
    "pred = clf.predict(test_vector_tf)\n",
    "acc = accuracy_score(test_label_list,pred)\n",
    "f1 = f1_score(test_label_list,pred,average=None)\n",
    "print(f1)\n",
    "print(acc)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Model with word vector dimension 50\n",
    "'''\n",
    "model = KeyedVectors.load_word2vec_format(\"glove.50d.txt\",binary=False)\n",
    "vocab = model.wv.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "\n",
    "for each in train_tweet_list:\n",
    "    t = np.zeros(50)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            t = t + model[i]\n",
    "    t = t / len(each)\n",
    "    X.append(t)\n",
    "    \n",
    "test_vector = []\n",
    "for each in test_tweet_list:\n",
    "    t = np.zeros(50)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            t = t + model[i]                    \n",
    "    t = t / len(each)\n",
    "    test_vector.append(t)\n",
    "    \n",
    "clf = KNeighborsClassifier(algorithm='auto')\n",
    "clf.fit(X,train_label_list)\n",
    "pred = clf.predict(test_vector)\n",
    "acc = accuracy_score(test_label_list,pred)\n",
    "f1 = f1_score(test_label_list,pred,average=None)\n",
    "print(f1)\n",
    "print(acc)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_tf = []\n",
    "for each in train_tweet_list:\n",
    "    t = np.zeros(50)\n",
    "    for i in each[1:]:\n",
    "        if i in vocab:\n",
    "            if len(i)==1:\n",
    "                t = t + model[i] * v1[i.lower()]\n",
    "            else:\n",
    "                t = t + model[i] * v[i.lower()]\n",
    "    t = t / len(each)\n",
    "    X_tf.append(t)\n",
    "'''\n",
    "converting test tweet dataset to word vectors\n",
    "'''\n",
    "test_vector_tf = []\n",
    "for each in test_tweet_list:\n",
    "    t = np.zeros(50)\n",
    "    for i in each[1:]:\n",
    "        if i in model.wv.vocab:\n",
    "            if len(i)==1:\n",
    "                t = t + model[i] * v1[i.lower()]\n",
    "            else:\n",
    "                t = t + model[i] * v[i.lower()]      \n",
    "    t = t / len(each)\n",
    "    test_vector_tf.append(t)\n",
    "clf = KNeighborsClassifier(algorithm='auto')\n",
    "clf.fit(X_tf,train_label_list)\n",
    "pred = clf.predict(test_vector_tf)\n",
    "acc = accuracy_score(test_label_list,pred)\n",
    "f1 = f1_score(test_label_list,pred,average=None)\n",
    "print(f1)\n",
    "print(acc)\n",
    "pred"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}