tamil_text_classifier_multinomialNB.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import glob\n",
    "from urllib.request import urlopen\n",
    "import wikipediaapi\n",
    "import random \n",
    "from collections import Counter \n",
    "import re\n",
    "from sklearn.model_selection import train_test_split\n",
    "from nltk import word_tokenize\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn import metrics\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Pre-processing functions\n",
    "def check_list(list_column, check_str):\n",
    "    if check_str in list_column: \n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "    \n",
    "def clean_text(text):\n",
    "    text = text.replace('\\nBULLET::::', ' ')\n",
    "    text = text.replace('BULLET::::-', ' ')\n",
    "    text = text.replace('BULLET::::', ' ')\n",
    "    text = text.replace('\\n', ' ')\n",
    "    text = text.replace('\\n\\n', ' ')\n",
    "    text = text.replace(r',', '')\n",
    "    text = text.replace('.', '')\n",
    "    text = text.replace(' - ', '')\n",
    "    text = text.replace('-', '')\n",
    "    text = text.replace('&nbsp;', ' ')\n",
    "    text = text.replace('Page', ' ')\n",
    "    text = text.replace(':', ' ')\n",
    "    text = text.replace(';', ' ')\n",
    "    text = text.replace('\"', '')\n",
    "    text = text.replace(\"'\", '')\n",
    "    text = text.replace('(', '')\n",
    "    text = text.replace('[', '')\n",
    "    text = text.replace(']', '')\n",
    "    text = text.replace(')', '')\n",
    "    text = text.strip()\n",
    "    text = re.sub(r'\\d+.', ' ', text)\n",
    "    text = re.sub(r' +', ' ', text)\n",
    "    return text  \n",
    "\n",
    "def remove_urls (vTEXT):\n",
    "    vTEXT = re.sub(r'(https|http)?:\\/\\/(\\w|\\.|\\/|\\?|\\=|\\&|\\%)*\\b', '', vTEXT, flags=re.MULTILINE)\n",
    "    return(vTEXT)\n",
    "\n",
    "def str_replace(text, title):\n",
    "    return text.replace(title + ' ' + title, '' + title + ' ', 1)  \n",
    "\n",
    "def get_tamil_stop_words():\n",
    "    swdf1 = pd.read_csv(\"data/TamilNLP_TamilStopWords.txt\",  header=None) \n",
    "    swdf2 = pd.read_csv(\"data/custom_tamil_stop_words.txt\",  header=None) \n",
    "    sw1 = swdf1[0].tolist()\n",
    "    sw2 = swdf2[0].tolist()\n",
    "    tamil_stop_words = list(set(sw1 + sw2))\n",
    "    return tamil_stop_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(110000, 8)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the labeled data\n",
    "data_files = glob.glob('data/tamil_wiki_data_with_parent_category/ready_data*')\n",
    "df_list = []\n",
    "for filename in data_files:\n",
    "    df_list.append(pd.read_csv(filename))\n",
    "    \n",
    "df = pd.concat(df_list, ignore_index=True)    \n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(110000, 8)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop duplicates in case any\n",
    "df = df.drop_duplicates(subset='title', keep=\"first\")\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(93517, 8)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop no category\n",
    "is_NoCat =  df['parent_category'] != \"NoCat\"\n",
    "df1 = df[is_NoCat]\n",
    "df1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(87753, 9)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop rows if it contains category crickter, because it is over represented in the data space\n",
    "cricketer_flag = df1.apply(lambda row: check_list(row['categories'], \"துடுப்பாட்டக்காரர்கள்\"),axis=1)\n",
    "df1 = df1.assign(is_cricketer=cricketer_flag.values) \n",
    "not_cricketer =  df1['is_cricketer'] == False\n",
    "df1 = df1[not_cricketer]\n",
    "df1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(76305, 10)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop rows if it contains category temple, because it is over represented in the data space\n",
    "temple_flag = df1.apply(lambda row: check_list(row['categories'], \"கோயில்கள்\"),axis=1)\n",
    "df1 = df1.assign(is_temple=temple_flag.values) \n",
    "not_temple =  df1['is_temple'] == False\n",
    "df1 = df1[not_temple]\n",
    "df1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70515, 10)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop rows if there was difficulty in determining the parent category\n",
    "is_depth_high =  df1['parent_category_recursion_depth'] < 12\n",
    "df2 = df1[is_depth_high]\n",
    "df2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(68392, 10)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter for articles with at least 500 category members\n",
    "df3 = df2[df2['parent_category'].map(df2['parent_category'].value_counts()) > 500]\n",
    "df3.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "புவியியல்      14692\n",
       "வரலாறு          8725\n",
       "சமூகம்          7296\n",
       "அரசியல்         3925\n",
       "உயிரியல்        3231\n",
       "சமயம்           2844\n",
       "பண்பாடு         2749\n",
       "இலக்கியம்       2645\n",
       "ஊடகவியல்        2508\n",
       "தொழினுட்பம்     2237\n",
       "வேதியியல்       2219\n",
       "திரைப்படம்      2095\n",
       "கணினியியல்      1913\n",
       "கணிதம்          1663\n",
       "மானிடவியல்      1400\n",
       "இசை             1287\n",
       "இயற்பியல்       1198\n",
       "மொழி            1121\n",
       "வானியல்          867\n",
       "உளவியல்          706\n",
       "கல்வி            686\n",
       "வணிகவியல்        647\n",
       "சட்டம்           638\n",
       "நலம்             578\n",
       "கட்டிடக்கலை      522\n",
       "Name: parent_category, dtype: int64"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show the categories\n",
    "df3.parent_category.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "இசை            650\n",
       "அரசியல்        650\n",
       "கணினியியல்     650\n",
       "சமயம்          650\n",
       "திரைப்படம்     650\n",
       "ஊடகவியல்       650\n",
       "கட்டிடக்கலை    650\n",
       "உளவியல்        650\n",
       "சட்டம்         650\n",
       "மானிடவியல்     650\n",
       "சமூகம்         650\n",
       "இலக்கியம்      650\n",
       "புவியியல்      650\n",
       "வேதியியல்      650\n",
       "வரலாறு         650\n",
       "நலம்           650\n",
       "தொழினுட்பம்    650\n",
       "பண்பாடு        650\n",
       "உயிரியல்       650\n",
       "வானியல்        650\n",
       "இயற்பியல்      650\n",
       "கணிதம்         650\n",
       "மொழி           650\n",
       "வணிகவியல்      650\n",
       "கல்வி          650\n",
       "Name: parent_category, dtype: int64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Equvalize the sample, over sample by small fraction\n",
    "grouped = df3.groupby('parent_category')\n",
    "df4 = grouped.apply(lambda x: x.sample(n=650, replace=True))\n",
    "df4.parent_category.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ஒற்றை வெளியேற்றப் போட்டி ஓர் ஒற்றைவெளியேற்றப் போட்டியில் singleelimination tournament knockout cup அல்லது sudden death tournament ஒவ்வொரு ஆட்டத்தின் தோற்ற போட்டியாளரும் போட்டியிலிருந்து உடனடியாக விலக்கப்படுகின்ற வகையில் அமைக்கப்படும் போட்டியாகும் இது தோற்ற போட்டியாளர் இனி போட்டியின் எந்தவொரு ஆட்டத்திலும் பங்கு கொள்ள மாட்டார் என்றில்லை சில போட்டிகளில் ஆறுதல் பரிசுக்காகவும் தோற்றவரிடையே முதலிரு நிலைகளை அடுத்த வரிசைகளைத் தீர்மானிக்கவும் தோற்றப் போட்டியாளர்களிடையே ஆட்டங்கள் நடத்தப்படுவதுண்டு இரட்டை வெளியேற்றப் போட்டியில் ஒருவர் இருமுறை தோற்றால் போட்டியிலிருந்து விலக்கப்படுவர்'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Clean the text\n",
    "df4['text'] = df4.apply(lambda row: str_replace(row['text'],row['title']),axis=1)\n",
    "df4['text'] = df4.apply(lambda row: clean_text(row['text']),axis=1)\n",
    "df4['text'] = df4.apply(lambda row: remove_urls(row['text']),axis=1)\n",
    "# Sample text\n",
    "df4.iloc[3333].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Save it to \n",
    "df4.to_csv(\"data/cleaned_tamil_wiki_text_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Train and test split\n",
    "train_set,test_set = train_test_split(df4,test_size=0.30,random_state=50)\n",
    "X_train = train_set.text\n",
    "X_test = test_set.text\n",
    "y_train = train_set.parent_category\n",
    "y_test = test_set.parent_category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get a CountVectorizer\n",
    "tamil_stop_words = get_tamil_stop_words()\n",
    "vect = CountVectorizer(tokenizer=word_tokenize, stop_words=tamil_stop_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Transform the text into vector\n",
    "vect.fit(X_train)\n",
    "X_train_dtm = vect.transform(X_train)\n",
    "X_train_dtm = vect.fit_transform(X_train)\n",
    "X_test_dtm = vect.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.60201401, 0.61254936, 0.62725275, 0.61629956, 0.59858844])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# instantiate a Multinomial Naive Bayes model\n",
    "nb = MultinomialNB()\n",
    "# See how the model preforms over various subset of training and test data\n",
    "scores = cross_val_score(nb, X = X_train_dtm, y = y_train, cv = 5)\n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 292 ms, sys: 96 ms, total: 388 ms\n",
      "Wall time: 385 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train/fit the model\n",
    "%time nb.fit(X_train_dtm, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6367179487179487"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show the accuracy of the model for the test data\n",
    "y_pred_class = nb.predict(X_test_dtm)\n",
    "metrics.accuracy_score(y_test, y_pred_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>title</th>\n",
       "      <th>source</th>\n",
       "      <th>parent_category</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>சுமத்ரான் காண்டாமிருக இனம் முற்றிலும் அழிந்துவ...</td>\n",
       "      <td>https://www.bbc.com/tamil/science-50534560</td>\n",
       "      <td>உயிரியல்</td>\n",
       "      <td>மனிதர்களின் வேட்டைகளினாலும் வாழிட பரப்பு அழிப்...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>பிளாஸ்டிக்கில் இருந்து எண்ணெய் எடுக்கும் முயற்...</td>\n",
       "      <td>https://www.bbc.com/tamil/global-50199538</td>\n",
       "      <td>வேதியியல்</td>\n",
       "      <td>தனது ஆய்வகத்தில் உலைக்கலனை கென்னத் போயிப்பெல்ம...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                              title  \\\n",
       "0         NaN  சுமத்ரான் காண்டாமிருக இனம் முற்றிலும் அழிந்துவ...   \n",
       "1         NaN  பிளாஸ்டிக்கில் இருந்து எண்ணெய் எடுக்கும் முயற்...   \n",
       "\n",
       "                                       source parent_category  \\\n",
       "0  https://www.bbc.com/tamil/science-50534560        உயிரியல்   \n",
       "1   https://www.bbc.com/tamil/global-50199538       வேதியியல்   \n",
       "\n",
       "                                                text  \n",
       "0  மனிதர்களின் வேட்டைகளினாலும் வாழிட பரப்பு அழிப்...  \n",
       "1  தனது ஆய்வகத்தில் உலைக்கலனை கென்னத் போயிப்பெல்ம...  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the external validation dataset\n",
    "validation_df = pd.read_csv(\"data/validation.csv\")\n",
    "validation_df['text'] = validation_df.apply(lambda row: clean_text(row['text']),axis=1)\n",
    "validation_df['text'] = validation_df.apply(lambda row: remove_urls(row['text']),axis=1)\n",
    "validation_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Convert into vector\n",
    "X_validation = validation_df.text\n",
    "y_validation = validation_df.parent_category\n",
    "X_validation_dtm = vect.transform(X_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7142857142857143"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Predict and evaulate the external dataset\n",
    "y_pred_class2 = nb.predict(X_validation_dtm)\n",
    "metrics.accuracy_score(y_validation, y_pred_class2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['உயிரியல்', 'தொழினுட்பம்', 'வானியல்', 'வானியல்', 'நலம்', 'சட்டம்',\n",
       "       'சட்டம்', 'உயிரியல்', 'தொழினுட்பம்', 'இயற்பியல்', 'வரலாறு', 'நலம்',\n",
       "       'பண்பாடு', 'இலக்கியம்'], dtype='<U11')"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print the external dataset\n",
    "y_pred_class2"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}