Project2_551.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## IMDB Movie Review Sentiment Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re, string, unicodedata\n",
    "from bs4 import BeautifulSoup\n",
    "from nltk.stem import PorterStemmer\n",
    "\n",
    "def remove_html(text):\n",
    "    soup = BeautifulSoup(text, \"html.parser\")\n",
    "    return soup.get_text()\n",
    "\n",
    "def remove_links_characters(text):\n",
    "    text = re.sub(r'^https?:\\/\\/.*[\\r\\n]*', '', text, flags=re.MULTILINE)\n",
    "    text = re.sub('\\[[^]]*\\]', '', text)\n",
    "    return text\n",
    "\n",
    "def regular_preprocess(text):\n",
    "    text = remove_html(text)\n",
    "    text = remove_links_characters(text)\n",
    "    return text\n",
    "\n",
    "def remove_stopwords(docs, stopwords):\n",
    "    docs_ref = []\n",
    "    for doc in docs:\n",
    "        word_list = doc.lower().split()\n",
    "        word_list_ref = [word for word in word_list if word not in stopwords]\n",
    "        word_str_ref = ' '.join(word_list_ref)\n",
    "        docs_ref.append(word_str_ref)\n",
    "    return docs_ref\n",
    "\n",
    "def stem_words(docs):\n",
    "    stemmer = PorterStemmer()\n",
    "    stems = []\n",
    "    for doc in docs:\n",
    "        word_list = doc.lower().split()\n",
    "        for word in word_list:\n",
    "            stem = stemmer.stem(word)\n",
    "            stems.append(stem)\n",
    "        stems_str = ' '.join(stems)\n",
    "        stems.append(stems_str)\n",
    "    return stems_str\n",
    "\n",
    "def preprocess(data):\n",
    "    refined_data = []\n",
    "    for dp in data:\n",
    "        refined_data.append(regular_preprocess(dp))        \n",
    "    return refined_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import os\n",
    "import csv\n",
    "import numpy as np\n",
    "from preprocessing import *\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.preprocessing import Normalizer\n",
    "from sklearn import metrics\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.svm import SVC\n",
    "from scipy.stats import uniform\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# copy contents of all files in both folders into a list\n",
    "train_data = []\n",
    "test_data = []\n",
    "\n",
    "# train data\n",
    "train_neg = glob.glob(os.path.join(os.getcwd(), \"Dataset/train/neg\", \"*.txt\"))\n",
    "for f_path in train_neg:\n",
    "    with open(f_path) as f:\n",
    "        train_data.append(f.read())\n",
    "\n",
    "train_pos = glob.glob(os.path.join(os.getcwd(), \"Dataset/train/pos\", \"*.txt\"))\n",
    "for f_path in train_pos:\n",
    "    with open(f_path) as f:\n",
    "        train_data.append(f.read())\n",
    "\n",
    "def sort_nicely(l):\n",
    "# Sort the given list in the way that humans expect.\n",
    "    convert = lambda text: int(text) if text.isdigit() else text\n",
    "    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]\n",
    "    l.sort(key=alphanum_key)\n",
    "\n",
    "test_files = glob.glob(os.path.join(os.getcwd(), \"Dataset/test\", \"*.txt\"))\n",
    "sort_nicely(test_files)\n",
    "test_files_ids = [int(re.sub(\"[^0-9]\",\"\", item)) for item in test_files]\n",
    "\n",
    "for f_path in test_files:\n",
    "    with open(f_path) as f:\n",
    "        test_data.append(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# targets: first 12500 are pos, next 12500 are neg\n",
    "targets = [0 if i<12500 else 1 for i in range(25000)]\n",
    "# print(train_data[0])\n",
    "train_data_clean = preprocess(train_data)\n",
    "test_data_clean = preprocess(test_data)\n",
    "# print(train_data_clean[0])\n",
    "\n",
    "# Using hold-out validation\n",
    "X_train, X_validation, y_train, y_validation = train_test_split(train_data_clean, targets, train_size=0.8, test_size=0.2, random_state=1)\n",
    "\n",
    "# Using k-fold (in the Grid Search)\n",
    "# X_train = train_data_clean\n",
    "# y_train = targets\n",
    "\n",
    "def display_results(y_val, y_pred):\n",
    "    print(metrics.classification_report(y_val, y_pred))\n",
    "    print(\"Accuracy % = \", metrics.accuracy_score(y_val, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finding best parameters using Grid Search (the general flow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SOMEpclf = Pipeline(['vect', CountVectorizer(...), ...])\n",
    "\n",
    "\n",
    "# params = {\n",
    "#     'vect__max_df': (0.5, 0.75, 1.0),\n",
    "#     'vect__max_features': (None, 5000, 10000, 50000),\n",
    "#     'vect__ngram_range': ((1, 1), (1, 2),(2,2)),  # unigrams or bigrams\n",
    "#     'tfidf__use_idf': (True, False),\n",
    "#     'tfidf__norm': ('l1', 'l2'),\n",
    "#     'clf__max_iter': (5,),\n",
    "#     'clf__alpha': (0.00001, 0.000001),\n",
    "#     'clf__penalty': ('l2', 'elasticnet'),\n",
    "#     'clf__max_iter': (10, 50, 80),\n",
    "# }\n",
    "\n",
    "\n",
    "# find the best parameters for both the feature extraction and the classifier\n",
    "\n",
    "# grid_search = GridSearchCV(MODEL_NAMEpclf, params, cv=5,\n",
    "#                            n_jobs=-1, verbose=10)\n",
    "\n",
    "# print(\"Performing grid search...\")\n",
    "# print(\"pipeline:\", [name for name, _ in MODEL_NAMEpclf.steps])\n",
    "# print(\"parameters:\")\n",
    "# print(params)\n",
    "# grid_search.fit(X_train, y_train)\n",
    "# print()\n",
    "\n",
    "# print(\"Best score: %0.3f\" % grid_search.best_score_)\n",
    "# print(\"Best parameters set:\")\n",
    "# best_parameters = grid_search.best_estimator_.get_params()\n",
    "# for param_name in sorted(params.keys()):\n",
    "#     print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25000\n"
     ]
    }
   ],
   "source": [
    "# params chosen by the grid search algorithm\n",
    "SGDpclf = Pipeline([\n",
    "    ('vect', CountVectorizer(analyzer='word', max_df=1.0, min_df=0,ngram_range=(1, 2))),\n",
    "    ('tfidf', TfidfTransformer(norm='l2')),\n",
    "    ('clf', SGDClassifier(alpha=1e-5, epsilon=0.1, random_state=1)),\n",
    "])\n",
    "\n",
    "SGDpclf.fit(X_train, y_train)\n",
    "y_pred_sgd_val = SGDpclf.predict(X_validation)\n",
    "\n",
    "y_pred_sgd_test = SGDpclf.predict(test_data_clean)\n",
    "print(len(y_pred_sgd_test))\n",
    "\n",
    "with open('submissionSGD.csv', 'w') as f:\n",
    "    writer = csv.writer(f)\n",
    "    writer.writerow((\"Id\", \"Category\"))\n",
    "    writer.writerows(zip(test_files_ids, y_pred_sgd_test))\n",
    "\n",
    "display_results(y_validation, y_pred_sgd_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.90      0.91      2517\n",
      "           1       0.90      0.92      0.91      2483\n",
      "\n",
      "   micro avg       0.91      0.91      0.91      5000\n",
      "   macro avg       0.91      0.91      0.91      5000\n",
      "weighted avg       0.91      0.91      0.91      5000\n",
      "\n",
      "Accuracy % =  0.9108\n"
     ]
    }
   ],
   "source": [
    "SVMpclf = Pipeline([\n",
    "    ('vect', CountVectorizer(max_df=0.5, ngram_range=(1,2))),\n",
    "    ('tfidf', TfidfTransformer(norm='l2')),\n",
    "    ('clf', LinearSVC(C=0.5)),\n",
    "])\n",
    "\n",
    "SVMpclf.fit(X_train, y_train)\n",
    "y_pred_svm_val = SVMpclf.predict(X_validation)\n",
    "\n",
    "y_pred_svm_test = SVMpclf.predict(test_data_clean)\n",
    "with open('submissionLSVM.csv', 'w') as f:\n",
    "    writer = csv.writer(f)\n",
    "    writer.writerow((\"Id\", \"Category\"))\n",
    "    writer.writerows(zip(test_files_ids, y_pred_svm_test))\n",
    "\n",
    "display_results(y_validation, y_pred_svm_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.89      0.90      2517\n",
      "           1       0.89      0.91      0.90      2483\n",
      "\n",
      "   micro avg       0.90      0.90      0.90      5000\n",
      "   macro avg       0.90      0.90      0.90      5000\n",
      "weighted avg       0.90      0.90      0.90      5000\n",
      "\n",
      "Accuracy % =  0.8966\n"
     ]
    }
   ],
   "source": [
    "LRpclf = Pipeline([\n",
    "    ('vect', CountVectorizer(binary=True, max_df=0.5, ngram_range=(1,2))),\n",
    "    ('tfidf', TfidfTransformer(norm='l2')),\n",
    "    ('clf', LogisticRegression()),\n",
    "])\n",
    "\n",
    "LRpclf.fit(X_train, y_train)\n",
    "y_pred_lr_val = LRpclf.predict(X_validation)\n",
    "\n",
    "y_pred_lr_test = LRpclf.predict(test_data_clean)\n",
    "with open('submissionLR.csv', 'w') as f:\n",
    "    writer = csv.writer(f)\n",
    "    writer.writerow((\"Id\", \"Category\"))\n",
    "    writer.writerows(zip(test_files_ids, y_pred_lr_test))\n",
    "\n",
    "display_results(y_validation, y_pred_lr_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using Major Vote"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "preds_val = pd.DataFrame()\n",
    "preds_val['pred_val_1'] = y_pred_sgd_val\n",
    "preds_val['pred_val_2'] = y_pred_svm_val\n",
    "preds_val['pred_val_3'] = y_pred_lr_val\n",
    "\n",
    "preds_val['maj_vote_val'] = preds_val.mode(axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# preds_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.90      0.91      2517\n",
      "           1       0.90      0.92      0.91      2483\n",
      "\n",
      "   micro avg       0.91      0.91      0.91      5000\n",
      "   macro avg       0.91      0.91      0.91      5000\n",
      "weighted avg       0.91      0.91      0.91      5000\n",
      "\n",
      "Accuracy % =  0.9096\n"
     ]
    }
   ],
   "source": [
    "display_results(y_validation, preds_val['maj_vote_val'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = pd.DataFrame()\n",
    "preds['pred1'] = y_pred_sgd_test\n",
    "preds['pred2'] = y_pred_svm_test\n",
    "preds['pred3'] = y_pred_lr_test\n",
    "\n",
    "preds['maj_vote'] = preds.mode(axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pred1</th>\n",
       "      <th>pred2</th>\n",
       "      <th>pred3</th>\n",
       "      <th>maj_vote</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24970</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24971</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24972</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24973</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24974</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24975</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24976</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24977</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24978</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24979</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24980</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24981</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24982</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24983</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24984</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24985</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24986</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24987</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24988</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24989</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24990</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24991</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24992</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24993</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24994</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24995</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24996</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24997</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24998</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24999</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       pred1  pred2  pred3  maj_vote\n",
       "0          0      0      0         0\n",
       "1          0      0      0         0\n",
       "2          0      0      0         0\n",
       "3          0      0      0         0\n",
       "4          1      1      1         1\n",
       "5          0      0      0         0\n",
       "6          0      0      0         0\n",
       "7          0      0      0         0\n",
       "8          0      0      0         0\n",
       "9          0      0      0         0\n",
       "10         0      0      0         0\n",
       "11         0      0      0         0\n",
       "12         1      1      1         1\n",
       "13         0      0      0         0\n",
       "14         1      1      1         1\n",
       "15         0      0      0         0\n",
       "16         0      0      0         0\n",
       "17         1      1      1         1\n",
       "18         0      0      0         0\n",
       "19         0      0      0         0\n",
       "20         0      0      0         0\n",
       "21         0      0      0         0\n",
       "22         0      0      0         0\n",
       "23         1      1      1         1\n",
       "24         1      1      1         1\n",
       "25         1      1      1         1\n",
       "26         0      0      0         0\n",
       "27         0      0      0         0\n",
       "28         0      0      0         0\n",
       "29         1      1      1         1\n",
       "...      ...    ...    ...       ...\n",
       "24970      0      1      1         1\n",
       "24971      0      0      0         0\n",
       "24972      1      1      1         1\n",
       "24973      0      0      0         0\n",
       "24974      0      0      1         0\n",
       "24975      1      1      1         1\n",
       "24976      0      0      1         0\n",
       "24977      1      1      1         1\n",
       "24978      0      0      0         0\n",
       "24979      1      1      1         1\n",
       "24980      1      1      1         1\n",
       "24981      0      0      0         0\n",
       "24982      1      0      1         1\n",
       "24983      1      1      1         1\n",
       "24984      1      1      1         1\n",
       "24985      0      0      0         0\n",
       "24986      1      1      1         1\n",
       "24987      0      0      0         0\n",
       "24988      0      0      0         0\n",
       "24989      0      0      0         0\n",
       "24990      1      1      1         1\n",
       "24991      1      1      1         1\n",
       "24992      1      1      1         1\n",
       "24993      0      0      0         0\n",
       "24994      1      1      1         1\n",
       "24995      0      1      1         1\n",
       "24996      1      0      0         0\n",
       "24997      1      1      1         1\n",
       "24998      1      1      1         1\n",
       "24999      1      1      1         1\n",
       "\n",
       "[25000 rows x 4 columns]"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('submission.csv', 'w') as f:\n",
    "    writer = csv.writer(f)\n",
    "    writer.writerow((\"Id\", \"Category\"))\n",
    "    writer.writerows(zip(test_files_ids, preds['maj_vote']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cross Validation of SVM and LR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Naive Bayes Implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}