07H.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Python ≥3.5 is required\n",
    "import sys\n",
    "assert sys.version_info >= (3, 5)\n",
    "\n",
    "# Scikit-Learn ≥0.20 is required\n",
    "import sklearn\n",
    "assert sklearn.__version__ >= \"0.20\"\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "np.random.seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "mpl.rc('axes', labelsize=14)\n",
    "mpl.rc('xtick', labelsize=12)\n",
    "mpl.rc('ytick', labelsize=12)\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"ensembles\"\n",
    "IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n",
    "os.makedirs(IMAGES_PATH, exist_ok=True)\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n",
    "    path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format=fig_extension, dpi=resolution)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_openml\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, VotingClassifier\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "mnist = fetch_openml(\"mnist_784\", version=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist.target = mnist.target.astype(np.uint8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((50000, 784), (10000, 784), (10000, 784))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(mnist[\"data\"], mnist.target, train_size=60000)\n",
    "X_train, X_val,  y_train, y_val  = train_test_split(X_train, y_train, train_size=50000)\n",
    "X_train.shape, X_val.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "erf_clf = ExtraTreesClassifier(n_estimators=100, random_state=42)\n",
    "rnd_clf = RandomForestClassifier(n_estimators=100, random_state=42)\n",
    "svm_clf = LinearSVC(random_state=42)\n",
    "mlp_clf = MLPClassifier(random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training the ExtraTreesClassifier(bootstrap=False, ccp_alpha=0.0, class_weight=None,\n",
      "                     criterion='gini', max_depth=None, max_features='auto',\n",
      "                     max_leaf_nodes=None, max_samples=None,\n",
      "                     min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                     min_samples_leaf=1, min_samples_split=2,\n",
      "                     min_weight_fraction_leaf=0.0, n_estimators=100,\n",
      "                     n_jobs=None, oob_score=False, random_state=42, verbose=0,\n",
      "                     warm_start=False)\n",
      "Training the RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
      "                       criterion='gini', max_depth=None, max_features='auto',\n",
      "                       max_leaf_nodes=None, max_samples=None,\n",
      "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                       min_samples_leaf=1, min_samples_split=2,\n",
      "                       min_weight_fraction_leaf=0.0, n_estimators=100,\n",
      "                       n_jobs=None, oob_score=False, random_state=42, verbose=0,\n",
      "                       warm_start=False)\n",
      "Training the LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
      "          intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
      "          multi_class='ovr', penalty='l2', random_state=42, tol=0.0001,\n",
      "          verbose=0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/biowar1337/anaconda3/lib/python3.7/site-packages/sklearn/svm/_base.py:947: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training the MLPClassifier(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
      "              beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
      "              hidden_layer_sizes=(100,), learning_rate='constant',\n",
      "              learning_rate_init=0.001, max_fun=15000, max_iter=200,\n",
      "              momentum=0.9, n_iter_no_change=10, nesterovs_momentum=True,\n",
      "              power_t=0.5, random_state=42, shuffle=True, solver='adam',\n",
      "              tol=0.0001, validation_fraction=0.1, verbose=False,\n",
      "              warm_start=False)\n"
     ]
    }
   ],
   "source": [
    "estimators = [erf_clf, rnd_clf, svm_clf, mlp_clf]\n",
    "for estimator in estimators:\n",
    "    print(f\"Training the {estimator}\")\n",
    "    estimator.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "acc_ls = [estimator.score(X_val, y_val) for estimator in estimators]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ExtraTreesClassifier has reached 0.9718 accuracy!!!\n",
      "RandomForestClassifier has reached 0.9703 accuracy!!!\n",
      "LinearSVC has reached 0.8636 accuracy!!!\n",
      "MLPClassifier has reached 0.9658 accuracy!!!\n"
     ]
    }
   ],
   "source": [
    "for clf, acc in zip(estimators, acc_ls):\n",
    "    print(f\"{clf.__class__.__name__} has reached {acc} accuracy!!!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "voting_hard_clf = VotingClassifier(estimators=[(\"erf\", erf_clf),\n",
    "                                               (\"rf\", rnd_clf),\n",
    "                                               (\"svm\", svm_clf),\n",
    "                                               (\"mlp\", mlp_clf)],\n",
    "                                   voting=\"hard\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/biowar1337/anaconda3/lib/python3.7/site-packages/sklearn/svm/_base.py:947: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('erf',\n",
       "                              ExtraTreesClassifier(bootstrap=False,\n",
       "                                                   ccp_alpha=0.0,\n",
       "                                                   class_weight=None,\n",
       "                                                   criterion='gini',\n",
       "                                                   max_depth=None,\n",
       "                                                   max_features='auto',\n",
       "                                                   max_leaf_nodes=None,\n",
       "                                                   max_samples=None,\n",
       "                                                   min_impurity_decrease=0.0,\n",
       "                                                   min_impurity_split=None,\n",
       "                                                   min_samples_leaf=1,\n",
       "                                                   min_samples_split=2,\n",
       "                                                   min_weight_fraction_leaf=0.0,\n",
       "                                                   n_estimators=100,\n",
       "                                                   n_jobs=None, oob_score...\n",
       "                                            epsilon=1e-08,\n",
       "                                            hidden_layer_sizes=(100,),\n",
       "                                            learning_rate='constant',\n",
       "                                            learning_rate_init=0.001,\n",
       "                                            max_fun=15000, max_iter=200,\n",
       "                                            momentum=0.9, n_iter_no_change=10,\n",
       "                                            nesterovs_momentum=True,\n",
       "                                            power_t=0.5, random_state=42,\n",
       "                                            shuffle=True, solver='adam',\n",
       "                                            tol=0.0001, validation_fraction=0.1,\n",
       "                                            verbose=False, warm_start=False))],\n",
       "                 flatten_transform=True, n_jobs=None, voting='hard',\n",
       "                 weights=None)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_hard_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9728"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_hard_clf.score(X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('erf',\n",
       "                              ExtraTreesClassifier(bootstrap=False,\n",
       "                                                   ccp_alpha=0.0,\n",
       "                                                   class_weight=None,\n",
       "                                                   criterion='gini',\n",
       "                                                   max_depth=None,\n",
       "                                                   max_features='auto',\n",
       "                                                   max_leaf_nodes=None,\n",
       "                                                   max_samples=None,\n",
       "                                                   min_impurity_decrease=0.0,\n",
       "                                                   min_impurity_split=None,\n",
       "                                                   min_samples_leaf=1,\n",
       "                                                   min_samples_split=2,\n",
       "                                                   min_weight_fraction_leaf=0.0,\n",
       "                                                   n_estimators=100,\n",
       "                                                   n_jobs=None, oob_score...\n",
       "                                            epsilon=1e-08,\n",
       "                                            hidden_layer_sizes=(100,),\n",
       "                                            learning_rate='constant',\n",
       "                                            learning_rate_init=0.001,\n",
       "                                            max_fun=15000, max_iter=200,\n",
       "                                            momentum=0.9, n_iter_no_change=10,\n",
       "                                            nesterovs_momentum=True,\n",
       "                                            power_t=0.5, random_state=42,\n",
       "                                            shuffle=True, solver='adam',\n",
       "                                            tol=0.0001, validation_fraction=0.1,\n",
       "                                            verbose=False, warm_start=False))],\n",
       "                 flatten_transform=True, n_jobs=None, voting='hard',\n",
       "                 weights=None)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_hard_clf.set_params(svm=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('erf',\n",
       "  ExtraTreesClassifier(bootstrap=False, ccp_alpha=0.0, class_weight=None,\n",
       "                       criterion='gini', max_depth=None, max_features='auto',\n",
       "                       max_leaf_nodes=None, max_samples=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "                       n_jobs=None, oob_score=False, random_state=42, verbose=0,\n",
       "                       warm_start=False)),\n",
       " ('rf',\n",
       "  RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
       "                         criterion='gini', max_depth=None, max_features='auto',\n",
       "                         max_leaf_nodes=None, max_samples=None,\n",
       "                         min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                         min_samples_leaf=1, min_samples_split=2,\n",
       "                         min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "                         n_jobs=None, oob_score=False, random_state=42, verbose=0,\n",
       "                         warm_start=False)),\n",
       " ('svm', None),\n",
       " ('mlp',\n",
       "  MLPClassifier(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
       "                beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "                hidden_layer_sizes=(100,), learning_rate='constant',\n",
       "                learning_rate_init=0.001, max_fun=15000, max_iter=200,\n",
       "                momentum=0.9, n_iter_no_change=10, nesterovs_momentum=True,\n",
       "                power_t=0.5, random_state=42, shuffle=True, solver='adam',\n",
       "                tol=0.0001, validation_fraction=0.1, verbose=False,\n",
       "                warm_start=False))]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_hard_clf.estimators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "del voting_hard_clf.estimators_[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[ExtraTreesClassifier(bootstrap=False, ccp_alpha=0.0, class_weight=None,\n",
       "                      criterion='gini', max_depth=None, max_features='auto',\n",
       "                      max_leaf_nodes=None, max_samples=None,\n",
       "                      min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                      min_samples_leaf=1, min_samples_split=2,\n",
       "                      min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "                      n_jobs=None, oob_score=False, random_state=42, verbose=0,\n",
       "                      warm_start=False),\n",
       " RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
       "                        criterion='gini', max_depth=None, max_features='auto',\n",
       "                        max_leaf_nodes=None, max_samples=None,\n",
       "                        min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                        min_samples_leaf=1, min_samples_split=2,\n",
       "                        min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "                        n_jobs=None, oob_score=False, random_state=42, verbose=0,\n",
       "                        warm_start=False),\n",
       " MLPClassifier(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
       "               beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "               hidden_layer_sizes=(100,), learning_rate='constant',\n",
       "               learning_rate_init=0.001, max_fun=15000, max_iter=200,\n",
       "               momentum=0.9, n_iter_no_change=10, nesterovs_momentum=True,\n",
       "               power_t=0.5, random_state=42, shuffle=True, solver='adam',\n",
       "               tol=0.0001, validation_fraction=0.1, verbose=False,\n",
       "               warm_start=False)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_hard_clf.estimators_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9746"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_hard_clf.score(X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "voting_hard_clf.voting = \"soft\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9726"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_hard_clf.score(X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9687"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_hard_clf.voting = \"hard\"\n",
    "voting_hard_clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.9676, 0.9654, 0.9597]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[estimator.score(X_test, y_test) for estimator in voting_hard_clf.estimators_]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_val_predictions = np.empty((len(X_val), len(estimators)), dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index, estimator in enumerate(estimators):\n",
    "    X_val_predictions[:, index] = estimator.predict(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 4)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_val_predictions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
       "                       criterion='gini', max_depth=None, max_features='auto',\n",
       "                       max_leaf_nodes=None, max_samples=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=200,\n",
       "                       n_jobs=None, oob_score=True, random_state=42, verbose=0,\n",
       "                       warm_start=False)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnd_forest_blender = RandomForestClassifier(n_estimators=200,\n",
    "                                            oob_score=True, \n",
    "                                            random_state=42)\n",
    "rnd_forest_blender.fit(X_val_predictions,\n",
    "                       y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9687"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnd_forest_blender.oob_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_predictions = np.empty((len(X_test), len(estimators)), dtype=np.float32)\n",
    "\n",
    "for index, estimator in enumerate(estimators):\n",
    "    X_test_predictions[:, index] = estimator.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = rnd_forest_blender.predict(X_test_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9676"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}