A wide variety of models for multi-class classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "611c231b-8839-4b64-9a6a-39cbabd45f26",
   "metadata": {},
   "source": [
    "# A wide variety of models for multi-class classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5fe17a91-fd69-4243-b29f-c08195e5c21a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X features= ['MedInc', 'HouseAge', 'AveRooms', 'AveBedrms', 'Population', 'AveOccup', 'Latitude', 'Longitude']\n",
      "(20640, 10) (13828, 10) (6812, 10)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEsCAYAAADHBxRtAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAixUlEQVR4nO3df5xdVX3u8c9DAiEiETAyjQk1VFMsEFGZ0lhaHAwXooABNSWIEBQbL40WNS0E9Wr9kSv+QPlRiUTABAHTXLyaXDAixk4tFcSA2BAgEiFCIBJAhYxgYOL3/rHWIXtPJsnJzJlz9mSe9+s1rzlnnb3n7NlrZj9nr732WooIzMzManZr9QaYmVm1OBjMzKzEwWBmZiUOBjMzK3EwmJlZiYPBzMxKhrd6A3Zk9OjRMX78+D6t+/vf/5699tqrsRtk/eI6qSbXS/X0t07uuOOOJyLiZX1Zt/LBMH78eFasWNGndTs7O+no6GjsBlm/uE6qyfVSPf2tE0m/6uu6bkoyM7MSB4OZmZU4GMzMrMTBYGZmJQ4GMzMrcTCYmVmJg8HMzEocDGZmVlL5G9z6Y+UjT3HmnBub/r5rLzi+6e9pZtYoPmMwM7MSB4OZmZU4GMzMrMTBYGZmJQ4GMzMrcTCYmVlJXcEgaR9J10u6T9K9kt4gaT9JN0u6P3/ft7D8+ZLWSFot6bhC+eGSVubXLpGkgfilzMys7+o9Y7gY+F5EvBo4DLgXmAMsj4gJwPL8HEkHA9OBQ4ApwGWShuWfMw+YCUzIX1Ma9HuYmVmD7DAYJI0CjgKuBIiI5yLid8BUYGFebCFwUn48FVgUEZsi4kFgDXCEpDHAqIi4NSICuLqwjpmZVUQ9dz7/GfA48HVJhwF3AOcAbRGxHiAi1kvaPy8/FritsP66XPZ8ftyzfCuSZpLOLGhra6Ozs7Pe36ekbSTMntjdp3X7o6/bOxR0dXV5/1SQ66V6Wlkn9QTDcOD1wAci4ieSLiY3G21Db9cNYjvlWxdGzAfmA7S3t0df5z299NolXLiy+aN+rD2to+nvOVh4buFqcr1UTyvrpJ5rDOuAdRHxk/z8elJQPJabh8jfNxSWP6Cw/jjg0Vw+rpdyMzOrkB0GQ0T8GnhY0kG5aDJwD7AUmJHLZgBL8uOlwHRJIyQdSLrIfHtudtooaVLujXRGYR0zM6uIettZPgBcK2kP4AHg3aRQWSzpLOAhYBpARKyStJgUHt3ArIjYnH/O2cACYCSwLH+ZmVmF1BUMEXEX0N7LS5O3sfxcYG4v5SuAQ3di+8zMrMl857OZmZXs0hP1WPOMr3NCpNkTuxs+eZInRjJrLJ8xmJlZiYPBzMxKHAxmZlbiYDAzsxIHg5mZlTgYzMysxMFgZmYlDgYzMytxMJiZWYmDwczMShwMZmZW4mAwM7MSB4OZmZU4GMzMrMTBYGZmJQ4GMzMrcTCYmVmJg8HMzEocDGZmVuJgMDOzEgeDmZmVOBjMzKykrmCQtFbSSkl3SVqRy/aTdLOk+/P3fQvLny9pjaTVko4rlB+ef84aSZdIUuN/JTMz64+dOWM4OiJeGxHt+fkcYHlETACW5+dIOhiYDhwCTAEukzQsrzMPmAlMyF9T+v8rmJlZI/WnKWkqsDA/XgicVChfFBGbIuJBYA1whKQxwKiIuDUiAri6sI6ZmVXE8DqXC+D7kgK4PCLmA20RsR4gItZL2j8vOxa4rbDuulz2fH7cs3wrkmaSzixoa2ujs7Ozzs0saxsJsyd292nd/ujr9g5m9e7ngaiTobi/G62rq8v7sWJaWSf1BsOREfFoPvjfLOm+7Szb23WD2E751oUpeOYDtLe3R0dHR52bWXbptUu4cGW9v2LjrD2to+nv2WpnzrmxruVmT+xueJ0Mxf3daJ2dnfT1/8wGRivrpK6mpIh4NH/fAHwbOAJ4LDcPkb9vyIuvAw4orD4OeDSXj+ul3MzMKmSHwSBpL0l71x4DxwJ3A0uBGXmxGcCS/HgpMF3SCEkHki4y356bnTZKmpR7I51RWMfMzCqinnP6NuDbuWfpcOC6iPiepJ8CiyWdBTwETAOIiFWSFgP3AN3ArIjYnH/W2cACYCSwLH+Z2U4aX2fTXb1mT+yuqzlw7QXHN/R9rZp2GAwR8QBwWC/lTwKTt7HOXGBuL+UrgEN3fjPNzKxZfOezmZmVOBjMzKzEwWBmZiUOBjMzK3EwmJlZiYPBzMxKHAxmZlbiYDAzsxIHg5mZlTgYzMysxMFgZmYlDgYzMytxMJiZWYmDwczMShwMZmZW4mAwM7MSB4OZmZU4GMzMrMTBYGZmJQ4GMzMrcTCYmVmJg8HMzEocDGZmVlJ3MEgaJulnkm7Iz/eTdLOk+/P3fQvLni9pjaTVko4rlB8uaWV+7RJJauyvY2Zm/bUzZwznAPcWns8BlkfEBGB5fo6kg4HpwCHAFOAyScPyOvOAmcCE/DWlX1tvZmYNV1cwSBoHHA9cUSieCizMjxcCJxXKF0XEpoh4EFgDHCFpDDAqIm6NiACuLqxjZmYVUe8Zw0XAucAfC2VtEbEeIH/fP5ePBR4uLLcul43Nj3uWm5lZhQzf0QKSTgA2RMQdkjrq+Jm9XTeI7ZT39p4zSU1OtLW10dnZWcfbbq1tJMye2N2ndfujr9s7mNW7nweiTry/+6/eehmK+7pVurq6Wra/dxgMwJHAWyW9BdgTGCXpGuAxSWMiYn1uJtqQl18HHFBYfxzwaC4f10v5ViJiPjAfoL29PTo6Our/jQouvXYJF66s51dsrLWndTT9PVvtzDk31rXc7IndDa8T7+/+q7dehuK+bpXOzk76euzrrx02JUXE+RExLiLGky4q/zAi3gUsBWbkxWYAS/LjpcB0SSMkHUi6yHx7bm7aKGlS7o10RmEdMzOriP58dLsAWCzpLOAhYBpARKyStBi4B+gGZkXE5rzO2cACYCSwLH+ZmVmF7FQwREQn0JkfPwlM3sZyc4G5vZSvAA7d2Y00M7Pm8Z3PZmZW4mAwM7MSB4OZmZU4GMzMrMTBYGZmJQ4GMzMraf5twWZmg8j4Bt9lXq8FU/ZqyfuCzxjMzKwHB4OZmZU4GMzMrMTBYGZmJQ4GMzMrcTCYmVmJg8HMzEocDGZmVuJgMDOzEgeDmZmVOBjMzKzEwWBmZiUOBjMzK3EwmJlZiYPBzMxKHAxmZlbiYDAzs5IdBoOkPSXdLunnklZJ+mQu30/SzZLuz9/3LaxzvqQ1klZLOq5Qfriklfm1SyRpYH4tMzPrq3rOGDYBb4qIw4DXAlMkTQLmAMsjYgKwPD9H0sHAdOAQYApwmaRh+WfNA2YCE/LXlMb9KmZm1gg7DIZIuvLT3fNXAFOBhbl8IXBSfjwVWBQRmyLiQWANcISkMcCoiLg1IgK4urCOmZlVRF3XGCQNk3QXsAG4OSJ+ArRFxHqA/H3/vPhY4OHC6uty2dj8uGe5mZlVyPB6FoqIzcBrJe0DfFvSodtZvLfrBrGd8q1/gDST1OREW1sbnZ2d9WzmVtpGwuyJ3X1atz/6ur2DWb37eSDqxPu7/+qtF+/r5unq6mrZ/q4rGGoi4neSOknXBh6TNCYi1udmog15sXXAAYXVxgGP5vJxvZT39j7zgfkA7e3t0dHRsTOb+YJLr13ChSt36ldsiLWndTT9PVvtzDk31rXc7IndDa8T7+/+q7devK+bZ8GUvejrsa+/6umV9LJ8poCkkcAxwH3AUmBGXmwGsCQ/XgpMlzRC0oGki8y35+amjZIm5d5IZxTWMTOziqjno9sYYGHuWbQbsDgibpB0K7BY0lnAQ8A0gIhYJWkxcA/QDczKTVEAZwMLgJHAsvxlZmYVssNgiIj/Bl7XS/mTwORtrDMXmNtL+Qpge9cnzMysxXzns5mZlTgYzMysxMFgZmYlDgYzMytxMJiZWYmDwczMShwMZmZW4mAwM7MSB4OZmZU4GMzMrMTBYGZmJQ4GMzMrcTCYmVmJg8HMzEocDGZmVuJgMDOzEgeDmZmVOBjMzKzEwWBmZiUOBjMzK3EwmJlZiYPBzMxKHAxmZlbiYDAzs5IdBoOkAyT9u6R7Ja2SdE4u30/SzZLuz9/3LaxzvqQ1klZLOq5Qfriklfm1SyRpYH4tMzPrq3rOGLqB2RHxF8AkYJakg4E5wPKImAAsz8/Jr00HDgGmAJdJGpZ/1jxgJjAhf01p4O9iZmYNsMNgiIj1EXFnfrwRuBcYC0wFFubFFgIn5cdTgUURsSkiHgTWAEdIGgOMiohbIyKAqwvrmJlZRezUNQZJ44HXAT8B2iJiPaTwAPbPi40FHi6sti6Xjc2Pe5abmVmFDK93QUkvBr4FfDAint7O5YHeXojtlPf2XjNJTU60tbXR2dlZ72aWtI2E2RO7+7Ruf/R1ewezevfzQNSJ93f/1Vsv3tfN09XV1bL9XVcwSNqdFArXRsT/zcWPSRoTEetzM9GGXL4OOKCw+jjg0Vw+rpfyrUTEfGA+QHt7e3R0dNT32/Rw6bVLuHBl3dnXMGtP62j6e7bamXNurGu52RO7G14n3t/9V2+9eF83z4Ipe9HXY19/1dMrScCVwL0R8aXCS0uBGfnxDGBJoXy6pBGSDiRdZL49NzdtlDQp/8wzCuuYmVlF1PPR7UjgdGClpLty2UeAC4DFks4CHgKmAUTEKkmLgXtIPZpmRcTmvN7ZwAJgJLAsf5mZWYXsMBgi4hZ6vz4AMHkb68wF5vZSvgI4dGc20MzMmst3PpuZWYmDwczMShwMZmZW4mAwM7MSB4OZmZU4GMzMrMTBYGZmJQ4GMzMrcTCYmVmJg8HMzEocDGZmVuJgMDOzEgeDmZmVOBjMzKzEwWBmZiUOBjMzK3EwmJlZiYPBzMxKHAxmZlbiYDAzsxIHg5mZlTgYzMysxMFgZmYlDgYzMyvZYTBIukrSBkl3F8r2k3SzpPvz930Lr50vaY2k1ZKOK5QfLmllfu0SSWr8r2NmZv1VzxnDAmBKj7I5wPKImAAsz8+RdDAwHTgkr3OZpGF5nXnATGBC/ur5M83MrAJ2GAwR8SPgNz2KpwIL8+OFwEmF8kURsSkiHgTWAEdIGgOMiohbIyKAqwvrmJlZhfT1GkNbRKwHyN/3z+VjgYcLy63LZWPz457lZmZWMcMb/PN6u24Q2ynv/YdIM0nNTrS1tdHZ2dmnjWkbCbMndvdp3f7o6/YOZvXu54GoE+/v/qu3Xryvm6erq6tl+7uvwfCYpDERsT43E23I5euAAwrLjQMezeXjeinvVUTMB+YDtLe3R0dHR5828tJrl3DhykZn346tPa2j6e/ZamfOubGu5WZP7G54nXh/91+99eJ93TwLpuxFX499/dXXpqSlwIz8eAawpFA+XdIISQeSLjLfnpubNkqalHsjnVFYx8zMKmSHHxEkfRPoAEZLWgd8ArgAWCzpLOAhYBpARKyStBi4B+gGZkXE5vyjzib1cBoJLMtfZmZWMTsMhog4dRsvTd7G8nOBub2UrwAO3amtMzOzpvOdz2ZmVuJgMDOzEgeDmZmVOBjMzKzEwWBmZiUOBjMzK3EwmJlZiYPBzMxKHAxmZlbiYDAzsxIHg5mZlTgYzMysxMFgZmYlDgYzMytxMJiZWYmDwczMShwMZmZW4mAwM7MSB4OZmZU4GMzMrMTBYGZmJQ4GMzMrcTCYmVmJg8HMzEqaHgySpkhaLWmNpDnNfn8zM9u+pgaDpGHAV4A3AwcDp0o6uJnbYGZm29fsM4YjgDUR8UBEPAcsAqY2eRvMzGw7FBHNezPpHcCUiHhvfn468FcR8f4ey80EZuanBwGr+/iWo4En+riuDQzXSTW5Xqqnv3Xyioh4WV9WHN6PN+0L9VK2VTJFxHxgfr/fTFoREe39/TnWOK6TanK9VE8r66TZTUnrgAMKz8cBjzZ5G8zMbDuaHQw/BSZIOlDSHsB0YGmTt8HMzLajqU1JEdEt6f3ATcAw4KqIWDWAb9nv5ihrONdJNbleqqdlddLUi89mZlZ9vvPZzMxKHAxmZlbiYDAzsxIHg+0S8nArVjGul9aR1OfORQ6GHvyHPDhFxGZJu0k6WdKerd4eS1wvrZN7gQ6T9ClJ++7Mug6GTNLrJI2OiM35+Ysl7dPizbId6DFC71TgxIj4Q6u2xxLXS+tI+rSkU/LTvYHXRMRvd+ZnOBi2eCvwTQBJewHfAP6spVtk9XiHpM78+Amgu/iipN6GYbGB53ppndXAxySdDDwPjM43FNfNwbDFF4AbJN0ELAeWRcSdLd4m24Zak18eS6ZL0jLgN8CdkkZL2kPS7kCfBhGzvnG9tI6k3SXtGRHXAP+Sv04H7gTGSvpzSWMlHbSjYB6yN7hJUuRfXtLuEfG8pAOBTuB3EXFY8bUWbqr1IOnL+eEzwMciIiR9h3TWdzdwGzAWGAlcERHXtWRDhxjXS+tIuhbYTBqL7tyI+KmktwIXA68AvggcRaqb70fEBdv9eUMxGHqEwvB8keblwGLg+rzYG4F/iIj1rdpO25qkucBfAucB/xu4H7gyIn4u6QrgTcAEYARwUET8rGUbO4S4XlpH0geBycDbgFnAqcDngCXAMcAFwD9FxPJ6f+aQa0qSNKIQCl8GjswvnQRcFxEXAf8K/BI4pbefYa2R20lfDlyUDyzTgI3AWZJelef5eBBYERHPAHfl9dyePYBybyPXS+vsBqyOiOfz8etC4H3A5Ii4CfgscJ2kSbV9vqN9P6SCQdK5wAclvTgXvYw0FDgRcVlEXJbLNwPn5Z1sFZFn/fsRcIKkAyKii/RHH8CH8jKTgWvz4yh+t8aTNDL3NnK9tM6PgRdJ+nOAiFgMXANcKml8fn58RNxW774fUsFA2oHHAu+TtD+prbPX6weFbqv+VNNikmYVnv6cFNwdkvbPB6GPAm+UdCRARHwxr+e6G0CSLgDOyPv5Z7hemkbSeyQdLelQ0nQGw0h1MU7SsIj4BmlKg3aAiFiR16vrmN/sGdxaQtJrgcci4hZJHwAuBZ4jfaL5R0n/SQqJfYCvR8Sm2rr+VNNakjrY8snnnyPiTkmvBo4GRkj6UUT8QtI6oNRP3nU3cCRdCbQBn877+a7cPfUEXC8DStI3gD8BHgBeCvwQOAe4ktSEdBtwIykUVhbXjYg/1vMeu3wwSLqedJPHaEmLgW8B/xO4DBgD/Ap4PbAv8EgxFKwSHgR+ABwlaVFETI+I6yQ9DUwC3i/p18BzEXFHS7d0iMhn2/tExAn5+ZGkD1o3k+5ZOBbXy4CQdADw8oiYnK+5TQC+BuwBvIcUEKfnZvOHIuLqvrzPLh0Mko4G9s078ShSgs4BLiLtxKuAe4BrioFQ7LVkrZNPe9cBt5C63V2Te7g8AiwgTfi0CHhpRPxHbZ16PxVZnz0JbJJ0GKnzxlmk7qhvAE6OiI9K+jfS/57rpUFyE9wmYKOkP42Ih4BVks4gTerzSER8Pt8n8qqIuDevt9P7fpcOBlKK7gYQET/Kp7XPk84YPgKcTwqHR4Fl4FCoglod1P6Yc1fi10XEiZIeBkaTmjC6SQek4no++AygfND5I7CW1PtoT2BqRKyTNBu4XlJ7RPx3YR3XSwPk49IGSb8Alkg6KiI2kupiHtAu6fp831UtFPq073fJi89KQ1qQu2o9Lul/5ecPkJol9gSOzBdkToqIZbV1HQqtJemjwMWSTpHUlot/AIyX9BFS09KtwLXqMXqk627gSDoJIHeJ3Exqzz46f700v3Yh6SJ06bjieukfSR+S9Je15xFxLqkX2P/LvcK6SV2AX0NqNqewbJ/2/S4XDJKuIY0T8spcdAnpdvBat7l7gTuAt+RTrAfyeu4p0WKSvkZq7ruD1NR3TH5pFakep0bEURHxJuCu/A9hAyxf7Fycm/EAiIhfAu8CngKmSpqer+F150+x1gCSrgY+CHxc0qRaeUScQ/q/uF3S6cBXSNcUnm7E++5ywQC8hHRB5hhJ44D/It0B+GpJCySNJvWc2FA8xfKnmtZSun3/TyPi5IhYSLrJcI6kl+QwPwH469ryEfHZvJ4DfQBJej2px95hwD49wuFBYAapA8cE4L6IOCOv53rpp3y/VRdwGvBt4KOS3lB7PSJmAV8CXgTcFhHvy+v1e9/vckNiSPoM6S7M4aT+vV8HxgHPkm7V30T6vd+dl/c1hQpQGuJ8bESsyr0tRgDfAd4RhSGDcx/tza3ZyqFJ0isj4peSxgCXA49HxFnbWd4XmvupdlySNII0Mu0+wHRgCvC5iLhlG+s1ZN/vMsFQ2JHHki5O3gJ8nPRJ578i4oN5uT0i3UHrP+CK6XnQl3QzMC0ifidpGvCD2Mlx5a3vevvQlK/7XEFqtpgl6e+Bm3IPGX/QapDC8aw4rtto4J2kcdw+C5xM6lF5b6Pff9A3JUl6Tf60WZsdaj3wd/kPdT/SaIMP5U87FELBPSUqJrbcbT688EmpTdJ1QIdDobl6O8BHxGPA35Gu220C3lgLhW2tYzunR7i+MPNaRDxB6n10M6kDxqEDEQowyINB0uWki5LzgA9L2pt0QebhfMFsb9Lp15HAa4vr+g+40v6Y7yt5ljTa7eO5PdVt1xUQEc+SruUtjoh3geulUXqcIXwAuKC4b3NX1HcCiyJiam2dRm/HoL2PQdKlpBubOiSdSBoddZ+I2CjpD6QxyCdHmmfhsYFKVmu8Hmdyq3MPDDf9VYSkycDPC82zrpcG6BEK7ycNo30icJqk/4yIX+XWkR9ExKfycgOy7wflNYZ8ZvAZ4JLcbQ5JS4AbIuJr+XmtjW54rVuj2z+rp7c/7ELdjY2IR7a1nA2ceve366XxlAaNfAcpFN4H/A/S6Kibeyw3YPt+0DUl5YPGRtLIjeuV7sSEdAfsnoVFXwRQ7OvuUKiW4nUeSadKmgClYZkf6bmcDbxt1UvtteKyrpfGknQOqfn7RGAm6V6eEyJis3qMjDqQ+37QBUPhoNEVEc/Elmk3fwu8GEDSV9lyc5RVQPGAUntcOG2+lDTezi97W9eBPnB2tl5cF42jPD+2pJfk7/uT7iJ/M3A2aVa2EyPNMDmsmSE86IKhZ2oWPEEaNuFKYERELGniZtl29Gg7HV48uEjajzQN5JSI+KMvYjaP66W1Ck1Dn5H0FxGxAfgX0jSox5Lu9K+FQlPv3RkUwSDpOEmH5HsQtmqPzg+fBP4eeCIKN681eVOtB209lepfF1+PiN9ExCWFfwB/Im0C10tr1Y5Nkt4LjKp1jsnHt18Cb25VKMAgCAalAfGOI01wfZOk8YXrCsVT27uBqyPivLzebv5jbi31PpXqI4XXe7ZX+47mJnC9tF7h2DSedF8CkEavjYhVORR2a9W+r3QwSHpZRPwe+KeI+BhpRMEvAyfW2uXycsMi4sGIODM/d0+JatjuVKqFT6w+s2su10sFSHo76Vro0vz8jcBcpXkuWnphv7LdVSXNJ/Us+nRErC6Un0H6o74qIn7oEKgebZlKdb2kg0lTqX6HdCv/WmCbU6nawHG9VEPtmCXpn0ktHaOBg4AO0vww34mI37RwEysdDItIE4I8DlwZ5Yk/3k+aJOStEfFUizbReqHCVKpAbSpVsWUq1ZtII0bWplL9XIs2dUhxvVRLvri/gjRs+ePAp4H1EbGmpRuWVS4Ychvb85I+CYwiDek7njS87KiIuDsv9znSCJwf9hlDNShNpfqxKE+lejBpKtWnSJ+GFuGpVJvK9dJ6PVs2lAbEm0a6g/zHrduy3lXuGkPhvoRvARvy9/XADcDswqJfAx4D/IdbHSMoTKVKaqb4OWkq1adIU6meQ+qOB/jg0ySlKW5xvTTFtnpT5pB4IiLm1UKhdk9DVVQmGCSdLmleoWgYcEREPAwcShq06xltmZltHenehRc1d0utJ6Wxc4iI75HmpPVUqhUSaYrbJyR9PD93vQwwSS8iDWWxVW/K3lo4qtbzqzLBACwDfi1pFEBE/Ay4U9L3SENnv410r8Lf5Nf/QLr28PsWba/xQieBDxd6if0raUjmD4OnUm0VSRdIOl/SebnoYtIQ5rPB9TLQIuIZ4J+315uyyqoUDJtJZwanwgt/oMNIQ10cHRF3kHpKLKyt4GsLraU0LPC4iDi+0Angx6SpVCdIWihPpdp0ki4EXk+awXCapC+SLnB+FzhI0tddLwNDaaTnmuEAEfEJ0tScbwMOz8tV6di7lUpdfM79d28Azo+Ia2rtbpEHkIotA3u5/bMCJH0IeDoirpQ0k9Tl7llSG/YTwKdIk+14KtUmkbQncA3wxYi4Ld8gOh94CLgQ2J3UkeNZXC8NpTSh1N8C34otQ5LvXrtuKukfgFMYBL0pKxUMAJKOIU2+c2FEXFko9/0KFSOpdkf6j4D3kEa8PYt0QfPciHhWnkq1aWoH99xcNAz4WkT8VmmY+kWkuS0+nJctHrBcL/0kqR34AvBJ0pDZ3dsIhy+QwrnSvSkrdzoTET8gDTf7MUkfkPSWXF7ZnTjUKANWs+W6z7yI+A/gTOAQUjOFp1JtjXtJMxa+RtLISMPUnwG8Id/YRuFA5XppgHwBfwZpys0rgL0kXVTofl8bxuerDILelJULBoCIuIV0d3M38CZJZ7Z2iwxA0gJJp8YWa0lnC38CHCNpQj7IPEVhiAVw2/VAKtYLQER8F/gJqQvq30raPyKeBDaSbhp9geulf3JvyssBIuKhfB/IStJd5XuTmlMBTlGafe0RBkFvyso1JfVG0qiIeLrV2zGU5S6o/wg8DZwXEdcXXjsOOAI4CngGeC4iprVkQ4eYHdTL+4DDSDeIDge6IuJtrdjOXVW+iD8L+HJEPK0tw13sAYwF5gDvIs0ueUpep/JNd4NizmeHQmvlP/IHgFeSelVclf+4F0PqJy9pOfB/gNH5jG9Q/AMMZnXUy+WSxgHjgLER8a28ni80N06xN+XlORR2y02oD0p6HXB9RMyAwdN0NyjOGKz1JI0g/b38QdIUYB7wkYj4Zn79JcWeFj74NEcd9VI623ZYN16hN+V5EXFdLtuNNDjhuyLirFrZYNn3DgbbKYVT5eNJN+3MAd4O/HtEXNHarRu6XC+tVehN+fmIWNDL64MmFMDBYDvQ2x+08qxS+Z/h+8DiiJjemi0cmlwv1SPpb4BvkIL5gYi4IZcPurNnB4NtJV9MXgfcX+tuuo3lvgS8IiLenp8Pqk9Fg43rpfokTSBNvvNKYGUURmoYTAbFxWdrnnyn7LGkO2OPlPRu0vj8z/dYbiTw3XzfiQ8+A8z1MjhExP3A/ZCu77R4c/rMZwz2gtpdyoU7aD8JvIZ0erx8W7fxD8ZT5cGk0ERUu47wKWAirhcbIA4GA0DSp0lTOv4sIq4qlHsq1RaS9BFSvdxV6/GSy10vNmAqeeezNZekr5KGsVgGXCxpUu21iLiadJv/J3KXVB98mkRpSPO/Au4CPi7p9NpruV5+jOvFBoDPGIY4SScD74mIE/Pzy4F7gPtI0w7+Opd/njQTWKUH/9pVSHob8N6IeEt+fixpgLY3Ac9HRHcud71Yw/mMYYiLiG+TRoNE0ntJg609R7rNf6bSMM6Qhm6u/OBfu5AfAufCC3c4353Ld4+Ibm0Zz/9yXC/WYA6GIUqFqVRjywTwK4BXRcQ80kHpnaQ5FsBTqTZFrpevRMTvSGdtRMRzEfEoaRC8rrzoCXnEzkExKJsNLg6GoWsZ8FitS12+eHlXRDwCEBH3Ab8AhufeLZ5KtTmWAY9L2jufGUjSsHzWMAx4haTFwAkR8bzrxQaCg2Ho2ky64HwqlOe7yAejfwN+GxF31Lo8ug27KWr18k54YVjsYaRhzAVcDzwaETNh8AzKZoOLb3AboiLN7PUZ4AZJGwuDf7UB7waejYgzc5n7wzdJb/USWyY72gisjy0zg7mLqg0I90oa4nob/EvSSyNN7OKDT4tso15eFRFr8mPXiw0YB4MVB/+6iDQv8Pdyuc8UWqhHvdwbEd/P5Q4FG1AOBgO2Gvzr7t6GDrbmc71YKzgYbCs9J3exanC9WLM4GMzMrMTdVc3MrMTBYGZmJQ4GMzMrcTCYmVmJg8HMzEocDGZmVvL/AaUQUlgviEGmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.datasets import fetch_california_housing\n",
    "cal_housing = fetch_california_housing()\n",
    "X = pd.DataFrame(cal_housing.data, columns=cal_housing.feature_names)\n",
    "y = cal_housing.target\n",
    "df = X.copy()\n",
    "df['y'] = y * 100000 # The raw value is in 100,000.\n",
    "X_features = cal_housing.feature_names\n",
    "print(\"X features=\", X_features)\n",
    "\n",
    "# Make the multi-class target\n",
    "def val_y(y):\n",
    "    if y<=100000:\n",
    "        return '(1) <=100k'\n",
    "    elif y<=160000:\n",
    "        return '(2) 100k - 160k'\n",
    "    elif y<=240000:\n",
    "        return '(3) 160k - 240k'\n",
    "    else:\n",
    "        return '(4) 240k+'\n",
    "    \n",
    "df['y_class'] = df['y'].apply(val_y)\n",
    "\n",
    "train, test = train_test_split(df, test_size=0.33, random_state=42)\n",
    "print(df.shape, train.shape, test.shape)\n",
    "\n",
    "df['y_class'].hist(xrot=45)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93a9d076-a7ba-49e0-bcdc-f3c2413fc21e",
   "metadata": {},
   "source": [
    "## (1) Multi-class Logistic Classification\n",
    "* Logistic Regression: [here](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html#sklearn.linear_model.LogisticRegression)\n",
    "* Logistic Regression with Cross Validation: [here](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegressionCV.html#sklearn.linear_model.LogisticRegressionCV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d541b35b-f9cd-4402-9b0f-6dfdcbfd4071",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6812, 6)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Actual</th>\n",
       "      <th>Y_prob (1) &lt;=100k</th>\n",
       "      <th>Y_prob (2) 100k - 160k</th>\n",
       "      <th>Y_prob (3) 160k - 240k</th>\n",
       "      <th>Y_prob (4) 240k+</th>\n",
       "      <th>Y_pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(1) &lt;=100k</td>\n",
       "      <td>0.453130</td>\n",
       "      <td>0.331161</td>\n",
       "      <td>0.206053</td>\n",
       "      <td>0.009656</td>\n",
       "      <td>(1) &lt;=100k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(1) &lt;=100k</td>\n",
       "      <td>0.238225</td>\n",
       "      <td>0.342282</td>\n",
       "      <td>0.328045</td>\n",
       "      <td>0.091448</td>\n",
       "      <td>(2) 100k - 160k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(4) 240k+</td>\n",
       "      <td>0.067076</td>\n",
       "      <td>0.108626</td>\n",
       "      <td>0.190556</td>\n",
       "      <td>0.633742</td>\n",
       "      <td>(4) 240k+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(3) 160k - 240k</td>\n",
       "      <td>0.003863</td>\n",
       "      <td>0.130184</td>\n",
       "      <td>0.320634</td>\n",
       "      <td>0.545318</td>\n",
       "      <td>(4) 240k+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(4) 240k+</td>\n",
       "      <td>0.109416</td>\n",
       "      <td>0.269652</td>\n",
       "      <td>0.330050</td>\n",
       "      <td>0.290882</td>\n",
       "      <td>(3) 160k - 240k</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Actual  Y_prob (1) <=100k  Y_prob (2) 100k - 160k  \\\n",
       "0       (1) <=100k           0.453130                0.331161   \n",
       "1       (1) <=100k           0.238225                0.342282   \n",
       "2        (4) 240k+           0.067076                0.108626   \n",
       "3  (3) 160k - 240k           0.003863                0.130184   \n",
       "4        (4) 240k+           0.109416                0.269652   \n",
       "\n",
       "   Y_prob (3) 160k - 240k  Y_prob (4) 240k+           Y_pred  \n",
       "0                0.206053          0.009656       (1) <=100k  \n",
       "1                0.328045          0.091448  (2) 100k - 160k  \n",
       "2                0.190556          0.633742        (4) 240k+  \n",
       "3                0.320634          0.545318        (4) 240k+  \n",
       "4                0.330050          0.290882  (3) 160k - 240k  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#modeling\n",
    "LM_model = LogisticRegression(penalty='l2', max_iter=500, multi_class='ovr', solver='liblinear')\n",
    "LM_model.fit(train[X_features],train['y_class'])\n",
    "\n",
    "# Prediction\n",
    "LM_pred_prob = pd.DataFrame(LM_model.predict_proba(test[X_features]))\n",
    "LM_pred = pd.DataFrame(LM_model.predict(test[X_features]))\n",
    "LM_pred = pd.concat([test.reset_index()['y_class'],LM_pred_prob,LM_pred],axis=1, ignore_index=True)\n",
    "print(LM_pred.shape)\n",
    "\n",
    "LM_pred.columns = ['Actual','Y_prob (1) <=100k', 'Y_prob (2) 100k - 160k','Y_prob (3) 160k - 240k', 'Y_prob (4) 240k+', 'Y_pred']\n",
    "LM_pred.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3042a628-0486-4274-a87c-7d5701f0c1f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Actual</th>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <th>(4) 240k+</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y_pred</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <td>769</td>\n",
       "      <td>383</td>\n",
       "      <td>154</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <td>265</td>\n",
       "      <td>355</td>\n",
       "      <td>207</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <td>127</td>\n",
       "      <td>679</td>\n",
       "      <td>832</td>\n",
       "      <td>303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(4) 240k+</th>\n",
       "      <td>44</td>\n",
       "      <td>173</td>\n",
       "      <td>714</td>\n",
       "      <td>1709</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Actual           (1) <=100k  (2) 100k - 160k  (3) 160k - 240k  (4) 240k+\n",
       "Y_pred                                                                  \n",
       "(1) <=100k              769              383              154         57\n",
       "(2) 100k - 160k         265              355              207         41\n",
       "(3) 160k - 240k         127              679              832        303\n",
       "(4) 240k+                44              173              714       1709"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(LM_pred['Y_pred'],LM_pred['Actual']) #.apply(lambda x: x/x.sum(), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "20279845-8194-4a96-a8b3-d4f7626285b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Actual</th>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <th>(4) 240k+</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y_pred</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <td>0.56</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <td>0.31</td>\n",
       "      <td>0.41</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <td>0.07</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.43</td>\n",
       "      <td>0.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(4) 240k+</th>\n",
       "      <td>0.02</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.65</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Actual           (1) <=100k  (2) 100k - 160k  (3) 160k - 240k  (4) 240k+\n",
       "Y_pred                                                                  \n",
       "(1) <=100k             0.56             0.28             0.11       0.04\n",
       "(2) 100k - 160k        0.31             0.41             0.24       0.05\n",
       "(3) 160k - 240k        0.07             0.35             0.43       0.16\n",
       "(4) 240k+              0.02             0.07             0.27       0.65"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(LM_pred['Y_pred'],LM_pred['Actual']).apply(lambda x: x/x.sum(), axis=1).round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f5863bd-4108-42db-9234-f7a47cd6616c",
   "metadata": {},
   "source": [
    "## (2) Multi-class Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0f8e5690-ec28-4c42-9aca-d1ad8e4b4d0a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Actual</th>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <th>(4) 240k+</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y_pred</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <td>0.73</td>\n",
       "      <td>0.21</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <td>0.17</td>\n",
       "      <td>0.56</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <td>0.03</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(4) 240k+</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.77</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Actual           (1) <=100k  (2) 100k - 160k  (3) 160k - 240k  (4) 240k+\n",
       "Y_pred                                                                  \n",
       "(1) <=100k             0.73             0.21             0.05       0.02\n",
       "(2) 100k - 160k        0.17             0.56             0.22       0.06\n",
       "(3) 160k - 240k        0.03             0.19             0.58       0.21\n",
       "(4) 240k+              0.01             0.04             0.19       0.77"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "# Modeling\n",
    "dt_model = DecisionTreeClassifier(random_state=0).fit(train[X_features],train['y_class'])\n",
    "\n",
    "# Prediction\n",
    "def Prediction(model):\n",
    "    pred_prob = pd.DataFrame(model.predict_proba(test[X_features]))\n",
    "    pred = pd.DataFrame(model.predict(test[X_features]))\n",
    "    pred = pd.concat([test.reset_index()['y_class'],pred_prob,pred],axis=1, ignore_index=True)\n",
    "    pred.columns = ['Actual','Y_prob (1) <=100k', 'Y_prob (2) 100k - 160k','Y_prob (3) 160k - 240k', 'Y_prob (4) 240k+', 'Y_pred']\n",
    "    return(pred)\n",
    "dt_pred = Prediction(dt_model)  \n",
    "\n",
    "# Actual vs. Prediction\n",
    "pd.crosstab(dt_pred['Y_pred'],dt_pred['Actual']).apply(lambda x: x/x.sum(), axis=1).round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccbbc636-6e9e-48b2-9920-b88ddb509760",
   "metadata": {},
   "source": [
    "## (3) Multi-class Random Forest\n",
    "* Random forest: [here](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html#sklearn.ensemble.RandomForestClassifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "adf99c3f-d7c5-4b13-b999-b2a2138d2e62",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Actual</th>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <th>(4) 240k+</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y_pred</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <td>0.77</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <td>0.17</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(4) 240k+</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.81</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Actual           (1) <=100k  (2) 100k - 160k  (3) 160k - 240k  (4) 240k+\n",
       "Y_pred                                                                  \n",
       "(1) <=100k             0.77             0.17             0.04       0.02\n",
       "(2) 100k - 160k        0.17             0.59             0.20       0.03\n",
       "(3) 160k - 240k        0.01             0.19             0.60       0.20\n",
       "(4) 240k+              0.01             0.02             0.16       0.81"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Modeling\n",
    "rf_model = RandomForestClassifier(max_depth=8, random_state=0).fit(train[X_features],train['y_class'])\n",
    "rf_pred = Prediction(rf_model)  \n",
    "\n",
    "# Actual vs. Prediction\n",
    "pd.crosstab(rf_pred['Y_pred'],rf_pred['Actual']).apply(lambda x: x/x.sum(), axis=1).round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72713807-4f23-483b-bfaf-cb808f764795",
   "metadata": {},
   "source": [
    "## (4) Multi-class Naive Bayes/Bernoulli\n",
    "* Naive Bayes/Bernoulli: [here](https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.BernoulliNB.html#sklearn.naive_bayes.BernoulliNB)\n",
    "* Naive Bayes/Gaussian: [here](https://scikit-learn.org/stable/modules/generated/sklearn.naive_bayes.GaussianNB.html#sklearn.naive_bayes.GaussianNB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "acb21d0a-7465-4117-9922-cb836180a1f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Actual</th>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <th>(4) 240k+</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y_pred</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <td>0.57</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <td>0.11</td>\n",
       "      <td>0.32</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <td>0.15</td>\n",
       "      <td>0.26</td>\n",
       "      <td>0.34</td>\n",
       "      <td>0.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(4) 240k+</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.08</td>\n",
       "      <td>0.90</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Actual           (1) <=100k  (2) 100k - 160k  (3) 160k - 240k  (4) 240k+\n",
       "Y_pred                                                                  \n",
       "(1) <=100k             0.57             0.28             0.10       0.05\n",
       "(2) 100k - 160k        0.11             0.32             0.38       0.18\n",
       "(3) 160k - 240k        0.15             0.26             0.34       0.25\n",
       "(4) 240k+              0.00             0.02             0.08       0.90"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import BernoulliNB\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "NB_model = GaussianNB().fit(train[X_features],train['y_class'])\n",
    "NB_pred = Prediction(NB_model)    \n",
    "pd.crosstab(NB_pred['Y_pred'],NB_pred['Actual']).apply(lambda x: x/x.sum(), axis=1).round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "73898f96-782f-4795-ba55-97e4e7e8421c",
   "metadata": {},
   "source": [
    "## (5) GMM\n",
    "* GMM: [here](https://scikit-learn.org/stable/modules/generated/sklearn.mixture.GaussianMixture.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ef1d7b9c-bb69-4bf4-94cd-0c29da54af0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Actual</th>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <th>(4) 240k+</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y_pred</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.29</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.16</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.22</td>\n",
       "      <td>0.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.13</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.38</td>\n",
       "      <td>0.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.19</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Actual  (1) <=100k  (2) 100k - 160k  (3) 160k - 240k  (4) 240k+\n",
       "Y_pred                                                         \n",
       "0             0.29             0.23             0.14       0.33\n",
       "1             0.16             0.30             0.22       0.33\n",
       "2             0.13             0.31             0.38       0.18\n",
       "3             0.19             0.19             0.24       0.38"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.mixture import GaussianMixture\n",
    "GMM_model = GaussianMixture(n_components=4, random_state=1,max_iter=100).fit(train[X_features],train['y_class'])\n",
    "GMM_pred = Prediction(GMM_model)    \n",
    "pd.crosstab(GMM_pred['Y_pred'],GMM_pred['Actual']).apply(lambda x: x/x.sum(), axis=1).round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8931098c-7c6f-4b70-a74e-e861633b98c1",
   "metadata": {},
   "source": [
    "## (6) K-nearest Neighbors (KNN)\n",
    "* KNeighborsClassifier: [here](https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html#sklearn.neighbors.KNeighborsClassifier)\n",
    "* NearestCentroid: [here](https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.NearestCentroid.html#sklearn.neighbors.NearestCentroid)\n",
    "and [here](https://link.springer.com/chapter/10.1007/978-3-642-17316-5_27)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e8d35370-3076-495e-8b86-df10ef8c2268",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Actual</th>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <th>(4) 240k+</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y_pred</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <td>0.27</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <td>0.17</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <td>0.14</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.34</td>\n",
       "      <td>0.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(4) 240k+</th>\n",
       "      <td>0.09</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.51</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Actual           (1) <=100k  (2) 100k - 160k  (3) 160k - 240k  (4) 240k+\n",
       "Y_pred                                                                  \n",
       "(1) <=100k             0.27             0.25             0.25       0.22\n",
       "(2) 100k - 160k        0.17             0.27             0.30       0.26\n",
       "(3) 160k - 240k        0.14             0.23             0.34       0.29\n",
       "(4) 240k+              0.09             0.16             0.24       0.51"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "KNN_model = KNeighborsClassifier(n_neighbors=3).fit(train[X_features],train['y_class'])\n",
    "KNN_pred = Prediction(KNN_model)    \n",
    "pd.crosstab(KNN_pred['Y_pred'],KNN_pred['Actual']).apply(lambda x: x/x.sum(), axis=1).round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "667821b1-9668-49c8-8cb6-31b8996fbb10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Actual</th>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <th>(4) 240k+</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y_pred</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <td>0.28</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.24</td>\n",
       "      <td>0.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <td>0.18</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <td>0.15</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.31</td>\n",
       "      <td>0.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(4) 240k+</th>\n",
       "      <td>0.11</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.49</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Actual           (1) <=100k  (2) 100k - 160k  (3) 160k - 240k  (4) 240k+\n",
       "Y_pred                                                                  \n",
       "(1) <=100k             0.28             0.27             0.24       0.20\n",
       "(2) 100k - 160k        0.18             0.27             0.30       0.25\n",
       "(3) 160k - 240k        0.15             0.23             0.31       0.31\n",
       "(4) 240k+              0.11             0.15             0.25       0.49"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "KNN_model = KNeighborsClassifier(n_neighbors=6).fit(train[X_features],train['y_class'])\n",
    "KNN_pred = Prediction(KNN_model)    \n",
    "pd.crosstab(KNN_pred['Y_pred'],KNN_pred['Actual']).apply(lambda x: x/x.sum(), axis=1).round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4315fe3-1c43-4095-85e6-34b0023a5991",
   "metadata": {},
   "source": [
    "## (7) Discriminant Analysis\n",
    "* Discriminant analysis: [here](https://scikit-learn.org/stable/modules/generated/sklearn.discriminant_analysis.LinearDiscriminantAnalysis.html#sklearn.discriminant_analysis.LinearDiscriminantAnalysis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a57e74fb-5093-4748-8811-00c524dc45e2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Actual</th>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <th>(4) 240k+</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Y_pred</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>(1) &lt;=100k</th>\n",
       "      <td>0.65</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(2) 100k - 160k</th>\n",
       "      <td>0.30</td>\n",
       "      <td>0.48</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(3) 160k - 240k</th>\n",
       "      <td>0.05</td>\n",
       "      <td>0.27</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>(4) 240k+</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.21</td>\n",
       "      <td>0.76</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Actual           (1) <=100k  (2) 100k - 160k  (3) 160k - 240k  (4) 240k+\n",
       "Y_pred                                                                  \n",
       "(1) <=100k             0.65             0.27             0.06       0.01\n",
       "(2) 100k - 160k        0.30             0.48             0.19       0.03\n",
       "(3) 160k - 240k        0.05             0.27             0.45       0.23\n",
       "(4) 240k+              0.00             0.02             0.21       0.76"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "disc_model = LinearDiscriminantAnalysis().fit(train[X_features],train['y_class'])\n",
    "disc_pred = Prediction(disc_model)    \n",
    "pd.crosstab(disc_pred['Y_pred'],disc_pred['Actual']).apply(lambda x: x/x.sum(), axis=1).round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9bfc3298-f069-4f4a-8c72-ce2f8a12f8f8",
   "metadata": {},
   "source": [
    "## (8) SVM\n",
    "* SVM: [here](https://scikit-learn.org/stable/modules/generated/sklearn.svm.LinearSVC.html#sklearn.svm.LinearSVC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9ed552ed-b6ac-4d2b-89fd-9f7db8020b9d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\datas\\anaconda3\\envs\\python38\\lib\\site-packages\\sklearn\\svm\\_base.py:985: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'LinearSVC' object has no attribute 'predict_proba'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_2608/84135091.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msvm\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mLinearSVC\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0msvm_model\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mLinearSVC\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmax_iter\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3000\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mX_features\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'y_class'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0msvm_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mPrediction\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msvm_model\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcrosstab\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msvm_pred\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'Y_pred'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msvm_pred\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'Actual'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mround\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_2608/867120899.py\u001b[0m in \u001b[0;36mPrediction\u001b[1;34m(model)\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[1;31m# Prediction\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mPrediction\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m     \u001b[0mpred_prob\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict_proba\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtest\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mX_features\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      9\u001b[0m     \u001b[0mpred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtest\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mX_features\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m     \u001b[0mpred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtest\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'y_class'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpred_prob\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpred\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mignore_index\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'LinearSVC' object has no attribute 'predict_proba'"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "svm_model = LinearSVC(max_iter=3000).fit(train[X_features],train['y_class'])\n",
    "svm_pred = Prediction(svm_model)    \n",
    "pd.crosstab(svm_pred['Y_pred'],svm_pred['Actual']).apply(lambda x: x/x.sum(), axis=1).round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3664376e-bb26-4ae3-9587-bf5cea8f856b",
   "metadata": {},
   "source": [
    "## (9) Neural Network Multi-class\n",
    "* sklearn.neural_network.MLPClassifier: [here](https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html#sklearn.neural_network.MLPClassifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87924a6e-a11c-458b-be7a-c1af53089dc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "NN_model = MLPClassifier(random_state=1, max_iter=300).fit(train[X_features],train['y_class'])\n",
    "NN_pred = Prediction(NN_model)    \n",
    "pd.crosstab(NN_pred['Y_pred'],NN_pred['Actual']).apply(lambda x: x/x.sum(), axis=1).round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0ee4ad0-88ca-4c84-981c-e840cbf10b41",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1dd2b59-9877-4acc-9a55-a37bded4e66c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}