Label-Shifter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding: utf8\n",
    "\n",
    "from label_shift.skwrapper import LabelShiftDetectorSKLearn\n",
    "\n",
    "from statsmodels.distributions.empirical_distribution import ECDF\n",
    "\n",
    "from sklearn import datasets\n",
    "from sklearn import metrics\n",
    "from sklearn import model_selection\n",
    "from sklearn import neural_network\n",
    "from sklearn import linear_model\n",
    "from sklearn import svm\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "plt.ion()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAABoCAYAAADYZ7pcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAJmklEQVR4nO3df6zVdR3H8dfbyy9B1KsoTNTEFB2ppd4w89fAZbj+kNYmaK3mamw2XU5qM0fTtqZubaYru0qpmFMp20hXCpg5tOkUCadSwABR6QoIqBFlcL3v/gDcHVzv+3PvPZ/z/Xy5z8c/3Hv3vp/Pe6977vsevjuf8zV3FwCgXAdV3QAAoHcMagAoHIMaAArHoAaAwjGoAaBwDGoAKNyQHIsOs+E+QqMGvE7nmHiNceO2hTX/3HF40n4jNuwKa3xXZ9JavflQO7TT/2f9+d5GZZu016nx3/HhB6Xl8f6m0WFNy9YdSWtFtuu9Le5+VF+/r5nZdh0e73PCcZuS1tq469CwZufKrqS1Iv3NVmpcvjvHx2ucduS7Yc22rpak/bauivfLPReSBrWZTZN0p6QWSb9299t6qx+hUTrHLu5zo/va8rVzw5ofzJ4f1vxo2WVJ+028/p2wpnNj2i9Pb170pz/+uKpsUxzzQDxcTx65OWmtP9w+NaxpnfdC0lqRP/vv39z7cV/ybWa2/5l6Tlhz7x23J6116zvTwpqOL2xPWivS32ylxuX7xrXxXHjpW+1hzfztrUn7PXjR5LCm0XNhX+FTJjNrkXSXpEslTZJ0hZlNGnBXINvMyDcfsm2ulGvUkyWtcfd17r5T0nxJaU9RESHbvMg3H7JtopRBPV7S290+37Dnaxg4ss2LfPMh2yZKuUbd08Xt/d4gxMxmSZolSSM0coBtDRpkm1eYL9n2G4/dJkp5Rr1B0nHdPj9WUse+Re4+193b3L1tqIY3qr8DHdnmFeZLtv3GY7eJUgb1Ukknm9kEMxsmaaakx/O2NWiQbV7kmw/ZNlF46cPdO83sGkmLtPtlOPe5+4rsnQ0CZJsX+eZDts2V9Dpqd39C0hOZe9lPymukZ45+L6y54/B/J+33p78tCmvOvvnqsGbM3PTXA1eVbYr1248Ia+4//rmktX514QVhTeu8pKX6pIp8uy46M6x57q57wprV8fkrSdJlRy4Pa9p1UtpifZAj29Xt8WuWb50az4XT7vxuWPP6936Z1NPPLzghrDnk0YG/jro3HCEHgMIxqAGgcAxqACgcgxoACsegBoDCMagBoHAMagAoHIMaAAqX5Q4vKTqnnh3WzBz9Slhz6bSZYc1hr65M6unyv8Zvar7tzI/CmjFJu1Ur5VDGPRN/kbBS2h07Dn1tWFLdgWDd9Pg9LW7ZckpYc+/TU5L2Wzvj7rAmfhv9Mpza/q+w5sEfx4di5ix5JKxJvXHAIY++mFSXE8+oAaBwDGoAKByDGgAKx6AGgMIxqAGgcAxqACgcgxoACsegBoDCVXbg5cMj463nbD49rOlKPMySYulrn27YWlV66+YvhjWPXfXTsGbi0LTDLCnGL94a1sRHierhlNvWhTW/fSs+XPXkdfHPSJKmrLgyrBmmN5PWqlrS7/MZp4YlKXd+unxd/DOQpCHj4lnVuZE7vADAoMagBoDCMagBoHAMagAoHIMaAArHoAaAwjGoAaBwDGoAKFx1B15a478RD71wblgzUS81oh1J0pDDdoY1nR+Uf6eS429+Pqy5rv2rYc0Tyxc3oh1J0q4xI8OaOjxraBl7dFiz6oYTw5pvX/x0I9qRJB38jf+GNQfKYSIp7VDMV876clhz5sKOtA0XxiXLpx0T1gzkUEwdfjcAYFBjUANA4RjUAFA4BjUAFI5BDQCFY1ADQOEY1ABQOAY1ABSOQQ0AhUs6mWhm6yVt1+4DTp3u3jbQjUe81xXWfP70tWHNBwl7DRk3NqFKmjFpWVjzuyfPT1orVY5sS7T5rIPDmnFLGr9vo/P9x63HhzVvTLt7IFt8bPKN30+qa930QkP266uSH7sppwBTThNK0tb7Roc1m246IqyZeHX/Tyb25Qj5FHff0u+d0BuyzYt88yHbJuDSBwAULnVQu6TFZrbMzGblbGgQItu8yDcfsm2S1Esf57l7h5kdLekpM1vp7s92L9jzg5olSSMUv1MaPka2efWaL9kOCI/dJkl6Ru3uHXv+3SxpgaTJPdTMdfc2d28bquGN7fIARrZ5RfmSbf/x2G2ecFCb2SgzG733Y0mXSHo9d2ODAdnmRb75kG1zpVz6GCtpgZntrX/Y3RPeShsJyDYv8s2HbJsoHNTuvk7SZ5vQy6BDtnmRbz5k21yV3Yrr0FXxUZWbjv1jWPPNWdeHNUOnv5vUU4oJP6zmcAHKcNID8U2tbmk7Jay5ccyqsOalW9qTepry9cvCmh0PxYc7WufV47G9un2/S+H7OeYvFtak3A5Qkn4z6fawZvr7Vyet1V+8jhoACsegBoDCMagBoHAMagAoHIMaAArHoAaAwjGoAaBwDGoAKFxlB166Xl0Z1sxonx3WzJn9SFhzx9qLk3pa+rmWpLoDwUebNoc1U1bEByme+cxjSft1np9wL56fJS1VqYOWLA9rlpwR383mmYuuCms652xL6inlZzDhwu+ENa3zkrar3ND349/Ta38yv2H7TX8+Psxy4pWvNGy/nvCMGgAKx6AGgMIxqAGgcAxqACgcgxoACsegBoDCMagBoHAMagAonLl74xc1e1fSm92+NEbSloZvlF+uvj/l7kf15xvJNkm/8u0hW6me+RaXrcRjN8EnZptlUO+3idnL7t6WfaMGq0PfdeixJ3Xpuy59dleXnuvS576q6JtLHwBQOAY1ABSuWYN6bpP2abQ69F2HHntSl77r0md3dem5Ln3uq+l9N+UaNQCg/7j0AQCFyz6ozWyama0yszVmdkPu/RrBzNab2Wtm9oqZvVx1P5+kjtlK9ciXbPOqY75VZpv10oeZtUhaLelLkjZIWirpCnf/e7ZNG8DM1ktqc/diX+NZ12yl8vMl27zqmm+V2eZ+Rj1Z0hp3X+fuOyXNlxTfNgQpyDYfss2LfPso96AeL+ntbp9v2PO10rmkxWa2zMxmVd3MJ6hrtlL5+ZJtXnXNt7Jsc98z0Xr4Wh1eZnKeu3eY2dGSnjKzle7+bNVN7aOu2Url50u2edU138qyzf2MeoOk47p9fqykjsx7Dpi7d+z5d7OkBdr9X7XS1DJbqRb5km1etcy3ymxzD+qlkk42swlmNkzSTEmPZ95zQMxslJmN3vuxpEskvV5tVz2qXbZSbfIl27xql2/V2Wa99OHunWZ2jaRFklok3efuK3Lu2QBjJS0wM2l3Pg+7+8JqW9pfTbOVapAv2eZV03wrzZaTiQBQOE4mAkDhGNQAUDgGNQAUjkENAIVjUANA4RjUAFA4BjUAFI5BDQCF+z+eTzI/Kc0wGQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "digits = datasets.load_digits()\n",
    "\n",
    "images_and_labels = list(zip(digits.images, digits.target))\n",
    "for index, (image, label) in enumerate(images_and_labels[:4]):\n",
    "    plt.subplot(2, 4, index + 1)\n",
    "    plt.imshow(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_samples = len(digits.images)\n",
    "data = digits.images.reshape((n_samples, -1))\n",
    "\n",
    "X = data\n",
    "y = digits.target\n",
    "X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y,\n",
    "                                                                    test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1437, 64)\n",
      "(360, 64)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ko_classes = {1, 2, 3, 4, 6, 7, 8, 9}\n",
    "ko_prob = 0.9\n",
    "\n",
    "keep = np.ones(len(y_train), dtype='bool')\n",
    "for i in range(len(y_train)):\n",
    "    if y_train[i] in ko_classes and np.random.rand() < ko_prob:\n",
    "        keep[i] = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2894919972164231\n"
     ]
    }
   ],
   "source": [
    "print(keep.sum() / keep.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train[keep]\n",
    "y_train = y_train[keep]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAOj0lEQVR4nO3dfYxldX3H8fenjE9gzKI7GNxdO2uzQampgUwoSmKMa1MQw/IHJhCLG0uzaYqKD42AJuUvE0yNT0lLsgV0TQlKVhqIWitBDGlSth1A5WG1bNAuI6s7RkGjSXXrt3/MoZmOd5mZe+bOZX/3/Uo2956He8932OU9Z8/cezdVhSSpLb837gEkSevPuEtSg4y7JDXIuEtSg4y7JDVoatwDAGzevLlmZmbGPYYknVDuv//+n1TV9KBtz4m4z8zMMDc3N+4xJOmEkuS/jrfNyzKS1CDjLkkNMu6S1CDjLkkNMu6S1CDjLkkNMu6S1CDjLkkNMu6S1KDnxDtU+5i55isjP8YPrr9w5MeQpPXkmbskNci4S1KDTvjLMtJG8jKgThSeuUtSg1aMe5KbkxxN8vCAbX+dpJJs7paT5DNJDiX5TpKzRzG0JOnZrebM/XPA+ctXJtkG/AlweMnqC4Ad3a89wA39R5QkrdWKca+qe4GfDtj0SeBDQC1Ztwv4fC26D9iU5PR1mVSStGpDXXNPchHww6r69rJNW4AnlizPd+sGPceeJHNJ5hYWFoYZQ5J0HGuOe5KTgY8AfzNo84B1NWAdVbW3qmaranZ6euA/AShJGtIwL4X8A2A78O0kAFuBB5Kcw+KZ+rYl+24Fnuw7pCRpbdZ85l5VD1XVaVU1U1UzLAb97Kr6EXAn8M7uVTPnAk9X1ZH1HVmStJLVvBTyVuDfgDOSzCe54ll2/yrwOHAI+Afgr9ZlSknSmqx4WaaqLlth+8yS+wVc2X8sSVIfvkNVkhpk3CWpQcZdkhpk3CWpQcZdkhpk3CWpQcZdkhpk3CWpQcZdkhpk3CWpQcZdkhpk3CWpQcZdkhpk3CWpQcZdkhpk3CWpQcZdkhpk3CWpQcZdkhq0mn8g++YkR5M8vGTd3yb5bpLvJPmnJJuWbLs2yaEk30vyp6MaXJJ0fKs5c/8ccP6ydXcBr62qPwL+E7gWIMmZwKXAH3aP+fskJ63btJKkVVkx7lV1L/DTZeu+XlXHusX7gK3d/V3AF6rqv6vq+8Ah4Jx1nFeStArrcc39z4F/7u5vAZ5Ysm2+WydJ2kC94p7kI8Ax4JZnVg3YrY7z2D1J5pLMLSws9BlDkrTM0HFPsht4G/COqnom4PPAtiW7bQWeHPT4qtpbVbNVNTs9PT3sGJKkAYaKe5LzgauBi6rqV0s23QlcmuQFSbYDO4B/7z+mJGktplbaIcmtwJuAzUnmgetYfHXMC4C7kgDcV1V/WVWPJLkNeJTFyzVXVtX/jGp4SdJgK8a9qi4bsPqmZ9n/o8BH+wwlSerHd6hKUoOMuyQ1yLhLUoOMuyQ1yLhLUoOMuyQ1yLhLUoOMuyQ1yLhLUoOMuyQ1yLhLUoOMuyQ1yLhLUoOMuyQ1yLhLUoOMuyQ1yLhLUoOMuyQ1yLhLUoOMuyQ1aMW4J7k5ydEkDy9Z99IkdyV5rLs9tVufJJ9JcijJd5KcPcrhJUmDrebM/XPA+cvWXQPcXVU7gLu7ZYALgB3drz3ADeszpiRpLVaMe1XdC/x02epdwL7u/j7g4iXrP1+L7gM2JTl9vYaVJK3OsNfcX15VRwC629O69VuAJ5bsN9+t+x1J9iSZSzK3sLAw5BiSpEHW+weqGbCuBu1YVXuraraqZqenp9d5DEmabMPG/cfPXG7pbo926+eBbUv22wo8Ofx4kqRhDBv3O4Hd3f3dwB1L1r+ze9XMucDTz1y+kSRtnKmVdkhyK/AmYHOSeeA64HrgtiRXAIeBt3e7fxV4K3AI+BXwrhHMLElawYpxr6rLjrNp54B9C7iy71CSpH58h6okNci4S1KDjLskNci4S1KDjLskNci4S1KDjLskNci4S1KDjLskNci4S1KDjLskNci4S1KDjLskNci4S1KDjLskNci4S1KDjLskNci4S1KDjLskNahX3JO8P8kjSR5OcmuSFybZnuRAkseSfDHJ89drWEnS6gwd9yRbgPcCs1X1WuAk4FLgY8Anq2oH8DPgivUYVJK0en0vy0wBL0oyBZwMHAHeDOzvtu8DLu55DEnSGg0d96r6IfBx4DCLUX8auB94qqqOdbvNA1sGPT7JniRzSeYWFhaGHUOSNECfyzKnAruA7cArgFOACwbsWoMeX1V7q2q2qmanp6eHHUOSNECfyzJvAb5fVQtV9RvgduANwKbuMg3AVuDJnjNKktaoT9wPA+cmOTlJgJ3Ao8A9wCXdPruBO/qNKElaqz7X3A+w+IPTB4CHuufaC1wNfCDJIeBlwE3rMKckaQ2mVt7l+KrqOuC6ZasfB87p87ySpH58h6okNci4S1KDjLskNci4S1KDjLskNci4S1KDjLskNci4S1KDjLskNci4S1KDjLskNci4S1KDjLskNci4S1KDjLskNci4S1KDjLskNci4S1KDjLskNci4S1KDesU9yaYk+5N8N8nBJK9P8tIkdyV5rLs9db2GlSStTt8z908DX6uqVwOvAw4C1wB3V9UO4O5uWZK0gYaOe5KXAG8EbgKoql9X1VPALmBft9s+4OK+Q0qS1qbPmfurgAXgs0keTHJjklOAl1fVEYDu9rRBD06yJ8lckrmFhYUeY0iSlusT9yngbOCGqjoL+CVruARTVXuraraqZqenp3uMIUlark/c54H5qjrQLe9nMfY/TnI6QHd7tN+IkqS1GjruVfUj4IkkZ3SrdgKPAncCu7t1u4E7ek0oSVqzqZ6Pfw9wS5LnA48D72LxG8ZtSa4ADgNv73kMSdIa9Yp7VX0LmB2waWef55Uk9eM7VCWpQcZdkhpk3CWpQcZdkhpk3CWpQcZdkhpk3CWpQcZdkhpk3CWpQcZdkhpk3CWpQcZdkhpk3CWpQcZdkhpk3CWpQcZdkhpk3CWpQcZdkhpk3CWpQb3jnuSkJA8m+XK3vD3JgSSPJfli949nS5I20HqcuV8FHFyy/DHgk1W1A/gZcMU6HEOStAa94p5kK3AhcGO3HODNwP5ul33AxX2OIUlau75n7p8CPgT8tlt+GfBUVR3rlueBLYMemGRPkrkkcwsLCz3HkCQtNXTck7wNOFpV9y9dPWDXGvT4qtpbVbNVNTs9PT3sGJKkAaZ6PPY84KIkbwVeCLyExTP5TUmmurP3rcCT/ceUJK3F0GfuVXVtVW2tqhngUuAbVfUO4B7gkm633cAdvaeUJK3JKF7nfjXwgSSHWLwGf9MIjiFJehZ9Lsv8n6r6JvDN7v7jwDnr8bySpOH4DlVJapBxl6QGGXdJapBxl6QGGXdJapBxl6QGGXdJapBxl6QGGXdJapBxl6QGGXdJapBxl6QGGXdJapBxl6QGGXdJapBxl6QGGXdJapBxl6QGGXdJatDQcU+yLck9SQ4meSTJVd36lya5K8lj3e2p6zeuJGk1+py5HwM+WFWvAc4FrkxyJnANcHdV7QDu7pYlSRtoatgHVtUR4Eh3/xdJDgJbgF3Am7rd9gHfBK7uNaW0xMw1Xxnp8//g+gtH+vzSRhg67kslmQHOAg4AL+/CT1UdSXLacR6zB9gD8MpXvnI9xpCaNupvauA3tpb0/oFqkhcDXwLeV1U/X+3jqmpvVc1W1ez09HTfMSRJS/SKe5LnsRj2W6rq9m71j5Oc3m0/HTjab0RJ0loNfVkmSYCbgINV9Yklm+4EdgPXd7d39JpQ0th5SejE0+ea+3nA5cBDSb7Vrfswi1G/LckVwGHg7f1GlCStVZ9Xy/wrkONs3jns80rSUv6tYTi+Q1WSGmTcJalBxl2SGrQub2KaVF4LlPRc5Zm7JDXIuEtSg7wscwIb5wdo+eFdmgQn8qVXz9wlqUHGXZIaZNwlqUHGXZIaZNwlqUHGXZIaZNwlqUHGXZIaZNwlqUHGXZIaZNwlqUHGXZIaZNwlqUEji3uS85N8L8mhJNeM6jiSpN81krgnOQn4O+AC4EzgsiRnjuJYkqTfNaoz93OAQ1X1eFX9GvgCsGtEx5IkLZOqWv8nTS4Bzq+qv+iWLwf+uKrevWSfPcCebvEM4HvrPsjxbQZ+soHHe67w654sft3t+/2qmh60YVT/ElMGrPt/30Wqai+wd0THf1ZJ5qpqdhzHHie/7sni1z3ZRnVZZh7YtmR5K/DkiI4lSVpmVHH/D2BHku1Jng9cCtw5omNJkpYZyWWZqjqW5N3AvwAnATdX1SOjONaQxnI56DnAr3uy+HVPsJH8QFWSNF6+Q1WSGmTcJalBExX3Sf1IhCTbktyT5GCSR5JcNe6ZNkqSk5I8mOTL455lIyXZlGR/ku92v++vH/dMGyHJ+7s/4w8nuTXJC8c907hMTNwn/CMRjgEfrKrXAOcCV07Q134VcHDcQ4zBp4GvVdWrgdcxAf8NkmwB3gvMVtVrWXwxx6XjnWp8JibuTPBHIlTVkap6oLv/Cxb/R98y3qlGL8lW4ELgxnHPspGSvAR4I3ATQFX9uqqeGu9UG2YKeFGSKeBkJvj9NZMU9y3AE0uW55mAwC2XZAY4Czgw3kk2xKeADwG/HfcgG+xVwALw2e6S1I1JThn3UKNWVT8EPg4cBo4AT1fV18c71fhMUtxX/EiE1iV5MfAl4H1V9fNxzzNKSd4GHK2q+8c9yxhMAWcDN1TVWcAvgeZ/xpTkVBb/Nr4deAVwSpI/G+9U4zNJcZ/oj0RI8jwWw35LVd0+7nk2wHnARUl+wOIluDcn+cfxjrRh5oH5qnrmb2f7WYx9694CfL+qFqrqN8DtwBvGPNPYTFLcJ/YjEZKExeuvB6vqE+OeZyNU1bVVtbWqZlj8vf5GVU3EWVxV/Qh4IskZ3aqdwKNjHGmjHAbOTXJy92d+JxPwg+TjGdWnQj7nnAAfiTBK5wGXAw8l+Va37sNV9dUxzqTReg9wS3ci8zjwrjHPM3JVdSDJfuABFl8h9iAT/FEEfvyAJDVoki7LSNLEMO6S1CDjLkkNMu6S1CDjLkkNMu6S1CDjLkkN+l9WJo+bV2vezwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "labels, count = np.unique(y_train, return_counts=True)\n",
    "plt.bar(labels, count);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.58      1.00      0.74        32\n",
      "           1       0.88      0.46      0.60        46\n",
      "           2       0.94      0.80      0.87        41\n",
      "           3       1.00      0.79      0.89        34\n",
      "           4       1.00      0.96      0.98        25\n",
      "           5       0.62      1.00      0.77        38\n",
      "           6       0.75      0.97      0.85        37\n",
      "           7       0.95      0.91      0.93        43\n",
      "           8       0.51      0.64      0.57        28\n",
      "           9       0.50      0.14      0.22        36\n",
      "\n",
      "    accuracy                           0.76       360\n",
      "   macro avg       0.77      0.77      0.74       360\n",
      "weighted avg       0.78      0.76      0.74       360\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# base = neural_network.MLPClassifier(hidden_layer_sizes=(128, 128))\n",
    "# base = svm.SVC(gamma=0.001)\n",
    "base = linear_model.LogisticRegressionCV()\n",
    "classifier = LabelShiftDetectorSKLearn(base)\n",
    "classifier.fit(X_train, y_train)\n",
    "y_pred = classifier.predict(X_test)\n",
    "print(metrics.classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f879bdd9278>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeMAAAHSCAYAAADfUaMwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd5hV1aH+8e9iBBFFUcGKiAWj2COxRBM1dkVQRJrIgIkmMSa56f7SrzfNVG8Sk1yMkd5RQBg10cQSY8Ma0RARGxZEikhnZtbvjz0kBAc4wJlZ5+zz/TzPPJyyPedlOzMva+991goxRiRJUjotUgeQJKnSWcaSJCVmGUuSlJhlLElSYpaxJEmJWcaSJCW2Xao3bt++fezcuXOqt5ckqVk9/vjj78QYOzT2XLIy7ty5MzNnzkz19pIkNasQwisbe87D1JIkJWYZS5KUmGUsSVJilrEkSYlZxpIkJWYZS5KUmGUsSVJilrEkSYlZxpIkJWYZS5KUmGUsSVJilrEkSYlZxpIkJWYZS5KU2GbLOITwhxDC2yGEZzfyfAgh/DKEMCeE8EwI4YPFjylJUn4VMjIeBpy7iefPA7o0fF0F/HbbY0mSVDk2W8YxxvuBRZvYpCcwImYeBtqFEPYuVsCtsWj5GtbW1aeMIEkqZ3VrYfErzfZ2xThnvC/w2nr35zU8lsxlv3+ET458PGUESVK5ql0DEwfDzWfByiXN8pbFKOPQyGOx0Q1DuCqEMDOEMHPBggVFeOv/VFtXzw13/5Pn31zKge13LPrrS5Jybu0qGD8Q/jEdPvIl2KFds7ztdkV4jXnAfuvd7wi80diGMcahwFCAbt26NVrYW6u2rp4Bv3+ER19axEXH7MPnz+xSzJeXJOXd2pUwbgC8+Gfo/gvodkWzvXUxRsbTgEENV1WfCLwbY3yzCK+7Rea+s5xHX1rEl88+hBv6HUvb1i2bO4IkqVytWQ6jL4UX/wI9b2zWIoYCRsYhhLHAaUD7EMI84DtAS4AY4++AGuB8YA6wAhjSVGE35Z1lqwE4er/mOaQgScqJ1e/B6D7w2sNw8f/B0X2bPcJmyzjG2H8zz0fgM0VLtJX++dZ7AHxgz7aJk0iSysbKJTC6N7z+BFxyMxzRK0mMYpwzLgkLl6+hqkWgXZtWqaNIksrBikUw8mKYPwv6DIfDLkwWJTdl/ML8Zey/WxtabecMn5KkzVi+EEb0hHdmQ99R8IFNzW3V9HJTxguXr2bPnVunjiFJKnXL3obhPWDxS9B/HBx8RupE+Vko4r1VtbRtnZt/W0iSmsLSN2HYBbDkFRgwoSSKGHI0Ms7K2I8zSZI24t15MPzCbGQ8cDLs/+HUif4lN2W8bHUtO21flTqGJKkULX4FhnfPrp6+/DbY7/jUif5Dbsp45Zo6dmiVm7+OJKlYFr6YnSNeswwGTYV9S2+l31y0V21dPWvq6mnTypGxJGk9C/4JI3pA3Rqovh32Pip1okblooxX12bLJW7vx5okSevMfy77+BIRqqfDnl1TJ9qoXLRXbX225kRVi8YWkJIkVZy3/p6dIw4tYHBNSRcx5KSM6xvKeDvLWJL0xpMwrDts1xqG1ECHQ1In2qxcHKb+18i4Khf/tpAkba3XHoNRl8AOu2TniHftnDpRQXLRXnXryjg4MpakivXKQzDyImizW3ZoukyKGHJSxrX12QVcHqaWpAr10v0wqhe03Ts7NN1uv9SJtkguyrihi72AS5Iq0Zx7YPSl0K4TDJ4BO++TOtEWy0UZ/2tkXGUZS1JF+ecfYWx/2L1LVsRt90ydaKvkoozXnTNu4TljSaocz0+HcQNgj8Ogehrs2D51oq2WjzKOfrRJkirKrNtgYjXsc0w2xWWb3VIn2ia5KOPaOif9kKSK8cwEmHQFdPwQDLwVdmiXOtE2y0UZ10cPU0tSRXhyNNx6Fex/Mlw2CVrvnDpRUeSkjLM/W+TibyNJatTMW2Dq1XDgaTBgAmy/U+pERZOL+lo3Mg6OjCUpnx4ZCtP/C7qcA/3HQas2qRMVVS7KOK4bGVvGkpQ/f/sV3PEVOLQ79B0FLVunTlR0uZibOq4bGSfOIUkqsgd+BvdcB10vgkt+D1UtUydqEvko44Y/HRlLUk7ECPddD/f+EI7sAxf9FqpyUVmNysXfrP5fk34kDiJJ2nYxZqPhv/4cjrkMevwKWlSlTtWk8lHG64bGlrEklbcY4Y/fhId+DccNgQt+XhEflclFGUf8nLEklb36erjza/DoUDj+k3De9VAhv9fzUcZeTS1J5a2+HmZ8AR4fBiddA2d/r2KKGHJSxv/+nHHiIJKkLVdfB9M+C0+Nho98CT72rYr7hZ6LMv73yDhtDknSFqqrhSmfgr9PhNO+Dqd+teKKGHJSxutGxl7BJUllpG4tTP4EPDcFzvh2NiquULko439/zjhpDElSoWpXw8QhMHsGnP19+PA1qRMllYsyliSVkbWrYMLl8MIf4byfwAlXpU6UnGUsSWo+a1bAuAEw917ofgN0G5I6UUmwjCVJzWP1MhjbD17+K/S8EY69LHWikmEZS5Ka3qqlMPpSmPco9LoJjro0daKSYhlLkprWyiUw6hJ48yno/Qc4/OLUiUqOZSxJajorFsHIi2H+LOgzAg69IHWikmQZS5KaxvJ3YERPeOcF6DcGDjk7daKSZRlLkorvvfkwogcsfhkGjIODPpY6UUmzjCVJxbX0DRh+YfbnZRPhgI+mTlTyLGNJUvEseS0r4uXvwMBbYf+TUicqC5axJKk4Fr+cFfHKd2HQFOjYLXWismEZS5K23cIXsyJeuwKqp8I+x6ZOVFbyUcZx85tIkprIgtkwvAfUr4Xq22GvI1MnKjv5KOMGoQLXwJSkpOY/l101TYDBM2CPw1InKkstUgeQJJWpN5+BYRdAi+1gSI1FvA0sY0nSlnv9iewcccs22Yi4fZfUicqaZSxJ2jKvPZrNrNV6l2xEvPtBqROVPctYklS4lx/M5presX1WxLvunzpRLljGkqTCzL0PRveGnfeBwTWwS8fUiXLDMpYkbd6cu2FMH9i1c3aOeOe9UyfKFctYkrRps++Esf2zi7Sqp8NOe6ROlDuWsSRp456/HcYPhD0Ph0HTYMfdUyfKJctYktS4ZyfDhGrY5xgYNBXa7JY6UW5ZxpKk93t6PEz+BOx3Alx+W/YxJjUZy1iS9J+eGAm3fRI6nwIDJ8H2bVMnyj3LWJL0b4/dDNOugYM+BgMmQKsdUyeqCLko4+iyTZK07R7+Hcz4IhxyLvQbAy13SJ2oYuSijNdxzSZJ2koP/hLu/Boc2h36jISWrVMnqii5WkJRkrQV7v8J/Pl7cHgv6DUUqlqmTlRxLGNJqlQxwr0/hPuuh6P6Qc8bocpaSMG9LkmVKEa4+7vw4A1w7EC48JfQoip1qoplGUtSpYkR7vo6PPwb6HYFnP8zaJGrS4jKjmUsSZWkvh7u+Ao89ns44VNw7o8gePlrapaxJFWK+nqY/nl4YgR8+HNw1nUWcYmwjCWpEtTXwdRr4Okx8NGvwOnfsIhLiGUsSXlXV5tNb/nspKyET/1q6kTagGUsSXlWtxYmXQHPT4MzvwunfCF1IjWioMvnQgjnhhBmhxDmhBCubeT5TiGEv4QQngwhPBNCOL/4USVJW6R2NUwYlBXxOT+wiEvYZss4hFAF3AicB3QF+ocQum6w2TeBCTHGY4F+wG+KHVSStAXWroRxl8HsGjj/p3DSZ1In0iYUMjI+HpgTY5wbY1wDjAN6brBNBHZuuL0L8EbxIkqStsiaFTC2H8y5O5vM4/grUyfSZhRyznhf4LX17s8DTthgm+8CfwwhfBbYETizKOkKFF20SZIyq5fBmL7w6t/got/AMQNSJ1IBChkZN3bt+4b11x8YFmPsCJwPjAwhvO+1QwhXhRBmhhBmLliwYMvTbi6oV+lLqmSrlsKoXvDqQ9DrJou4jBRSxvOA/da735H3H4b+ODABIMb4ENAaaL/hC8UYh8YYu8UYu3Xo0GHrEkuS3m/lYhh5Ebz+OFx6CxzZO3UibYFCyvgxoEsI4YAQQiuyC7SmbbDNq8AZACGEw8jKuPhDX0nS+61YBMN7wFt/z9Yi7rrhZT0qdZs9ZxxjrA0hXAPcBVQBf4gxzgohXAfMjDFOA74E3BRC+ALZIezBMXomV5Ka3LIFMKInLJwD/cZAl7NSJ9JWKGjSjxhjDVCzwWPfXu/2c8DJxY0mSdqk997KRsRLXoUB4+Gg01Mn0lZyBi5JKkdL34DhF8LSN2HgJOh8SupE2gaWsSSVmyWvZkW8fCFcfit0OjF1Im0jy1iSysmil7IiXrUUBk2FjselTqQisIwlqVy8Mycr4tqVUD0N9jkmdSIViWUsSeVgweysiOvroHo67HVE6kQqooJWbZIkJTR/FtzSsBje4BkWcQ5ZxpJUyt58GoZ1h6pWMLgG9jg0dSI1gVyUsdOLSMqleY9nh6Zb7QhDZkD7g1MnUhPJRRmvExpd00KSytCrj2Qza7VuB0NqYLcDUydSE8pVGUtSLrz8IIy8GHbaA4bcAe06pU6kJmYZS1IpmXsvjLoEdumYjYh32Td1IjUDy1iSSsULd8OYvtkh6cEzoO1eqROpmVjGklQKZt8B4/pD+0Ng8HTYyTXfK4llLEmpPTcVxg+EPY/IZtZqs1vqRGpmlrEkpfT3STBxCOx7HAyaAjvsmjqRErCMJSmVp8bCrVdmqy4NnAytd0mdSIlYxpKUwhMjYMqnofNH4LKJsH3b1ImUkGUsSc3tsd/DtM/CwWfAgPHZDFuqaJaxJDWnh38LM74Eh5wH/cZAyx1SJ1IJsIwlqbn89Qa481o4rAf0GQHbbZ86kUqE6xlLUnO478fwl+/DEZfAxUOhyl+/+rdcfDe4aJOkkhVjVsL3/wSO6gcX/QZaVKVOpRKTizJeJ7hok6RSEiPc/R148H/h2Mvhwv+1iNWoXJWxJJWMGOHO/weP/Ba6fRzO/ym08DIdNc4ylqRiq6+Hmi/DzJvhxKvhnB946E6bZBlLUjHV18Htn4cnR8LJn4cz/9si1mZZxpJULHW1MPUz8Mw4+OhX4fSvW8QqiGUsScVQtxZu+yQ8OxlO/yac+pXUiVRGLGNJ2la1a2DyFfD87XDWddnhaWkLWMaStC1qV8OEavjnHXDuj+DET6dOpDJkGUvS1lq7EsZdBi/eAxf8DD70idSJVKYsY0naGmuWw9h+8NID0ONX8MFBqROpjFnGkrSlVr8HY/rCqw/Bxb+Do/ulTqQyZxlL0pZY9S6M6g2vPw6X/D5b+EHaRpaxJBVq5WIY2QveegYuHQZde6ROpJzIRRnH6LpNkprY8oUwsicsmA19R8EHzkudSDmSizKWpCa1bAGM6AGL5kK/sdDlzNSJlDOWsSRtyntvwfAesORVGDAeDjwtdSLlkGUsSRvz7usw/EJYNh8GTobOJ6dOpJyyjCWpMYtfyYp45WIYeCt0OiF1IuWYZSxJG1o0Nzs0vXopDJoC+x6XOpFyzjKWpPW980JWxLWroPp22Pvo1IlUASxjSVrn7X9kh6aJMHg67Hl46kSqEC1SB5CkkvDWszDsAggtYPAMi1jNyjKWpDeeguHdYbvtYUgNdPhA6kSqMJaxpMo2b2Z2jrhV22xEvPtBqROpAlnGkirXqw/DiIugza4wZAbsdkDqRKpQlrGkyvTSA9miD233hCF3QLtOqROpglnGkirPi3+B0ZdCu/1gcA3svE/qRKpwuShj12ySVLAX/gRj+sJuB0L19GxkLCWWq88Zh5A6gaSS9o8amFgNexwGl0+BNrulTiQBORkZS9JmzZoCEy6HvY6EQdMsYpUUy1hS/j0zESZdAft2y0bEO7RLnUj6D5axpHx7agzceiV0OilbBrH1zqkTSe9jGUvKr8eHwZSr4cBT4bKJsP1OqRNJjbKMJeXTozfB7Z+Hg8+E/uOhVZvUiaSNsowl5c9DN0LNl+EDF0C/0dCydepE0ibl6qNNksQDP4d7/hu69oRLboaqlqkTSZtlGUvKhxjhvh/DvT+AI3rDxf8HVf6KU3nwO1VS+YsR/vw9eOCncPQA6PlraFGVOpVUMMtYUnmLEf70Lfjbr+CD1dD9Bmjh5TAqL5axpPIVI9x5LTzyO/jQlXDejy1ilSXLWFJ5qq+HGV+Ex2+BEz8D53zfCepVtnJRxtFlm6TKUl8H0z4HT42CU74AZ3zHIlZZy0UZrxPwh1HKvbpamHo1PDMeTr0WTrvWIlbZy1UZS8q5urXZPNOzboOPfQs++uXUiaSisIwllYfaNTBpCPxjOpz1P3Dy51InkorGMpZU+tauggmD4IW74Nzr4cRPpU4kFZVlLKm0rV0J4wbAi3+G7r+AblekTiQVnWUsqXStWQ5j+sLLf4WeN8KxA1MnkppEQZ+ODyGcG0KYHUKYE0K4diPb9AkhPBdCmBVCGFPcmJIqzur3YFRveOXBbJ5pi1g5ttmRcQihCrgROAuYBzwWQpgWY3xuvW26AP8PODnGuDiEsEdTBZZUAVYugdG94fUnspWXjuiVOpHUpAoZGR8PzIkxzo0xrgHGAT032OZK4MYY42KAGOPbxY0pqWKsWAQjL4I3noI+wy1iVYRCynhf4LX17s9reGx9hwCHhBAeDCE8HEI4t1gBJVWQ5QthRA+YPwv6joLDLkydSGoWhVzA1djUNhtOQLkd0AU4DegIPBBCOCLGuOQ/XiiEq4CrADp16rTFYSXl2LK3YXgPWPwS9B8HB5+ROpHUbAoZGc8D9lvvfkfgjUa2mRpjXBtjfAmYTVbO/yHGODTG2C3G2K1Dhw5bm1lS3ix9E4ZdAEtegQETLGJVnELK+DGgSwjhgBBCK6AfMG2DbaYApwOEENqTHbaeW8ygm+ZKEVLZenceDDsflr4BAyfDgaemTiQ1u82WcYyxFrgGuAt4HpgQY5wVQrguhNCjYbO7gIUhhOeAvwBfiTEubKrQG+Nc8VKZWfwK3HIeLH8HLr8N9v9w6kRSEgVN+hFjrAFqNnjs2+vdjsAXG74kafMWvpidI16zDAZNhX0/mDqRlIwzcElqfgv+mV01XbcGqm+HvY9KnUhKyjKW1LzmPwcjegIRqqfDnl1TJ5KSK2g6TEkqirf+DsO7Q2gBg2ssYqmBZSypebzxJAzrDtu1hiE10OGQ1ImkkuFhaklN77XHYNQlsMMu2TniXTunTiSVFEfGkprWKw9lc0232S07NG0RS+9jGUtqOi/dD6N6Qdu9s0PT7fbb/H8jVSDLWFLTmHMPjL4U2nWCwTNg531SJ5JKlmUsqfj++UcY2x9275IVcds9UyeSSpplLKm4np8O4wbAHodB9TTYsX3qRFLJs4wlFc+s22BiNex9dDbFZZvdUieSykIuyji6aJOU3jMTYNIV0PFD2aIPO7RLnUgqG7ko43VctUlK5MnRcOtVsP/JcNkkaL1z6kRSWclVGUtKYOYtMPVqOPA0GDABtt8pdSKp7FjGkrbeI0Nh+n9Bl3Og/zho1SZ1IqksWcaSts7ffgV3fAUO7Q59R0HL1qkTSWXLuaklbbkHfgb3XAddL4JLfg9VLVMnksqaZSypcDHCfdfDvT+EI/vARb+FKn+NSNvKnyJJhYkxGw3/9edwzGXQ41fQoip1KikXLGNJmxcj/PGb8NCv4bjBcMEvoIWXnEjFYhlL2rT6erjza/DoUDj+Kjjvx36oXyoyy1jSxtXXw4wvwOPD4KRr4OzvWcRSE7CMJTWuvg6mfRaeGg0f+RJ87FsWsdRELGNJ71dXC1M+BX+fCKd9HU79qkUsNSHLWNJ/qlsLkz8Bz02BM76djYolNalclLGLNklFUrsaJg6B2TPg7O/Dh69JnUiqCLko43UCHkaTttraVTDhcnjhj3DeT+CEq1InkipGrspY0lZaswLGDYC590L3G6DbkNSJpIpiGUuVbvUyGNsPXv4r9LwRjr0sdSKp4ljGUiVbtRRGXwrzHoVeN8FRl6ZOJFUky1iqVCuXwKhL4M2noPcf4PCLUyeSKpZlLFWiFYtg5MUwfxb0GQGHXpA6kVTRLGOp0ix/B0b0hHdegH5j4JCzUyeSKp5lLFWS9+bDiB6w+GUYMA4O+ljqRJKwjKXKsfQNGH5h9udlE+GAj6ZOJKmBZSxVgiWvZUW8/B0YeCvsf1LqRJLWYxlLebf45ayIV74Lg6ZAx26pE0nagGUs5dnCF7MiXrsCqqfCPsemTiSpEZaxlFcLZsPwHlC/Fqpvh72OTJ1I0kbkooyjyzZJ/2n+c9lV0wQYPAP2OCx1Ikmb0CJ1gGJy7XMJePMZGHYBtNgOhtRYxFIZyFUZSxXv9Seyc8Qt22Qj4vZdUieSVIBcHKaWBLz2aDbX9A7toHo67Lp/6kSSCuTIWMqDlx/M5presT0MucMilsqMZSyVu7n3wejesPM+MLgGdumYOpGkLWQZS+Vszt0wpg/s2jk7R7zz3qkTSdoKlrFUrmbfCWP7ZxdpVU+HnfZInUjSVrKMpXL0/O0wfiDseTgMmgY77p46kaRtYBlL5ebZyTChGvY5BgZNhTa7pU4kaRtZxlI5eXo8TP4E7HcCXH4btN4ldSJJRWAZS+XiiZFw2yeh8ykwcBJs3zZ1IklFYhlL5eCxm2HaNXDQ6TBgArTaMXUiSUWUizKOuFKEcuzh38GML0KXc6DfWGi5Q+pEkoosF2W8jutEKHce/CXc+TU4tDv0HQUtW6dOJKkJODe1VKru/wn8+XtweC/oNRSqWqZOJKmJWMZSqYkR7v0h3Hc9HNUPet4IVf6oSnnmT7hUSmKEu78LD94Axw6EC38JLapSp5LUxCxjqVTECHd9Ax6+EbpdAef/DFrk6rIOSRthGUuloL4e7vgqPHYTnPApOPdHELwkUaoUlrGUWn09TP88PDECPvw5OOs6i1iqMJaxlFJ9HUy9Bp4eAx/9Cpz+DYtYqkCWsZRKXW02veWzk7ISPvWrqRNJSsQyllKoWwuTroDnp8GZ34VTvpA6kaSELGOpudWuhomDYXYNnPMDOOkzqRNJSswylprT2pUw/nKY8yc4/6dw/JWpE0kqAZax1FzWrIBx/WHufdlkHsdVp04kqUTkooyjizap1K1eBmP6wqt/g4t+A8cMSJ1IUgnJRRmv4ydCVJJWLYXRl8K8x6DXTXBk79SJJJWYXJWxVHJWLoZRl8CbT8Olt0DXnqkTSSpBlrHUVFYsghE9YcE/oM9IOPT81IkklaiCZqEPIZwbQpgdQpgTQrh2E9v1DiHEEEK34kWUytCyBTCsOyyYDf3GWMSSNmmzZRxCqAJuBM4DugL9QwhdG9muLfA54JFih5TKyntvwbALYNFcGDAeupyVOpGkElfIyPh4YE6McW6McQ0wDmjsxNf/AD8GVhUxn1Relr6RFfG782DgJDjo9NSJJJWBQsp4X+C19e7Pa3jsX0IIxwL7xRinFzGbVF6WvAq3nAfvzYfLb4XOp6ROJKlMFHIBV2MfGPrXJ3tDCC2AXwCDN/tCIVwFXAXQqVOnwhJK5WDRSzD8wuxjTIOmQsfjUieSVEYKGRnPA/Zb735H4I317rcFjgDuDSG8DJwITGvsIq4Y49AYY7cYY7cOHTpsfWqplLwzB245H9Ysg+ppFrGkLVbIyPgxoEsI4QDgdaAf8K/pg2KM7wLt190PIdwLfDnGOLO4UaUStGB2NiKur4Pq6bDXEakTSSpDmx0ZxxhrgWuAu4DngQkxxlkhhOtCCD2aOqBUsubPykbEAINnWMSStlpBk37EGGuAmg0e+/ZGtj1t22NJJe7Np2HERbBda6i+HdofnDqRpDJW0KQfktYz7/Hs0HSrHWHIDItY0jbLRRm7aJOazauPZFNctm4HQ2pgtwNTJ5KUA7ko439z2SY1oZcfhJEXw057wJA7oJ0fz5NUHDkrY6mJzL03W31pl47ZiHiXfTf7n0hSoSxjaXNeuBvG9M0OSQ+eAW33Sp1IUs5YxtKmzL4DxvWH9l2yq6Z3crIaScVnGUsb89xUGD8Q9jwiK+Idd0+dSFJOWcZSY/4+CSYOgX2Pg0FTYIddUyeSlGOWsbShp8bCrVdCpxNh4GRovUvqRJJyzjKW1vfECJjyaej8EbhsImzfNnUiSRXAMpbWeez3MO2zcPAZMGB8NsOWJDUDy1gCePi3MONLcMh50G8MtNwhdSJJFcQylv56A9x5LRzWA/qMgO22T51IUoUpaNUmKbfu+zH85ftwxCVw8VCo8kdCUvPzN48qU4xZCd//EziqH1z0G2hRlTqVpAqVizKO0XWbtAVihLu/Aw/+Lxx7OVz4vxaxpKRyUcbrBBdt0ubECHf+P3jkt9Dt43D+T6GFl05ISitXZSxtUn091HwZZt4MJ14N5/zAf8FJKgmWsSpDfR3c/nl4ciSc/Hk4878tYkklwzJW/tXXwZSr4Zlx8NGvwulft4gllRTLWPlWtxZu+yQ8OxlO/yac+pXUiSTpfSxj5VftGph8BTx/O5x1XXZ4WpJKkGWsfKpdDROq4Z93wLk/ghM/nTqRJG2UZaz8WbsSxl0GL94DF/wMPvSJ1IkkaZMsY+XLmuUwth+89AD0+BV8cFDqRJK0WZax8mP1ezCmL7z6EFz8Ozi6X+pEklQQy1j5sOpdGNUbXn8cLvl9tvCDJJUJy1jlb+ViGNkL3noGLh0GXXukTiRJW8QyVnlbvhBG9oQFs6HvKPjAeakTSdIWs4xVvpYtgBE9YNFc6DcWupyZOpEkbZVclbETHFaQ996C4T1gyaswYDwceFrqRJK01XJVxqoQ774Owy+EZfNh4GTofHLqRJK0TSxjlZfFr2RFvHIxDLwVOp2QOpEkbTPLWOVj0dzs0PTqpTBoCux7XOpEklQUlrHKwzsvZEVcuwqqb4e9j06dSJKKxjJW6Xv7H9mhaSIMng57Hp46kSQVlWWs0vbWszCiJ7Sogurp0OEDqRNJUtG1SB1A2qg3noLh3aGqFQyusYgl5ZZlrNI0b2Z2jrhVWxhSA+0PTp1IkpqMZazS8+rDMOIiaLMrDJkBux2QOpEkNSnLWKXlpQeyRR/a7glD7oB2nVInkqQmZxmrdLz4Fxh9KbTbLztHvPM+qcWgI6gAABD5SURBVBNJUrPIRRnHmDqBttkLf4IxfWG3A7OrptvumTqRJDWbXH20KQSXiihL/6iBidWwx2Fw+RRos1vqRJLUrHIxMlYZmzUFJlwOex0Jg6ZZxJIqkmWsdJ6ZCJOugH27ZSPiHdqlTiRJSVjGSuOpMXDrldDppGwZxNY7p04kSclYxmp+jw+DKVfDgafCZRNh+51SJ5KkpCxjNa9Hb4LbPw8Hnwn9x0OrNqkTSVJylrGaz0M3Qs2X4QMXQL/R0LJ16kSSVBJy9dEmlbAHfg73/Dd07QmX3AxVLVMnkqSSYRmr6d17Pdz7AziiN1z8f1Dlt50krc/fimo6McKfvwcP/BSOHgA9f52tSyxJ+g+WsZpGjPCnb8HffgUfrIbuN0ALL1GQpMZYxiq+GOHOa+GR38GHroTzfmwRS9ImWMYqrvp6mPFFePwWOPEzcM73wTnDJWmTclHGEZdtKgn1dTDtc/DUKDjlC3DGdyxiSSpALsp4HX/tJ1RXC1OvhmfGw6nXwmnXWsSSVKBclbESqVubzTM96zb42Lfgo19OnUiSyoplrG1TuwYmDYF/TIez/gdO/lzqRJJUdixjbb21q2DCIHjhLjj3ejjxU6kTSVJZsoy1ddauhHED4MU/Q/dfQLcrUieSpLJlGWvLrVkOY/rCy3+FnjfCsQNTJ5KksmYZa8usfg9G94HXHs7mmT66b+pEklT2LGMVbuUSGN0bXn8iW3npiF6pE0lSLljGKsyKRTCqF7z1LPQZDoddmDqRJOWGZazNW74QRvaEBbOh7yj4wLmpE0lSrljG2rRlb8PwHrD4Jeg/Dg4+I3UiScody1gbt/RNGNED3p0HAybAgaemTiRJuWQZq3HvzoPhF2Yj44GTYf8Pp04kSblV0CKzIYRzQwizQwhzQgjXNvL8F0MIz4UQngkh3BNC2L/4UTcuumhTcS1+BW45D5a/A5ffZhFLUhPbbBmHEKqAG4HzgK5A/xBC1w02exLoFmM8CpgE/LjYQQvhIkFFsPBFuOV8WLUUBk2F/Y5PnUiScq+QkfHxwJwY49wY4xpgHNBz/Q1ijH+JMa5ouPsw0LG4MdUsFvwThl0AtSuh+nbY94OpE0lSRSikjPcFXlvv/ryGxzbm48Ad2xJKCcx/Livi+lqong57H5U6kSRVjEIu4Grs4G+jZ2lDCAOBbkCjl92GEK4CrgLo1KlTgRHV5N76O4zoCS1aZiPiDoekTiRJFaWQkfE8YL/17ncE3thwoxDCmcA3gB4xxtWNvVCMcWiMsVuMsVuHDh22Jq+K7Y0nYVh32K41DKmxiCUpgULK+DGgSwjhgBBCK6AfMG39DUIIxwL/R1bEbxc/pprEa4/B8J7QeuesiHc/KHUiSapImy3jGGMtcA1wF/A8MCHGOCuEcF0IoUfDZj8BdgImhhCeCiFM28jLqVS88hCMvAja7AaDa2DXzqkTSVLFKmjSjxhjDVCzwWPfXu/2mUXOpab00v3ZesQ77wvV02DnfVInkqSKVtCkH8qROffA6EuhXScYPMMilqQSYBlXkn/+Ecb2h927ZEXcds/UiSRJWMaV4/npMG4A7HFYdmh6x/apE0mSGljGlWDWbTCxGvY+Opviss1uqRNJktZjGefdMxNg0hXQ8UPZog87tEudSJK0gVyUsas2bcSTo+HWq2D/k+GySdnniSVJJScXZbxOaHTmzgo18xaYejUceBoMmADb75Q6kSRpI3JVxmrwyFCY/l/Q5RzoPw5atUmdSJK0CZZx3vztV3DHV+DQ7tB3FLRsnTqRJGkzCpqBS2XigZ/BPddB14vgkt9DVcvUiSRJBbCM8yBGuO96uPeHcGQfuOi3UOX/WkkqF/7GLncxZqPhv/4cjrkMevwKWlSlTiVJ2gKWcTmLEf74TXjo13DcYLjgF9DCywAkqdxYxuWqvh7u/Bo8OhSOvwrO+zEEP9olSeXIMi5H9fUw4wvw+DA46Ro4+3sWsSSVMcu43NTXwbTPwlOj4SNfgo99yyKWpDJnGZeTulqY8in4+0Q47etw6lctYknKAcu4XNSthcmfgOemwBnfzkbFkqRcyEUZ536diNrVMHEIzJ4BZ38fPnxN6kSSpCLKRRmvk8sjtmtXwYTL4YU/wnk/gROuSp1IklRkuSrj3FmzAsYNgLn3QvcboNuQ1IkkSU3AMi5Vq5fB2H7w8l+h541w7GWpE0mSmohlXIpWLYXRl8K8R6HXUDiqT+pEkqQmZBmXmpVLYNQl8OZT0PsPcPjFqRNJkpqYZVxKViyCkRfD/FnQZwQcekHqRJKkZmAZl4rl78CInvDOC9BvDBxydupEkqRmYhmXgvfmw4gesPhlGDAODvpY6kSSpGZkGae29A0YfmH252UT4YCPpk4kSWpmlnFKS17Linj5OzDwVtj/pNSJJEkJWMapLH45K+KV78KgKdCxW+pEkqRELOMUFr6YFfGa5VA9FfY5NnUiSVJClnFzWzAbhveA+rUweDrsdWTqRJKkxHJRxjGWybpN85/LrpomwOAZsMdhqRNJkkpAi9QBKsabz8CwC6DFdjCkxiKWJP2LZdwcXn8iO0fcsk02Im7fJXUiSVIJycVh6pL22qPZXNM7tIPq6bDr/qkTSZJKjCPjpvTK37K5pndsD0PusIglSY2yjJvK3PuyEfHO+8DgGtilY+pEkqQSZRk3hTl3w5g+sGvn7BzxznunTiRJKmGWcbHNvhPG9s8u0qqeDjvtkTqRJKnEWcbF9PztMH4g7Hk4DJoGO+6eOpEkqQxYxsXy7GSYUA37HAODpkKb3VInkiSVCcu4GJ4eD5M/AfudAJffBq13SZ1IklRGLONt9cRIuO2T0PkUGDgJtm+bOpEkqcxYxtvisZth2jVw0OkwYAK02jF1IklSGbKMt9bDv4MZX4Qu50C/sdByh9SJJEllKhdl3OxrNj34S7jza3Bod+g7Clq2bu4EkqQcydXc1CE0w5vc/xP48/fg8F7QayhUtWyGN5Uk5VmuyrhJxQj3/hDuux6O6gc9b4Qqd58kadvZJoWIEe7+Ljx4Axw7EC78JbSoSp1KkpQTlvHmxAh3fQMevhG6XQHn/wxa5OJUuySpRFjGm1JfD3d8FR67CU74FJz7o2Y6MS1JqiSW8cbU18P0z8MTI+DDn4OzrrOIJUlNwjJuTH0dTL0Gnh4DH/kyfOybFrEkqclYxhuqq82mt3x2Epz+DTj1q6kTSZJyzjJeX91amHQFPD8NzvwunPKF1IkkSRXAMl6ndjVMHAyza+CcH8BJn0mdSJJUISxjgLUrYfzlMOdPcP5P4fgrUyeSJFUQy3jNChjXH+bel03mcVx16kSSpApT2WW8ehmM6Quv/g0u+g0cMyB1IklSBcpHGW/Nsk2rlsLoS2HeY9DrJjiyd9FjSZJUiHyUcYNQ6GeBVy6GUZfAm0/DpbdA155NG0ySpE3IVRkXZMUiGNET3n4e+oyEQ89PnUiSVOEqq4yXLciKeOEc6D8WupyVOpEkSRVUxu+9BcN7wJJXYcB4OOj01IkkSQIqpYyXvgHDL4Slb8LASdD5lNSJJEn6l/yX8ZJXsyJevhAuvxU6nZg6kSRJ/yHfZbzopayIVy2FQVOh43GpE0mS9D75LeOFL8Kw7lC7EqqnwT7HpE4kSVKj8lnGC2ZnI+L6OqieDnsdkTqRJEkb1aKQjUII54YQZocQ5oQQrm3k+e1DCOMbnn8khNC52EELNn8W3NLw2eHBMyxiSVLJ22wZhxCqgBuB84CuQP8QQtcNNvs4sDjGeDDwC+D6YgctRMu3/54dmq5qBYNrYI9DU8SQJGmLFDIyPh6YE2OcG2NcA4wDNpw/sicwvOH2JOCMUPDclMVxdJhD+8m9odWOMGQGtD+4Od9ekqStVkgZ7wu8tt79eQ2PNbpNjLEWeBfYvRgBC7H7oicZ1eqH1G+/Cwypgd0ObK63liRpmxVSxo2NcDdcJ6mQbQghXBVCmBlCmLlgwYJC8hXkjIPb0navg9ju43dCu05Fe11JkppDIWU8D9hvvfsdgTc2tk0IYTtgF2DRhi8UYxwaY+wWY+zWoUOHrUvcmINOh0/eD7tsOGCXJKn0FVLGjwFdQggHhBBaAf2AaRtsMw2obrjdG/hzjHFrVhneei2qmvXtJEkqls1+zjjGWBtCuAa4C6gC/hBjnBVCuA6YGWOcBtwMjAwhzCEbEfdrytCSJOVJQZN+xBhrgJoNHvv2erdXAZcWN5okSZWhoEk/JElS07GMJUlKzDKWJCkxy1iSpMQsY0mSErOMJUlKzDKWJCkxy1iSpMQsY0mSErOMJUlKzDKWJCkxy1iSpMQsY0mSErOMJUlKzDKWJCmxEGNM88YhLABeKeJLtgfeKeLrVSr347ZzH2479+G2cx9uu2Lvw/1jjB0aeyJZGRdbCGFmjLFb6hzlzv247dyH2859uO3ch9uuOfehh6klSUrMMpYkKbE8lfHQ1AFywv247dyH2859uO3ch9uu2fZhbs4ZS5JUrvI0MpYkqSyVXRmHEM4NIcwOIcwJIVzbyPPbhxDGNzz/SAihc/OnLG0F7MMvhhCeCyE8E0K4J4Swf4qcpWxz+3C97XqHEGIIwataG1HIfgwh9Gn4fpwVQhjT3BlLXQE/z51CCH8JITzZ8DN9foqcpSqE8IcQwtshhGc38nwIIfyyYf8+E0L4YJMEiTGWzRdQBbwIHAi0Ap4Gum6wzdXA7xpu9wPGp85dSl8F7sPTgTYNtz/tPtzyfdiwXVvgfuBhoFvq3KX2VeD3YhfgSWDXhvt7pM5dSl8F7sOhwKcbbncFXk6du5S+gI8CHwSe3cjz5wN3AAE4EXikKXKU28j4eGBOjHFujHENMA7oucE2PYHhDbcnAWeEEEIzZix1m92HMca/xBhXNNx9GOjYzBlLXSHfhwD/A/wYWNWc4cpIIfvxSuDGGONigBjj282csdQVsg8jsHPD7V2AN5oxX8mLMd4PLNrEJj2BETHzMNAuhLB3sXOUWxnvC7y23v15DY81uk2MsRZ4F9i9WdKVh0L24fo+TvavQv3bZvdhCOFYYL8Y4/TmDFZmCvlePAQ4JITwYAjh4RDCuc2WrjwUsg+/CwwMIcwDaoDPNk+03NjS35lbZbtiv2ATa2yEu+Hl4IVsU8kK3j8hhIFAN+DUJk1Ufja5D0MILYBfAIObK1CZKuR7cTuyQ9WnkR2heSCEcESMcUkTZysXhezD/sCwGOPPQggnASMb9mF908fLhWbplHIbGc8D9lvvfkfef8jlX9uEELYjOyyzqUMQlaaQfUgI4UzgG0CPGOPqZspWLja3D9sCRwD3hhBeJjvPNM2LuN6n0J/nqTHGtTHGl4DZZOWsTCH78OPABIAY40NAa7I5l1WYgn5nbqtyK+PHgC4hhANCCK3ILtCatsE204Dqhtu9gT/HhrPwAgrYhw2HWP+PrIg9R/d+m9yHMcZ3Y4ztY4ydY4ydyc6794gxzkwTt2QV8vM8heyCQkII7ckOW89t1pSlrZB9+CpwBkAI4TCyMl7QrCnL2zRgUMNV1ScC78YY3yz2m5TVYeoYY20I4RrgLrKrCP8QY5wVQrgOmBljnAbcTHYYZg7ZiLhfusSlp8B9+BNgJ2Biw7Vvr8YYeyQLXWIK3IfajAL3413A2SGE54A64CsxxoXpUpeWAvfhl4CbQghfIDu8OtgByr+FEMaSnQZp33Be/TtAS4AY4+/IzrOfD8wBVgBDmiSH/08kSUqr3A5TS5KUO5axJEmJWcaSJCVmGUuSlJhlLElSYpaxJEmJWcaSJCVmGUuSlNj/B40ceF5CkoDUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 8))\n",
    "p, pvals = classifier.label_shift_detector(X_test, return_bootstrap=True)\n",
    "ecdf = ECDF(pvals)\n",
    "plt.plot(ecdf.x, ecdf.y)\n",
    "plt.plot(ecdf.y, ecdf.y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.5755259056400494e-05 67.03379854533202 1.8120352283989751\n"
     ]
    }
   ],
   "source": [
    "p, nrm, kld = classifier.label_shift_detector(X_test, y_test)\n",
    "print(p, nrm, kld)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.58      1.00      0.74        32\n",
      "           1       0.88      0.46      0.60        46\n",
      "           2       0.94      0.80      0.87        41\n",
      "           3       1.00      0.79      0.89        34\n",
      "           4       1.00      0.96      0.98        25\n",
      "           5       0.62      1.00      0.77        38\n",
      "           6       0.75      0.97      0.85        37\n",
      "           7       0.95      0.91      0.93        43\n",
      "           8       0.51      0.64      0.57        28\n",
      "           9       0.50      0.14      0.22        36\n",
      "\n",
      "    accuracy                           0.76       360\n",
      "   macro avg       0.77      0.77      0.74       360\n",
      "weighted avg       0.78      0.76      0.74       360\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_pred = classifier.predict(X_test)\n",
    "print(metrics.classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      1.00      0.96        32\n",
      "           1       0.94      0.63      0.75        46\n",
      "           2       0.98      0.98      0.98        41\n",
      "           3       1.00      0.74      0.85        34\n",
      "           4       0.89      0.96      0.92        25\n",
      "           5       0.64      1.00      0.78        38\n",
      "           6       0.82      0.97      0.89        37\n",
      "           7       0.98      0.93      0.95        43\n",
      "           8       0.73      0.68      0.70        28\n",
      "           9       0.94      0.81      0.87        36\n",
      "\n",
      "    accuracy                           0.87       360\n",
      "   macro avg       0.88      0.87      0.86       360\n",
      "weighted avg       0.89      0.87      0.87       360\n",
      "\n"
     ]
    }
   ],
   "source": [
    "weights = classifier.wt_est_[:, 0].copy()\n",
    "weights = weights / weights.sum()\n",
    "class_weights = {}\n",
    "for k in range(len(weights)):\n",
    "    class_weights[k] = weights[k]\n",
    "\n",
    "new_classifier = linear_model.LogisticRegressionCV(class_weight=class_weights)\n",
    "new_classifier.fit(X_train, y_train)\n",
    "y_pred = new_classifier.predict(X_test)\n",
    "print(metrics.classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}