tfm/multiclass_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3W dataset's General Presentation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a general presentation of the 3W dataset, to the best of its authors' knowledge, the first realistic and public dataset with rare undesirable real events in oil wells that can be readily used as a benchmark dataset for development of machine learning techniques related to inherent difficulties of actual data.\n",
    "\n",
    "For more information about the theory behind this dataset, refer to the paper **A Realistic and Public Dataset with Rare Undesirable Real Events in Oil Wells** published in the **Journal of Petroleum Science and Engineering** (link [here](https://doi.org/10.1016/j.petrol.2019.106223))."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Introduction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This Jupyter Notebook presents the 3W dataset in a general way. For this, some tables, graphs, and statistics are presented."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Imports and Configurations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import pathlib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import re\n",
    "import pickle\n",
    "import bisect\n",
    "import sklearn\n",
    "import sklearn.model_selection\n",
    "from sklearn.linear_model import SGDClassifier, PassiveAggressiveClassifier\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.metrics import classification_report, ConfusionMatrixDisplay, multilabel_confusion_matrix, f1_score\n",
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'svg'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Instances' Structure"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below, all 3W dataset's instances are loaded and the first one of each knowledge source (real, simulated and hand-drawn) is partially displayed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class d3w():\n",
    "    '''\n",
    "    Class for managing Petrobras 3W dataset\n",
    "    '''\n",
    "    def __init__(self, path3w):\n",
    "        self.path3w = path3w\n",
    "        self.df = self.__load_df()\n",
    "        return\n",
    "\n",
    "    def __load_df(self):\n",
    "\n",
    "        d = dict()\n",
    "        d['origin'] = []\n",
    "        d['label'] = []\n",
    "        d['path'] = []\n",
    "        d['nlines'] = []\n",
    "        for i in pathlib.Path(self.path3w).iterdir():\n",
    "            if i.stem.isnumeric():\n",
    "                print(i)\n",
    "                label = int(i.stem)\n",
    "                for fp in i.iterdir():\n",
    "                    # Considers only csv files\n",
    "                    if fp.suffix == \".csv\":\n",
    "\n",
    "                        if (fp.stem.startswith(\"SIMULATED\")):\n",
    "                            d['origin'].append('S')\n",
    "                        elif fp.stem.startswith(\"DRAWN\"):\n",
    "                            d['origin'].append('D')\n",
    "                        else:\n",
    "                            d['origin'].append('R')\n",
    "                        \n",
    "                        d['label'].append(label)\n",
    "                        d['path'].append(fp)\n",
    "                        d['nlines'].append(self.file_len(fp)-1)\n",
    "                        \n",
    "        return pd.DataFrame(d)\n",
    "    \n",
    "    def split(self, real=True, simul=True, drawn=True, test_size=0.2, val_size=0.1, sample_n=None):\n",
    "        \n",
    "        tmp0_df = self.get_df(real, simul, drawn)\n",
    "        \n",
    "        if sample_n is not None:\n",
    "            N = len(tmp0_df.index)\n",
    "            if N > sample_n:\n",
    "                ds_list = []\n",
    "                for i, ni in tmp0_df.groupby('label').count().nlines.items():\n",
    "                    ns = ni*sample_n//N\n",
    "                    ds_list.append(tmp0_df[tmp0_df.label == i].sample(n=ns, random_state=200560))\n",
    "                tmp0_df = pd.concat(ds_list)            \n",
    "        \n",
    "        tmp_df, test_df = sklearn.model_selection.train_test_split(tmp0_df, \n",
    "                                                        test_size=test_size, \n",
    "                                                        random_state=200560, \n",
    "                                                        shuffle=True, \n",
    "                                                        stratify=tmp0_df['label'])\n",
    "        \n",
    "        if val_size == 0:\n",
    "            print('Instances Train: {}  Test: {}'.format(len(tmp_df.index), \n",
    "                                                         len(test_df.index)))\n",
    "            return tmp_df.reset_index(drop=True),\\\n",
    "                   test_df.reset_index(drop=True)\n",
    "        \n",
    "        train_df, val_df = sklearn.model_selection.train_test_split(tmp_df, test_size=val_size, \n",
    "                                                        random_state=200560, \n",
    "                                                        shuffle=True, \n",
    "                                                        stratify=tmp_df['label'])\n",
    "        print('Instances Train: {}  Test: {}  Validation: {}'.format(len(train_df.index), \n",
    "                                                                     len(test_df.index), \n",
    "                                                                     len(val_df.index)))\n",
    "        \n",
    "        return train_df.reset_index(drop=True),\\\n",
    "               test_df.reset_index(drop=True),\\\n",
    "               val_df.reset_index(drop=True)\n",
    "    \n",
    "    def file_len(self, filename):\n",
    "        j = 0\n",
    "        with open(filename) as f:\n",
    "            for i, x in enumerate(f):\n",
    "                if x.strip() == '':\n",
    "                    j += 1\n",
    "        return i + 1 - j\n",
    "    \n",
    "    def get_df(self, real=True, simul=True, drawn=True):\n",
    "        sel = []\n",
    "        if real:\n",
    "            sel.append('R')\n",
    "        if simul:\n",
    "            sel.append('S')\n",
    "        if drawn:\n",
    "            sel.append('D')\n",
    "        if sel:\n",
    "            return self.df[self.df['origin'].isin(sel)].drop(columns=['origin']).reset_index(drop=True)\n",
    "    \n",
    "    @property\n",
    "    def all(self):\n",
    "        return self.df.drop(columns=['origin'])\n",
    "    @property\n",
    "    def real(self):\n",
    "        return self.df[self.df['origin']=='R'].drop(columns=['origin']).reset_index(drop=True)\n",
    "    @property\n",
    "    def simul(self):\n",
    "        return self.df[self.df['origin']=='S'].drop(columns=['origin']).reset_index(drop=True)\n",
    "    @property\n",
    "    def drawn(self):\n",
    "        return self.df[self.df['origin']=='D'].drop(columns=['origin']).reset_index(drop=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "if pathlib.Path('dset_sgd.pkl').exists():\n",
    "  with open('dset_sgd.pkl', 'rb') as f:\n",
    "    dset = pickle.load(f)\n",
    "else:\n",
    "    dset = d3w('../dataset')\n",
    "    with open('dset_sgd.pkl', 'wb') as f:\n",
    "      pickle.dump(dset, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Instances Train: 201  Test: 60  Validation: 36\n"
     ]
    }
   ],
   "source": [
    "flist0 = ['P-PDG', 'P-TPT', 'T-TPT', 'P-MON-CKP', 'T-JUS-CKP', 'P-JUS-CKGL', 'T-JUS-CKGL', 'QGL']\n",
    "categories=[0,1,2,3,4,5,6,7,8,101,102,103,104,105,106,107,108]\n",
    "\n",
    "train_df, test_df, val_df = dset.split(real=True, simul=True, drawn=True, test_size=0.2, val_size=0.15, sample_n=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Each instance is stored in a CSV file and loaded into a pandas DataFrame. Each observation is stored in a line in the CSV file and loaded as a line in the pandas DataFrame. The first line of each CSV file contains a header with column identifiers. Each column of CSV files stores the following type of information:\n",
    "\n",
    "* **timestamp**: observations timestamps loaded into pandas DataFrame as its index;\n",
    "* **P-PDG**: pressure variable at the Permanent Downhole Gauge (PDG);\n",
    "* **P-TPT**: pressure variable at the Temperature and Pressure Transducer (TPT);\n",
    "* **T-TPT**: temperature variable at the Temperature and Pressure Transducer (TPT);\n",
    "* **P-MON-CKP**: pressure variable upstream of the production choke (CKP);\n",
    "* **T-JUS-CKP**: temperature variable downstream of the production choke (CKP);\n",
    "* **P-JUS-CKGL**: pressure variable upstream of the gas lift choke (CKGL);\n",
    "* **T-JUS-CKGL**: temperature variable upstream of the gas lift choke (CKGL);\n",
    "* **QGL**: gas lift flow rate;\n",
    "* **class**: observations labels associated with three types of periods (normal, fault transient, and faulty steady state).\n",
    "\n",
    "Other information are also loaded into each pandas Dataframe:\n",
    "\n",
    "* **label**: instance label (event type);\n",
    "* **well**: well name. Hand-drawn and simulated instances have fixed names. Real instances have names masked with incremental id;\n",
    "* **id**: instance identifier. Hand-drawn and simulated instances have incremental id. Each real instance has an id generated from its first timestamp.\n",
    "\n",
    "More information about these variables can be obtained from the following publicly available documents:\n",
    "\n",
    "* ***Option in Portuguese***: R.E.V. Vargas. Base de dados e benchmarks para prognóstico de anomalias em sistemas de elevação de petróleo. Universidade Federal do Espírito Santo. Doctoral thesis. 2019. https://github.com/ricardovvargas/3w_dataset/raw/master/docs/doctoral_thesis_ricardo_vargas.pdf.\n",
    "* ***Option in English***: B.G. Carvalho. Evaluating machine learning techniques for detection of flow instability events in offshore oil wells. Universidade Federal do Espírito Santo. Master's degree dissertation. 2021. https://github.com/ricardovvargas/3w_dataset/raw/master/docs/master_degree_dissertation_bruno_carvalho.pdf."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following table shows the amount of instances that compose the 3W dataset, by knowledge source (real, simulated and hand-drawn instances) and by instance label."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomDataGen():\n",
    "    '''\n",
    "    Generator for Keras models of 3W dataset based on:\n",
    "    https://medium.com/analytics-vidhya/write-your-own-custom-data-generator-for-tensorflow-keras-1252b64e41c3\n",
    "    \n",
    "    '''\n",
    "    \n",
    "    def __init__(self, df, X_col, y_col, categories,\n",
    "                 batch_size,\n",
    "                 seq_length=15,\n",
    "                 tmp_path='/tmp'):\n",
    "        \n",
    "        self.df = df.copy().reset_index()\n",
    "        self.X_col = X_col\n",
    "        self.y_col = y_col\n",
    "        self.categories = categories\n",
    "        self.batch_size = batch_size\n",
    "        self.seq_length = seq_length\n",
    "        self.tmp_path = pathlib.Path(tmp_path)\n",
    "        self.file = None\n",
    "        self.dt = None\n",
    "        \n",
    "        self.n = self.__calc_n()\n",
    "    \n",
    "    def __calc_n(self):\n",
    "        self.df['nbatches'] = np.int32(np.ceil((np.ceil(self.df['nlines'] / 60)-self.seq_length+1)/self.batch_size))\n",
    "        self.df['ibatch'] = self.df['nbatches'].cumsum() - 1\n",
    "        return int(self.df['nbatches'].sum())\n",
    "    \n",
    "    def plot(self, ifile):\n",
    "        \n",
    "        ds = self.__get_ds(self.df['path'][ifile], Norm=False)\n",
    "        \n",
    "        fig, axs = plt.subplots(nrows=len(self.X_col)+1, figsize=(10, 12), sharex=True)\n",
    "        \n",
    "        fig.suptitle(self.df['path'][ifile])\n",
    "\n",
    "        for i, vs in enumerate(self.X_col):\n",
    "            axs[i].plot(ds.index, ds[(vs, 'mean')])\n",
    "            axs[i].fill_between(ds.index, ds[(vs, 'mean')]-1.96*ds[(vs, 'std')], \n",
    "                            ds[(vs, 'mean')]+1.96*ds[(vs, 'std')], \n",
    "                            alpha=0.2)\n",
    "            axs[i].set_ylabel(vs)\n",
    "            axs[i].grid()\n",
    "        \n",
    "        id = np.argsort(ds[(self.y_col, 'mode')])\n",
    "        \n",
    "        axs[i+1].scatter([ds.index[i] for i in id], [str(ds[(self.y_col, 'mode')][i]) for i in id], marker='.')\n",
    "        \n",
    "        axs[i+1].set_ylabel(self.y_col)\n",
    "        \n",
    "        axs[i+1].set_xlabel('minute')\n",
    "\n",
    "        plt.show()    \n",
    "    \n",
    "    def on_epoch_end(self):\n",
    "            pass\n",
    "    \n",
    "    def pkl_path(self, p):\n",
    "        new_p = self.tmp_path.joinpath(p.parts[-2], p.stem+'.pkl')\n",
    "        new_p.parent.mkdir(exist_ok=True, parents=True)\n",
    "        return new_p\n",
    "    \n",
    "    def __get_ds(self, p, Norm=True):\n",
    "    \n",
    "        pkl_p = self.pkl_path(p)\n",
    "        \n",
    "        if pkl_p.exists():\n",
    "            ds = pd.read_pickle(pkl_p)\n",
    "        else:\n",
    "        \n",
    "            dfo = pd.read_csv(p, index_col=\"timestamp\", parse_dates=[\"timestamp\"])\n",
    "\n",
    "            if np.any(dfo[self.y_col].isna()):\n",
    "                dfo[self.y_col] = dfo[self.y_col].fillna(method='ffill')\n",
    "            dfo[self.y_col] = dfo[self.y_col].astype('int')\n",
    "\n",
    "            flist = []\n",
    "            flist0 = []\n",
    "            for f in self.X_col:\n",
    "                nas = np.sum(dfo[f].isna())\n",
    "                if nas > 0:\n",
    "                    if nas < len(dfo.index) * 0.2:\n",
    "                        dfo[f] = dfo[f].fillna(method='ffill')\n",
    "                        flist.append(f)\n",
    "                    else:\n",
    "                        flist0.append(f)\n",
    "                else:\n",
    "                    flist.append(f)\n",
    "\n",
    "            fdict=dict()\n",
    "            for f in flist:\n",
    "                fdict[f] = ['mean','std']\n",
    "\n",
    "            def mode(series):\n",
    "                return pd.Series.mode(series)[0]\n",
    "\n",
    "            fdict[self.y_col] = [mode]\n",
    "\n",
    "            dfo['minute'] = (dfo.index-dfo.index[0])//np.timedelta64(1,'m')\n",
    "\n",
    "            ds = dfo.groupby('minute').agg(fdict)\n",
    "\n",
    "            #ds = ds.iloc[:-1]\n",
    "\n",
    "            for f in flist0:\n",
    "                ds[f, 'mean'] = np.NaN\n",
    "                ds[f, 'std'] = np.NaN\n",
    "                \n",
    "            ds.to_pickle(pkl_p)\n",
    "        \n",
    "        if Norm:\n",
    "            ds = self.__Norm(ds)\n",
    "            if ds.isnull().any().any():\n",
    "                print(p, pkl_p)\n",
    "        \n",
    "        return ds[self.X_col + [self.y_col]]\n",
    "        \n",
    "    def __Norm(self, ds, nas_v=0):\n",
    "        dn = ds.fillna(value=nas_v)\n",
    "        sc = sklearn.preprocessing.StandardScaler()\n",
    "        dn = pd.DataFrame(sc.fit_transform(dn.values), \n",
    "                                           index=dn.index, \n",
    "                                           columns=dn.columns)\n",
    "        dn[(self.y_col, 'mode')] = ds[(self.y_col, 'mode')]\n",
    "        return dn\n",
    "        \n",
    "    def __get_output(self, y):\n",
    "        \n",
    "        ohe = sklearn.preprocessing.OneHotEncoder(categories=self.categories, sparse= False)\n",
    "        \n",
    "        return ohe.fit_transform(y)\n",
    "    \n",
    "    def __get_data(self, i, j, p):\n",
    "        # Generates data containing batch_size samples\n",
    "\n",
    "        if p != self.file:\n",
    "            self.ts = self.__get_ds(p)\n",
    "            self.file = p\n",
    "\n",
    "        nf = len(self.X_col)\n",
    "        mat = np.zeros(shape=(self.batch_size, 2*nf*self.seq_length))\n",
    "        \n",
    "        for k in range(self.batch_size):\n",
    "            if self.seq_length+k+j*self.batch_size > len(self.ts.index):\n",
    "                mat = mat[:k]\n",
    "                break\n",
    "            mat[k] = np.reshape(self.ts[k+j*self.batch_size:self.seq_length+k+j*self.batch_size][self.X_col].to_numpy(), (1, -1))\n",
    "        return mat, self.ts.iloc[j*self.batch_size+self.seq_length-1:j*self.batch_size+self.seq_length+self.batch_size-1][(self.y_col, 'mode')].to_numpy()\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        \n",
    "        i = bisect.bisect_left(self.df.ibatch, index)\n",
    "        if i > 0:\n",
    "            j = index - self.df.ibatch[i-1] - 1\n",
    "        else:\n",
    "            j = index\n",
    "        \n",
    "        p = self.df.path[i]       \n",
    "        \n",
    "        # print(index, i, j, p)\n",
    "        \n",
    "        X, y = self.__get_data(i, j, p)        \n",
    "        \n",
    "        return X, y #self.__get_output(y)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.n\n",
    "    \n",
    "    def get_y(self):\n",
    "        y = np.empty(0, 'int')\n",
    "        for p in self.df['path']:\n",
    "            pkl_p = self.pkl_path(p)\n",
    "            if pkl_p.exists():\n",
    "                ds = pd.read_pickle(pkl_p)\n",
    "            else:\n",
    "                ds = self.__get_ds(p, Norm = False)\n",
    "            y = np.append(y, ds[self.y_col].iloc[self.seq_length-1:].astype('int'))\n",
    "        return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = CustomDataGen(train_df, flist0, 'class', categories, 64, 30, 'D:/datatmp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 0.33815170112613957,\n",
       " 1: 1.0394139194139194,\n",
       " 2: 37.834666666666664,\n",
       " 3: 0.6812147401272356,\n",
       " 4: 1.7592064476131433,\n",
       " 5: 0.32695010945961517,\n",
       " 6: 0.8598787878787879,\n",
       " 7: 6.168695652173913,\n",
       " 8: 6.980565805658056,\n",
       " 101: 0.563015873015873,\n",
       " 102: 23.451239669421486,\n",
       " 105: 1.5604069287874622,\n",
       " 106: 2.14969696969697,\n",
       " 107: 0.895000788519161,\n",
       " 108: 2.2281900274833135,\n",
       " 103: 1.0,\n",
       " 104: 1.0}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yss = train.get_y()\n",
    "w = dict(zip(np.unique(yss), compute_class_weight('balanced', classes=np.unique(yss), y=yss)))\n",
    "for j in categories:\n",
    "    if j not in w.keys():\n",
    "        w[j] = 1.0\n",
    "w"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1: 784 / 784    \n",
      "   val f1 score macro:  0.041 \n",
      "\n",
      "2: 784 / 784    \n",
      "   val f1 score macro:  0.013 \n",
      "\n",
      "3: 784 / 784    \n",
      "   val f1 score macro:  0.014 \n",
      "\n",
      "4: 784 / 784    \n",
      "   val f1 score macro:  0.018 \n",
      "\n",
      "\n",
      "\n",
      "Wall time: 5min 57s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "clf = OneVsRestClassifier(SGDClassifier(loss='perceptron', alpha=0.01, penalty='elasticnet', l1_ratio=0., class_weight=w), n_jobs=-1)\n",
    "#clf = SGDClassifier(penalty='elasticnet', l1_ratio=0.75, class_weight=w, n_jobs=-1)\n",
    "#clf = PassiveAggressiveClassifier(C=10, class_weight=w, n_jobs=-1)\n",
    "train = CustomDataGen(train_df, flist0, 'class', categories, 128, 30, 'D:/datatmp')\n",
    "val = CustomDataGen(val_df, flist0, 'class', categories, 128, 30, 'D:/datatmp')\n",
    "n = train.__len__()\n",
    "n_epochs = 4\n",
    "for iter in range(n_epochs):\n",
    "    train = CustomDataGen(train_df, flist0, 'class', categories, 128, 30, 'D:/datatmp')\n",
    "    for i in range(n):\n",
    "        print('\\r' + str(iter+1) + ': ' + str(i+1) +  ' / ' + str(n), end='    ', flush=True)\n",
    "        X, y = train.__getitem__(i)\n",
    "        clf.partial_fit(X, y, classes=categories)\n",
    "    ys = []\n",
    "    ypreds = []\n",
    "    for i in range(val.__len__()):\n",
    "        X, y = val.__getitem__(i)\n",
    "        ypreds += list(clf.predict(X))\n",
    "        ys += list(y)\n",
    "    print('\\n   val f1 score macro: ', round(f1_score(ys, ypreds, zero_division=1, average='macro'), 3), '\\n')\n",
    "    \n",
    "    train_df = train_df.sample(frac=1, random_state=200560).reset_index(drop=True)\n",
    "print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "394 / 394\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.07      0.13      4827\n",
      "           1       1.00      0.00      0.00      1347\n",
      "           2       1.00      0.00      0.00         9\n",
      "           3       0.00      0.00      0.00      2195\n",
      "           4       1.00      0.00      0.00       996\n",
      "           5       0.00      0.00      0.00      5654\n",
      "           6       0.00      0.00      0.00      2100\n",
      "           8       1.00      0.00      0.00       299\n",
      "         101       0.00      0.00      0.00      2918\n",
      "         102       1.00      0.00      0.00        97\n",
      "         105       0.07      0.71      0.13       786\n",
      "         106       0.00      0.00      0.00       840\n",
      "         107       0.00      1.00      0.00         0\n",
      "         108       0.00      0.00      0.00       541\n",
      "\n",
      "    accuracy                           0.04     22609\n",
      "   macro avg       0.43      0.13      0.02     22609\n",
      "weighted avg       0.31      0.04      0.03     22609\n",
      "\n",
      "Wall time: 19.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "test = CustomDataGen(test_df, flist0, 'class', categories, 64, 30, 'D:/datatmp')\n",
    "ys = []\n",
    "ypreds = []\n",
    "n = test.__len__()\n",
    "for i in range(n):\n",
    "    print('\\r' + str(i+1) +  ' / ' + str(n), end='', flush=True)\n",
    "    X, y = test.__getitem__(i)\n",
    "    ypreds += list(clf.predict(X))\n",
    "    ys += list(y)\n",
    "print('\\n')\n",
    "print(classification_report(ys, ypreds, zero_division=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "784 / 784\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.69      0.07      0.13     16783\n",
      "           1       1.00      0.00      0.00      5460\n",
      "           2       1.00      0.00      0.00       150\n",
      "           3       0.00      0.00      0.00      8331\n",
      "           4       0.00      0.00      0.00      3226\n",
      "           5       0.00      0.00      0.00     17358\n",
      "           6       0.00      0.00      0.00      6600\n",
      "           7       1.00      0.00      0.00       920\n",
      "           8       1.00      0.00      0.00       813\n",
      "         101       0.00      0.00      0.00     10080\n",
      "         102       0.00      0.00      0.00       242\n",
      "         105       0.05      0.44      0.09      3637\n",
      "         106       0.00      0.00      0.00      2640\n",
      "         107       0.07      0.46      0.12      6341\n",
      "         108       0.00      0.00      0.00      2547\n",
      "\n",
      "    accuracy                           0.07     85128\n",
      "   macro avg       0.32      0.06      0.02     85128\n",
      "weighted avg       0.23      0.07      0.04     85128\n",
      "\n",
      "Wall time: 1min 3s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "ys = []\n",
    "ypreds = []\n",
    "n = train.__len__()\n",
    "for i in range(n):\n",
    "    print('\\r' + str(i+1) +  ' / ' + str(n), end='', flush=True)\n",
    "    X, y = train.__getitem__(i)\n",
    "    ypreds += list(clf.predict(X))\n",
    "    ys += list(y)\n",
    "print('\\n')\n",
    "print(classification_report(ys, ypreds, zero_division=True))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Sumário",
   "title_sidebar": "Sumário",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}