predictive-maintenance-wrangling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "98e827c0-2d16-4963-9672-2488581336f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import seaborn as sns\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
    "\n",
    "from common import get_data\n",
    "from scipy.stats import zscore\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c0837eb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data = get_data('data/train_selected.csv')\n",
    "test_data = get_data('data/test_selected_ttf.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22cec7bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cycle</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>s3</th>\n",
       "      <th>s4</th>\n",
       "      <th>ttf</th>\n",
       "      <th>label_bnc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>1398.91</td>\n",
       "      <td>554.42</td>\n",
       "      <td>47.23</td>\n",
       "      <td>521.79</td>\n",
       "      <td>112</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>49</td>\n",
       "      <td>1410.83</td>\n",
       "      <td>553.52</td>\n",
       "      <td>47.67</td>\n",
       "      <td>521.74</td>\n",
       "      <td>98</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>126</td>\n",
       "      <td>1418.89</td>\n",
       "      <td>552.59</td>\n",
       "      <td>47.88</td>\n",
       "      <td>520.83</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>106</td>\n",
       "      <td>1406.88</td>\n",
       "      <td>552.64</td>\n",
       "      <td>47.65</td>\n",
       "      <td>521.88</td>\n",
       "      <td>82</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>98</td>\n",
       "      <td>1419.36</td>\n",
       "      <td>553.29</td>\n",
       "      <td>47.46</td>\n",
       "      <td>521.00</td>\n",
       "      <td>91</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  cycle       s1      s2     s3      s4  ttf  label_bnc\n",
       "0   1     31  1398.91  554.42  47.23  521.79  112          0\n",
       "1   2     49  1410.83  553.52  47.67  521.74   98          0\n",
       "2   3    126  1418.89  552.59  47.88  520.83   69          0\n",
       "3   4    106  1406.88  552.64  47.65  521.88   82          0\n",
       "4   5     98  1419.36  553.29  47.46  521.00   91          0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bdc491e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20631, 8)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Find the number of rows and columns in the training data\n",
    "training_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8badbabb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 8)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Find the number of rows and columns in the test data\n",
    "test_data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8ee77fd",
   "metadata": {},
   "source": [
    "# Prepare the base training data for modelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6244c717",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = ['ID', 'CYCLE', 'S1', 'S2', 'S3', 'S4', 'TTF', 'TTF_LABEL']\n",
    "training_data.columns = columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b6ef7c2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save training data to csv\n",
    "training_data.to_csv('data/train_base.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0347dd8",
   "metadata": {},
   "source": [
    "# Prepare the base testing data for modelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9265db0f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cycle</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>s3</th>\n",
       "      <th>s4</th>\n",
       "      <th>ttf</th>\n",
       "      <th>label_bnc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>1398.91</td>\n",
       "      <td>554.42</td>\n",
       "      <td>47.23</td>\n",
       "      <td>521.79</td>\n",
       "      <td>112</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>49</td>\n",
       "      <td>1410.83</td>\n",
       "      <td>553.52</td>\n",
       "      <td>47.67</td>\n",
       "      <td>521.74</td>\n",
       "      <td>98</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>126</td>\n",
       "      <td>1418.89</td>\n",
       "      <td>552.59</td>\n",
       "      <td>47.88</td>\n",
       "      <td>520.83</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>106</td>\n",
       "      <td>1406.88</td>\n",
       "      <td>552.64</td>\n",
       "      <td>47.65</td>\n",
       "      <td>521.88</td>\n",
       "      <td>82</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>98</td>\n",
       "      <td>1419.36</td>\n",
       "      <td>553.29</td>\n",
       "      <td>47.46</td>\n",
       "      <td>521.00</td>\n",
       "      <td>91</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  cycle       s1      s2     s3      s4  ttf  label_bnc\n",
       "0   1     31  1398.91  554.42  47.23  521.79  112          0\n",
       "1   2     49  1410.83  553.52  47.67  521.74   98          0\n",
       "2   3    126  1418.89  552.59  47.88  520.83   69          0\n",
       "3   4    106  1406.88  552.64  47.65  521.88   82          0\n",
       "4   5     98  1419.36  553.29  47.46  521.00   91          0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1c06bdab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    75\n",
       "1    25\n",
       "Name: label_bnc, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.label_bnc.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "feaf9423",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Update column names\n",
    "test_data.columns = columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "19574569",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Cycle</th>\n",
       "      <th>S1</th>\n",
       "      <th>S2</th>\n",
       "      <th>S3</th>\n",
       "      <th>S4</th>\n",
       "      <th>TTF</th>\n",
       "      <th>TTF_LABEL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>1398.91</td>\n",
       "      <td>554.42</td>\n",
       "      <td>47.23</td>\n",
       "      <td>521.79</td>\n",
       "      <td>112</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>49</td>\n",
       "      <td>1410.83</td>\n",
       "      <td>553.52</td>\n",
       "      <td>47.67</td>\n",
       "      <td>521.74</td>\n",
       "      <td>98</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>126</td>\n",
       "      <td>1418.89</td>\n",
       "      <td>552.59</td>\n",
       "      <td>47.88</td>\n",
       "      <td>520.83</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>106</td>\n",
       "      <td>1406.88</td>\n",
       "      <td>552.64</td>\n",
       "      <td>47.65</td>\n",
       "      <td>521.88</td>\n",
       "      <td>82</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>98</td>\n",
       "      <td>1419.36</td>\n",
       "      <td>553.29</td>\n",
       "      <td>47.46</td>\n",
       "      <td>521.00</td>\n",
       "      <td>91</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  Cycle       S1      S2     S3      S4  TTF  TTF_LABEL\n",
       "0   1     31  1398.91  554.42  47.23  521.79  112          0\n",
       "1   2     49  1410.83  553.52  47.67  521.74   98          0\n",
       "2   3    126  1418.89  552.59  47.88  520.83   69          0\n",
       "3   4    106  1406.88  552.64  47.65  521.88   82          0\n",
       "4   5     98  1419.36  553.29  47.46  521.00   91          0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a7793343",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save test data to csv\n",
    "test_data.to_csv('data/test_base.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>CYCLE</th>\n",
       "      <th>S1</th>\n",
       "      <th>S2</th>\n",
       "      <th>S3</th>\n",
       "      <th>S4</th>\n",
       "      <th>TTF</th>\n",
       "      <th>TTF_LABEL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1400.60</td>\n",
       "      <td>554.36</td>\n",
       "      <td>47.47</td>\n",
       "      <td>521.66</td>\n",
       "      <td>191</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1403.14</td>\n",
       "      <td>553.75</td>\n",
       "      <td>47.49</td>\n",
       "      <td>522.28</td>\n",
       "      <td>190</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1404.20</td>\n",
       "      <td>554.26</td>\n",
       "      <td>47.27</td>\n",
       "      <td>522.42</td>\n",
       "      <td>189</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20628</th>\n",
       "      <td>100</td>\n",
       "      <td>198</td>\n",
       "      <td>1428.18</td>\n",
       "      <td>550.94</td>\n",
       "      <td>48.09</td>\n",
       "      <td>520.01</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20629</th>\n",
       "      <td>100</td>\n",
       "      <td>199</td>\n",
       "      <td>1426.53</td>\n",
       "      <td>550.68</td>\n",
       "      <td>48.39</td>\n",
       "      <td>519.67</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20630</th>\n",
       "      <td>100</td>\n",
       "      <td>200</td>\n",
       "      <td>1432.14</td>\n",
       "      <td>550.79</td>\n",
       "      <td>48.20</td>\n",
       "      <td>519.30</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ID  CYCLE       S1      S2     S3      S4  TTF  TTF_LABEL\n",
       "0        1      1  1400.60  554.36  47.47  521.66  191          0\n",
       "1        1      2  1403.14  553.75  47.49  522.28  190          0\n",
       "2        1      3  1404.20  554.26  47.27  522.42  189          0\n",
       "20628  100    198  1428.18  550.94  48.09  520.01    2          1\n",
       "20629  100    199  1426.53  550.68  48.39  519.67    1          1\n",
       "20630  100    200  1432.14  550.79  48.20  519.30    0          1"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat([training_data.head(3),training_data.tail(3)],axis=0) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0279b2e3",
   "metadata": {},
   "source": [
    "## Data scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "baccc859",
   "metadata": {},
   "outputs": [],
   "source": [
    "scaled_columns = ['S1', 'S2', 'S3', 'S4']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2988975a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StandardScaler()"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "standard_scaler = StandardScaler()\n",
    "\n",
    "standard_scaler.fit(training_data[columns])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f6c24af4",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = standard_scaler.transform(training_data[columns])\n",
    "X_test = standard_scaler.transform(test_data[columns])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "69916c2c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>S1</th>\n",
       "      <th>S2</th>\n",
       "      <th>S3</th>\n",
       "      <th>S4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.925936</td>\n",
       "      <td>1.121141</td>\n",
       "      <td>-0.266467</td>\n",
       "      <td>0.334262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.643726</td>\n",
       "      <td>0.431930</td>\n",
       "      <td>-0.191583</td>\n",
       "      <td>1.174899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.525953</td>\n",
       "      <td>1.008155</td>\n",
       "      <td>-1.015303</td>\n",
       "      <td>1.364721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.784831</td>\n",
       "      <td>1.222827</td>\n",
       "      <td>-1.539489</td>\n",
       "      <td>1.961302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.301518</td>\n",
       "      <td>0.714393</td>\n",
       "      <td>-0.977861</td>\n",
       "      <td>1.052871</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         S1        S2        S3        S4\n",
       "0 -0.925936  1.121141 -0.266467  0.334262\n",
       "1 -0.643726  0.431930 -0.191583  1.174899\n",
       "2 -0.525953  1.008155 -1.015303  1.364721\n",
       "3 -0.784831  1.222827 -1.539489  1.961302\n",
       "4 -0.301518  0.714393 -0.977861  1.052871"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = pd.DataFrame(X_train, columns=columns)\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7b660c04",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>S1</th>\n",
       "      <th>S2</th>\n",
       "      <th>S3</th>\n",
       "      <th>S4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.113706</td>\n",
       "      <td>1.188932</td>\n",
       "      <td>-1.165071</td>\n",
       "      <td>0.510525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.210682</td>\n",
       "      <td>0.172064</td>\n",
       "      <td>0.482370</td>\n",
       "      <td>0.442731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.106199</td>\n",
       "      <td>-0.878699</td>\n",
       "      <td>1.268649</td>\n",
       "      <td>-0.791108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.228188</td>\n",
       "      <td>-0.822207</td>\n",
       "      <td>0.407486</td>\n",
       "      <td>0.632553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.158419</td>\n",
       "      <td>-0.087802</td>\n",
       "      <td>-0.303908</td>\n",
       "      <td>-0.560610</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         S1        S2        S3        S4\n",
       "0 -1.113706  1.188932 -1.165071  0.510525\n",
       "1  0.210682  0.172064  0.482370  0.442731\n",
       "2  1.106199 -0.878699  1.268649 -0.791108\n",
       "3 -0.228188 -0.822207  0.407486  0.632553\n",
       "4  1.158419 -0.087802 -0.303908 -0.560610"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test = pd.DataFrame(X_test, columns=columns)\n",
    "X_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "3f89db86",
   "metadata": {},
   "outputs": [],
   "source": [
    "scaled_training_data = pd.concat([training_data[[\"ID\", \"Cycle\", \"TTF\", \"TTF_LABEL\"]], X_train], axis=1)\n",
    "scaled_training_data.to_csv('data/training_standard_scaled.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ed4f1852",
   "metadata": {},
   "outputs": [],
   "source": [
    "scaled_test_data = pd.concat([test_data[[\"ID\", \"Cycle\", \"TTF\", \"TTF_LABEL\"]], X_test], axis=1)\n",
    "scaled_test_data.to_csv('data/test_standard_scaled.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83339237",
   "metadata": {},
   "source": [
    "### MinMaxScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5c47bf6d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MinMaxScaler()"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_max_scaler = MinMaxScaler()\n",
    "min_max_scaler.fit(training_data[scaled_columns])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1ddd9b86",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = min_max_scaler.transform(training_data[scaled_columns])\n",
    "X_test = min_max_scaler.transform(test_data[scaled_columns])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1b90d54a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>S1</th>\n",
       "      <th>S2</th>\n",
       "      <th>S3</th>\n",
       "      <th>S4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.309757</td>\n",
       "      <td>0.726248</td>\n",
       "      <td>0.369048</td>\n",
       "      <td>0.633262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.352633</td>\n",
       "      <td>0.628019</td>\n",
       "      <td>0.380952</td>\n",
       "      <td>0.765458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.370527</td>\n",
       "      <td>0.710145</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.795309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.331195</td>\n",
       "      <td>0.740741</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.889126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.404625</td>\n",
       "      <td>0.668277</td>\n",
       "      <td>0.255952</td>\n",
       "      <td>0.746269</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         S1        S2        S3        S4\n",
       "0  0.309757  0.726248  0.369048  0.633262\n",
       "1  0.352633  0.628019  0.380952  0.765458\n",
       "2  0.370527  0.710145  0.250000  0.795309\n",
       "3  0.331195  0.740741  0.166667  0.889126\n",
       "4  0.404625  0.668277  0.255952  0.746269"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = pd.DataFrame(X_train, columns=scaled_columns)\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "251bbea1",
   "metadata": {},
   "outputs": [],
   "source": [
    "min_max_scaled_training_data = pd.concat([training_data[[\"ID\", \"Cycle\", \"TTF\", \"TTF_LABEL\"]], X_train], axis=1)\n",
    "min_max_scaled_training_data.to_csv('data/training_min_max_scaled.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8fc35b57",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = pd.DataFrame(X_test, columns=scaled_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "826d353d",
   "metadata": {},
   "outputs": [],
   "source": [
    "min_max_scaled_test_data = pd.concat([test_data[[\"ID\", \"Cycle\", \"TTF\", \"TTF_LABEL\"]], X_test], axis=1)\n",
    "min_max_scaled_test_data.to_csv('data/test_min_max_scaled.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "728953e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26.0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quantiles = training_data.quantile([0.25, 0.75])\n",
    "quantiles.iloc[0, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "9e7f40d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "127"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "z_scores = zscore(training_data[scaled_columns])\n",
    "z_scores[(z_scores > 3) | (z_scores < -3)].count().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "e8e1531c",
   "metadata": {},
   "outputs": [],
   "source": [
    "k = 3\n",
    "to_remove = {}\n",
    "for col in scaled_columns:\n",
    "    num_out_outliers_index = z_scores[(z_scores[col] > k) | (z_scores[col] < -k)].index\n",
    "    to_remove[col] = num_out_outliers_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "234e9782",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Removed: 115\n",
      "This is 0.5574136008918618% of our dataset\n"
     ]
    }
   ],
   "source": [
    "combined = set()\n",
    "for indexes in to_remove.values():\n",
    "    combined.update(set(indexes))\n",
    "print(\"Removed:\", len(combined))\n",
    "print(f\"This is {len(combined) / training_data.shape[0] * 100}% of our dataset\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e19677fc",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "370c432e96fa8207a363aa19249434d63375148c4a55e8542936cb07680fb1ca"
  },
  "kernelspec": {
   "display_name": "Python 3.9.1 64-bit ('SLM-Code': pyenv)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}