Preprocess_Helper_Example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3b023ba5-cc36-442b-a594-b7c1002117b7",
   "metadata": {},
   "source": [
    "# Please use this notebook to load in your own data to test out the preprocess helper!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f376b20c-88e7-4fcd-8c58-a75d4b39a8db",
   "metadata": {},
   "outputs": [],
   "source": [
    "from preprocess_help_DEV import DataPreProcess\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import warnings\n",
    "import pickle\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5dc8657-fe18-475c-a323-6f48f7b5272c",
   "metadata": {},
   "source": [
    "#### Load your data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d3fe3865-943d-4c2b-aef3-af588d7a3b3b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(344, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>species</th>\n",
       "      <th>island</th>\n",
       "      <th>bill_length_mm</th>\n",
       "      <th>bill_depth_mm</th>\n",
       "      <th>flipper_length_mm</th>\n",
       "      <th>body_mass_g</th>\n",
       "      <th>sex</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Adelie</td>\n",
       "      <td>Torgersen</td>\n",
       "      <td>39.1</td>\n",
       "      <td>18.7</td>\n",
       "      <td>181.0</td>\n",
       "      <td>3750.0</td>\n",
       "      <td>Male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Adelie</td>\n",
       "      <td>Torgersen</td>\n",
       "      <td>39.5</td>\n",
       "      <td>17.4</td>\n",
       "      <td>186.0</td>\n",
       "      <td>3800.0</td>\n",
       "      <td>Female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Adelie</td>\n",
       "      <td>Torgersen</td>\n",
       "      <td>40.3</td>\n",
       "      <td>18.0</td>\n",
       "      <td>195.0</td>\n",
       "      <td>3250.0</td>\n",
       "      <td>Female</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  species     island  bill_length_mm  bill_depth_mm  flipper_length_mm  \\\n",
       "0  Adelie  Torgersen            39.1           18.7              181.0   \n",
       "1  Adelie  Torgersen            39.5           17.4              186.0   \n",
       "2  Adelie  Torgersen            40.3           18.0              195.0   \n",
       "\n",
       "   body_mass_g     sex  \n",
       "0       3750.0    Male  \n",
       "1       3800.0  Female  \n",
       "2       3250.0  Female  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#my_df = pd.read_csv(\"PUT YOUR FILENAME HERE\") # LOAD YOUR DATA HERE!\n",
    "my_df = sns.load_dataset('penguins')\n",
    "print(my_df.shape)\n",
    "my_df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d792457-e2a1-4ee6-be1f-80072eb24fb3",
   "metadata": {},
   "source": [
    "#### Initialize the class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4d5be7d5-cf3a-4ed5-b482-d23e02f99cd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor = DataPreProcess() # initialize the class"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0588b95e-959f-41d5-99a5-00ca59cc2adc",
   "metadata": {},
   "source": [
    "#### Use the \"process_columns\" method to step through each column and decide what to do with it\n",
    "options are: ['skip', 'id', 'response', 'float', 'category', 'binary', 'ordinal', 'other']\n",
    "* **skip**: this is for columns you don't want to use in your pipeline or modeling\n",
    "* **id**: id columns will not be used in your pipeline TODO: these are meant to be saved for use later if needed to match back to original data.\n",
    "* **response**: if chosen you'll have the option to keep the response as a float, string, or one-hot encode this TODO: do the transformation and return in another method\n",
    "* **float**: if chosen you will have the option to view what the yeo-johnson transformation looks like and then choose if you want this column transformed in the preprocessing pipeline\n",
    "* **category**: this will one-hot encode the column in the pipeline. TODO: allow option to keep as string (ex when using a tree-based method that allows categories)\n",
    "* **binary**: this will treat the column like a normal float column. TODO: are there any special treatments for binary variables?\n",
    "* **ordinal**: at the moment it will treat it like a float. TODO: look into any special treatments for ordinal variables?\n",
    "* **other**: this will ignore the column in the pipeline. TODO: let the data scientist review the 'other' columns again later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ee06422c-2efe-4f38-86a0-b1d840ed03c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_____________________________\n",
      "Pre-processing selection\n",
      "\n",
      "100%\t[\u001b[5;37;42m                                                  \u001b[0m] 7/7 columns\n",
      "\n",
      "\n",
      "*** sex *** \n",
      "\n",
      "\n",
      "count      333\n",
      "unique       2\n",
      "top       Male\n",
      "freq       168\n",
      "Name: sex, dtype: object\n",
      "\u001b[5;30;42m11 (3%) number of NAs\u001b[0m\n",
      "sex will be used as a categorical feature\n",
      "\n",
      "DONE :)\n",
      "***********\n"
     ]
    }
   ],
   "source": [
    "preprocessor.process_columns(my_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e136656-aa5e-41ba-814a-4b7411227fc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# process or update a single column. need to pass in df again\n",
    "preprocessor.process_columns(my_df, single_col='species')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cedb8789-46db-4ba5-8d9b-949bb9bb3553",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bill_length_mm': 'yeojohnson',\n",
       " 'bill_depth_mm': 'box-cox',\n",
       " 'flipper_length_mm': 'none',\n",
       " 'body_mass_g': 'none'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor.skew_transformer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1743581-05a0-4621-b6cb-8dffddf85e3a",
   "metadata": {},
   "source": [
    "#### Create the pipeline. this will create the pipeline framework that fits the decisions made above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "29a05230-577d-49ac-a531-6acda5dc8925",
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor.create_pipeline(my_df) # this will create the pipeline with the steps that fit your decisions above"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13faecec-8c5e-4c66-9389-b4b3c9a50783",
   "metadata": {},
   "source": [
    "#### Fit the pipeline with data. If no dataframe is passed, it will use the original dataframe seen above. Or you can do your test/train split and then pass your training dataframe to fit the pipeline. \n",
    "\n",
    "As long as the columns have the same name, there is no need to specify which columns to keep because you've already decided what to do with them. You can pass the whole dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "09e83e5b-7ed5-48a8-aaec-71e2513f8e77",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "y = preprocessor.return_response_values(my_df) # use the class to return the response column you specified. if not specified, just prints message\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(my_df, y, test_size=0.33, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5c252b29-0328-4bc2-b1a8-8d7674dc9b2a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bill_length_mm', 'bill_depth_mm', 'flipper_length_mm', 'body_mass_g']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor.float_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "217e437e-27ce-48b5-87fb-b5a760c758ec",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a9cf7a11-35ea-48cf-a303-024f0bbc7c5d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Pipeline]  (step 1 of 7) Processing numeric_capping_quant, total=   0.0s\n",
      "[Pipeline]  (step 2 of 7) Processing numeric_imputation, total=   0.0s\n",
      "[Pipeline]  (step 3 of 7) Processing category_null_value, total=   0.0s\n",
      "[Pipeline]  (step 4 of 7) Processing rare_level_grouping, total=   0.0s\n",
      "[Pipeline] .. (step 5 of 7) Processing one_hot_encoding, total=   0.0s\n",
      "[Pipeline]  (step 6 of 7) Processing float transformers, total=   0.0s\n",
      "[Pipeline] ............ (step 7 of 7) Processing minmax, total=   0.0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/seth.johnston/anaconda3/envs/lead_scoring_CSC/lib/python3.10/site-packages/sklearn/preprocessing/_encoders.py:868: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "#preprocessor.fit_pipeline() # fit the pipeline. if no dataframe is passed it will fit on the orinigal dataframe\n",
    "preprocessor.fit_pipeline(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56acd18a-0479-4fcd-97a2-a43bb13b9843",
   "metadata": {},
   "source": [
    "#### You can now return the pipeline to be used later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a93f4da7-770e-45e5-b861-db729a4d147c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;numeric_capping_quant&#x27;,\n",
       "                 CappingTransformer(capping_values={&#x27;bill_depth_mm&#x27;: [14.2,\n",
       "                                                                      19.5],\n",
       "                                                    &#x27;bill_length_mm&#x27;: [36.5,\n",
       "                                                                       51.01],\n",
       "                                                    &#x27;body_mass_g&#x27;: [3245.0,\n",
       "                                                                    5355.0],\n",
       "                                                    &#x27;flipper_length_mm&#x27;: [185.0,\n",
       "                                                                          220.0]},\n",
       "                                    quantiles={&#x27;bill_depth_mm&#x27;: [0.1, 0.9],\n",
       "                                               &#x27;bill_length_mm&#x27;: [0.1, 0.9],\n",
       "                                               &#x27;body_mass_g&#x27;: [0.1, 0.9],\n",
       "                                               &#x27;flipper_length_mm&#x27;: [0.1,\n",
       "                                                                     0.9]})),\n",
       "                (&#x27;numeric_imputation&#x27;,\n",
       "                 MeanImpu...\n",
       "                                   transformers=[(&#x27;yj&#x27;, PowerTransformer(),\n",
       "                                                  [&#x27;bill_length_mm&#x27;]),\n",
       "                                                 (&#x27;box&#x27;,\n",
       "                                                  PowerTransformer(method=&#x27;box-cox&#x27;),\n",
       "                                                  [&#x27;bill_depth_mm&#x27;])],\n",
       "                                   verbose_feature_names_out=False)),\n",
       "                (&#x27;minmax&#x27;,\n",
       "                 ColumnTransformer(remainder=&#x27;passthrough&#x27;, sparse_threshold=0,\n",
       "                                   transformers=[(&#x27;min_max_ct&#x27;, MinMaxScaler(),\n",
       "                                                  [&#x27;bill_length_mm&#x27;,\n",
       "                                                   &#x27;bill_depth_mm&#x27;,\n",
       "                                                   &#x27;flipper_length_mm&#x27;,\n",
       "                                                   &#x27;body_mass_g&#x27;])],\n",
       "                                   verbose_feature_names_out=False))],\n",
       "         verbose=True)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-20\" type=\"checkbox\" ><label for=\"sk-estimator-id-20\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;numeric_capping_quant&#x27;,\n",
       "                 CappingTransformer(capping_values={&#x27;bill_depth_mm&#x27;: [14.2,\n",
       "                                                                      19.5],\n",
       "                                                    &#x27;bill_length_mm&#x27;: [36.5,\n",
       "                                                                       51.01],\n",
       "                                                    &#x27;body_mass_g&#x27;: [3245.0,\n",
       "                                                                    5355.0],\n",
       "                                                    &#x27;flipper_length_mm&#x27;: [185.0,\n",
       "                                                                          220.0]},\n",
       "                                    quantiles={&#x27;bill_depth_mm&#x27;: [0.1, 0.9],\n",
       "                                               &#x27;bill_length_mm&#x27;: [0.1, 0.9],\n",
       "                                               &#x27;body_mass_g&#x27;: [0.1, 0.9],\n",
       "                                               &#x27;flipper_length_mm&#x27;: [0.1,\n",
       "                                                                     0.9]})),\n",
       "                (&#x27;numeric_imputation&#x27;,\n",
       "                 MeanImpu...\n",
       "                                   transformers=[(&#x27;yj&#x27;, PowerTransformer(),\n",
       "                                                  [&#x27;bill_length_mm&#x27;]),\n",
       "                                                 (&#x27;box&#x27;,\n",
       "                                                  PowerTransformer(method=&#x27;box-cox&#x27;),\n",
       "                                                  [&#x27;bill_depth_mm&#x27;])],\n",
       "                                   verbose_feature_names_out=False)),\n",
       "                (&#x27;minmax&#x27;,\n",
       "                 ColumnTransformer(remainder=&#x27;passthrough&#x27;, sparse_threshold=0,\n",
       "                                   transformers=[(&#x27;min_max_ct&#x27;, MinMaxScaler(),\n",
       "                                                  [&#x27;bill_length_mm&#x27;,\n",
       "                                                   &#x27;bill_depth_mm&#x27;,\n",
       "                                                   &#x27;flipper_length_mm&#x27;,\n",
       "                                                   &#x27;body_mass_g&#x27;])],\n",
       "                                   verbose_feature_names_out=False))],\n",
       "         verbose=True)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-21\" type=\"checkbox\" ><label for=\"sk-estimator-id-21\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">CappingTransformer</label><div class=\"sk-toggleable__content\"><pre>CappingTransformer(capping_values={&#x27;bill_depth_mm&#x27;: [14.2, 19.5],\n",
       "                                   &#x27;bill_length_mm&#x27;: [36.5, 51.01],\n",
       "                                   &#x27;body_mass_g&#x27;: [3245.0, 5355.0],\n",
       "                                   &#x27;flipper_length_mm&#x27;: [185.0, 220.0]},\n",
       "                   quantiles={&#x27;bill_depth_mm&#x27;: [0.1, 0.9],\n",
       "                              &#x27;bill_length_mm&#x27;: [0.1, 0.9],\n",
       "                              &#x27;body_mass_g&#x27;: [0.1, 0.9],\n",
       "                              &#x27;flipper_length_mm&#x27;: [0.1, 0.9]})</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-22\" type=\"checkbox\" ><label for=\"sk-estimator-id-22\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MeanImputer</label><div class=\"sk-toggleable__content\"><pre>MeanImputer(columns=[&#x27;bill_length_mm&#x27;, &#x27;bill_depth_mm&#x27;, &#x27;flipper_length_mm&#x27;,\n",
       "                     &#x27;body_mass_g&#x27;])</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-23\" type=\"checkbox\" ><label for=\"sk-estimator-id-23\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">ArbitraryImputer</label><div class=\"sk-toggleable__content\"><pre>ArbitraryImputer(columns=[&#x27;island&#x27;, &#x27;sex&#x27;], impute_value=&#x27;missing&#x27;)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-24\" type=\"checkbox\" ><label for=\"sk-estimator-id-24\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GroupRareLevelsTransformer</label><div class=\"sk-toggleable__content\"><pre>GroupRareLevelsTransformer(columns=[&#x27;island&#x27;, &#x27;sex&#x27;], cut_off_percent=0.1)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-25\" type=\"checkbox\" ><label for=\"sk-estimator-id-25\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">OneHotEncodingTransformer</label><div class=\"sk-toggleable__content\"><pre>OneHotEncodingTransformer(columns=[&#x27;island&#x27;, &#x27;sex&#x27;], drop_original=True)</pre></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-26\" type=\"checkbox\" ><label for=\"sk-estimator-id-26\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">float transformers: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(remainder=&#x27;passthrough&#x27;, sparse_threshold=0,\n",
       "                  transformers=[(&#x27;yj&#x27;, PowerTransformer(), [&#x27;bill_length_mm&#x27;]),\n",
       "                                (&#x27;box&#x27;, PowerTransformer(method=&#x27;box-cox&#x27;),\n",
       "                                 [&#x27;bill_depth_mm&#x27;])],\n",
       "                  verbose_feature_names_out=False)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-27\" type=\"checkbox\" ><label for=\"sk-estimator-id-27\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">yj</label><div class=\"sk-toggleable__content\"><pre>[&#x27;bill_length_mm&#x27;]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-28\" type=\"checkbox\" ><label for=\"sk-estimator-id-28\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">PowerTransformer</label><div class=\"sk-toggleable__content\"><pre>PowerTransformer()</pre></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-29\" type=\"checkbox\" ><label for=\"sk-estimator-id-29\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">box</label><div class=\"sk-toggleable__content\"><pre>[&#x27;bill_depth_mm&#x27;]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-30\" type=\"checkbox\" ><label for=\"sk-estimator-id-30\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">PowerTransformer</label><div class=\"sk-toggleable__content\"><pre>PowerTransformer(method=&#x27;box-cox&#x27;)</pre></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-31\" type=\"checkbox\" ><label for=\"sk-estimator-id-31\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">remainder</label><div class=\"sk-toggleable__content\"><pre>[&#x27;flipper_length_mm&#x27;, &#x27;body_mass_g&#x27;, &#x27;island_Biscoe&#x27;, &#x27;island_Dream&#x27;, &#x27;island_Torgersen&#x27;, &#x27;sex_Female&#x27;, &#x27;sex_Male&#x27;, &#x27;sex_rare&#x27;]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-32\" type=\"checkbox\" ><label for=\"sk-estimator-id-32\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">passthrough</label><div class=\"sk-toggleable__content\"><pre>passthrough</pre></div></div></div></div></div></div></div></div><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-33\" type=\"checkbox\" ><label for=\"sk-estimator-id-33\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">minmax: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(remainder=&#x27;passthrough&#x27;, sparse_threshold=0,\n",
       "                  transformers=[(&#x27;min_max_ct&#x27;, MinMaxScaler(),\n",
       "                                 [&#x27;bill_length_mm&#x27;, &#x27;bill_depth_mm&#x27;,\n",
       "                                  &#x27;flipper_length_mm&#x27;, &#x27;body_mass_g&#x27;])],\n",
       "                  verbose_feature_names_out=False)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-34\" type=\"checkbox\" ><label for=\"sk-estimator-id-34\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">min_max_ct</label><div class=\"sk-toggleable__content\"><pre>[&#x27;bill_length_mm&#x27;, &#x27;bill_depth_mm&#x27;, &#x27;flipper_length_mm&#x27;, &#x27;body_mass_g&#x27;]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-35\" type=\"checkbox\" ><label for=\"sk-estimator-id-35\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MinMaxScaler</label><div class=\"sk-toggleable__content\"><pre>MinMaxScaler()</pre></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-36\" type=\"checkbox\" ><label for=\"sk-estimator-id-36\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">remainder</label><div class=\"sk-toggleable__content\"><pre>[&#x27;island_Biscoe&#x27;, &#x27;island_Dream&#x27;, &#x27;island_Torgersen&#x27;, &#x27;sex_Female&#x27;, &#x27;sex_Male&#x27;, &#x27;sex_rare&#x27;]</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-37\" type=\"checkbox\" ><label for=\"sk-estimator-id-37\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">passthrough</label><div class=\"sk-toggleable__content\"><pre>passthrough</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "Pipeline(steps=[('numeric_capping_quant',\n",
       "                 CappingTransformer(capping_values={'bill_depth_mm': [14.2,\n",
       "                                                                      19.5],\n",
       "                                                    'bill_length_mm': [36.5,\n",
       "                                                                       51.01],\n",
       "                                                    'body_mass_g': [3245.0,\n",
       "                                                                    5355.0],\n",
       "                                                    'flipper_length_mm': [185.0,\n",
       "                                                                          220.0]},\n",
       "                                    quantiles={'bill_depth_mm': [0.1, 0.9],\n",
       "                                               'bill_length_mm': [0.1, 0.9],\n",
       "                                               'body_mass_g': [0.1, 0.9],\n",
       "                                               'flipper_length_mm': [0.1,\n",
       "                                                                     0.9]})),\n",
       "                ('numeric_imputation',\n",
       "                 MeanImpu...\n",
       "                                   transformers=[('yj', PowerTransformer(),\n",
       "                                                  ['bill_length_mm']),\n",
       "                                                 ('box',\n",
       "                                                  PowerTransformer(method='box-cox'),\n",
       "                                                  ['bill_depth_mm'])],\n",
       "                                   verbose_feature_names_out=False)),\n",
       "                ('minmax',\n",
       "                 ColumnTransformer(remainder='passthrough', sparse_threshold=0,\n",
       "                                   transformers=[('min_max_ct', MinMaxScaler(),\n",
       "                                                  ['bill_length_mm',\n",
       "                                                   'bill_depth_mm',\n",
       "                                                   'flipper_length_mm',\n",
       "                                                   'body_mass_g'])],\n",
       "                                   verbose_feature_names_out=False))],\n",
       "         verbose=True)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_pipe = preprocessor.return_pipeline()\n",
    "my_pipe"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49c04a54-eed9-4dff-9143-12a112983ba5",
   "metadata": {},
   "source": [
    "#### You can also use a fitted pipeline to transform new data.\n",
    "\n",
    "Again, you don't have to clean up the dataframe, you've decided what columns you want to keep and as long as all of those columns are present in your new dataframe it will transform the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "83abddab-7d1c-4510-9d05-34d9e0f1fedc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bill_length_mm</th>\n",
       "      <th>bill_depth_mm</th>\n",
       "      <th>flipper_length_mm</th>\n",
       "      <th>body_mass_g</th>\n",
       "      <th>island_Biscoe</th>\n",
       "      <th>island_Dream</th>\n",
       "      <th>island_Torgersen</th>\n",
       "      <th>sex_Female</th>\n",
       "      <th>sex_Male</th>\n",
       "      <th>sex_rare</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>0.992518</td>\n",
       "      <td>0.910420</td>\n",
       "      <td>0.314286</td>\n",
       "      <td>0.144550</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>0.602892</td>\n",
       "      <td>0.635488</td>\n",
       "      <td>0.371429</td>\n",
       "      <td>0.334123</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225</th>\n",
       "      <td>0.692099</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.618483</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>0.602892</td>\n",
       "      <td>0.403386</td>\n",
       "      <td>0.171429</td>\n",
       "      <td>0.002370</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>318</th>\n",
       "      <td>0.822105</td>\n",
       "      <td>0.030579</td>\n",
       "      <td>0.514286</td>\n",
       "      <td>0.654028</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>0.160449</td>\n",
       "      <td>0.595370</td>\n",
       "      <td>0.171429</td>\n",
       "      <td>0.014218</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>0.637228</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.828571</td>\n",
       "      <td>0.547393</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>0.076838</td>\n",
       "      <td>0.910420</td>\n",
       "      <td>0.257143</td>\n",
       "      <td>0.239336</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>0.630363</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.171429</td>\n",
       "      <td>0.642180</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>0.985715</td>\n",
       "      <td>0.780464</td>\n",
       "      <td>0.457143</td>\n",
       "      <td>0.571090</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>114 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     bill_length_mm  bill_depth_mm  flipper_length_mm  body_mass_g  \\\n",
       "194        0.992518       0.910420           0.314286     0.144550   \n",
       "157        0.602892       0.635488           0.371429     0.334123   \n",
       "225        0.692099       0.000000           0.714286     0.618483   \n",
       "208        0.602892       0.403386           0.171429     0.002370   \n",
       "318        0.822105       0.030579           0.514286     0.654028   \n",
       "..              ...            ...                ...          ...   \n",
       "126        0.160449       0.595370           0.171429     0.014218   \n",
       "282        0.637228       0.000000           0.828571     0.547393   \n",
       "59         0.076838       0.910420           0.257143     0.239336   \n",
       "111        0.630363       1.000000           0.171429     0.642180   \n",
       "197        0.985715       0.780464           0.457143     0.571090   \n",
       "\n",
       "     island_Biscoe  island_Dream  island_Torgersen  sex_Female  sex_Male  \\\n",
       "194            0.0           1.0               0.0         0.0       1.0   \n",
       "157            0.0           1.0               0.0         1.0       0.0   \n",
       "225            1.0           0.0               0.0         1.0       0.0   \n",
       "208            0.0           1.0               0.0         1.0       0.0   \n",
       "318            1.0           0.0               0.0         1.0       0.0   \n",
       "..             ...           ...               ...         ...       ...   \n",
       "126            0.0           0.0               1.0         1.0       0.0   \n",
       "282            1.0           0.0               0.0         1.0       0.0   \n",
       "59             1.0           0.0               0.0         0.0       1.0   \n",
       "111            1.0           0.0               0.0         0.0       1.0   \n",
       "197            0.0           1.0               0.0         0.0       1.0   \n",
       "\n",
       "     sex_rare  \n",
       "194       0.0  \n",
       "157       0.0  \n",
       "225       0.0  \n",
       "208       0.0  \n",
       "318       0.0  \n",
       "..        ...  \n",
       "126       0.0  \n",
       "282       0.0  \n",
       "59        0.0  \n",
       "111       0.0  \n",
       "197       0.0  \n",
       "\n",
       "[114 rows x 10 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transform_test_df = preprocessor.transform_new_data(X_test)\n",
    "transform_test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "98155d1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Adelie       152\n",
       "Gentoo       124\n",
       "Chinstrap     68\n",
       "dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(y).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd525de8-c4a7-4e3c-89f1-6ddad873dda6",
   "metadata": {},
   "source": [
    "#### Finally, you can save the fitted preprocessor in a pickle file to be loaded later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f5f5460b-7ae4-4f06-8c61-5ea212555205",
   "metadata": {},
   "outputs": [],
   "source": [
    "# to save fitted class as pickle\n",
    "with open('my_preprocesser.pkl', 'wb') as file:\n",
    "    pickle.dump(preprocessor, file)\n",
    "    \n",
    "# to load pickled preprocessor later\n",
    "#with open('my_preprocesser.pkl', 'rb') as file:\n",
    "#    preprocessor = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b64c5232",
   "metadata": {},
   "outputs": [],
   "source": [
    "# to load pickled preprocessor later\n",
    "with open('my_preprocesser.pkl', 'rb') as file:\n",
    "    preprocessor = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d0fc926",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6778f03d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c6c54b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# dev work\n",
    "preprocessor.float_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8947a2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "\n",
    "def select_caps(df, list_cols):\n",
    "    quant_decision_dict = {}\n",
    "    val_decision_dict = {}\n",
    "    for c in list_cols:\n",
    "        clear_output(wait=True)\n",
    "        temp_vals = df[c].dropna()\n",
    "        min_val = np.min(temp_vals)\n",
    "        max_val = np.max(temp_vals)\n",
    "        bot_5 = np.percentile(temp_vals, 5)\n",
    "        bot_10 = np.percentile(temp_vals, 10)\n",
    "        top_95 = np.percentile(temp_vals, 95)\n",
    "        top_90 = np.percentile(temp_vals, 90)\n",
    "\n",
    "        plt.hist(temp_vals, bins=40)\n",
    "        plt.axvline(x=bot_5, color='b', ls='--', label='5th percentile: {}'.format(bot_5))\n",
    "        plt.axvline(x=bot_10, color='r', ls='-', label='10th percentile: {}'.format(bot_10))\n",
    "        plt.axvline(x=top_90, color='purple', ls='-.', label='90th percentile: {}'.format(top_90))\n",
    "        plt.axvline(x=top_95, color='orange', ls=':', label='95th percentile: {}'.format(top_95))\n",
    "        plt.xticks(rotation=45, ha=\"right\")\n",
    "        plt.legend(bbox_to_anchor=(1.0, 1), loc='upper left')\n",
    "        plt.title(c)\n",
    "        plt.show()\n",
    "        cap_decision_lower = float(input(\"Input lower cap (percent or value): \"))\n",
    "        cap_decision_upper = float(input(\"Input upper cap (percent of value): \"))\n",
    "        assert cap_decision_lower < cap_decision_upper, \"upper bound can't be smaller than lower bound\"\n",
    "\n",
    "        if (cap_decision_lower < 0) or (cap_decision_lower > 1) or (cap_decision_upper < 0) or (cap_decision_upper > 1):\n",
    "            val_decision_dict[c] = [float(cap_decision_lower), float(cap_decision_upper)]\n",
    "        elif ((min_val >= 0) and (min_val < 1)) and ((max_val >= 0) and (max_val <= 1)):\n",
    "            val_or_quant = 'nothing yet'\n",
    "            while val_or_quant not in ['quantiles', 'values']:\n",
    "                val_or_quant = input(\"Are cap decisions 'quantiles' or 'values'?\")\n",
    "            if val_or_quant == 'quantiles':\n",
    "                quant_decision_dict[c] = [cap_decision_lower, cap_decision_upper]\n",
    "            else: \n",
    "                val_decision_dict[c] = [cap_decision_lower, cap_decision_upper]\n",
    "        else:\n",
    "            quant_decision_dict[c] = [cap_decision_lower, cap_decision_upper]\n",
    "\n",
    "    if quant_decision_dict == {}:\n",
    "        quant_decision_dict = None\n",
    "    if val_decision_dict == {}:\n",
    "        val_decision_dict = None\n",
    "\n",
    "    return quant_decision_dict, val_decision_dict\n",
    "\n",
    "select_caps(my_df, preprocessor.float_cols)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "201ff686",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.4 ('lead_scoring_CSC')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "vscode": {
   "interpreter": {
    "hash": "b2db10dbc859dd672a9104f53bc0f77829de7539fb5f3454783897e1ccfd6ca4"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}