pipeline_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "af9927bb-80aa-4862-9cb6-881cafa895f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.base import BaseEstimator, TransformerMixin, ClassifierMixin\n",
    "import statsmodels.api as sm\n",
    "from optbinning import OptimalBinning\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Settings\n",
    "pd.set_option('display.max_colwidth', None)\n",
    "pd.set_option('display.max_rows', None)\n",
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "821f5554-03b7-46e0-86d2-edd9bc735001",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize an empty DataFrame\n",
    "data = pd.DataFrame()\n",
    "\n",
    "# Generate data for 10 years\n",
    "for year in range(2010, 2020):\n",
    "    # 10 features, each with 100 observations per year\n",
    "    year_data = pd.DataFrame({\n",
    "        'year': [year]*100,\n",
    "        'feature1': np.random.rand(100),\n",
    "        'feature2': np.random.rand(100),\n",
    "        'feature3': np.random.rand(100),\n",
    "        'feature4': np.random.rand(100),\n",
    "        'feature5': np.random.rand(100),\n",
    "        'feature6': np.random.rand(100),\n",
    "        'feature7': np.random.rand(100),\n",
    "        'feature8': np.random.rand(100),\n",
    "        'feature9': np.random.rand(100),\n",
    "        'feature10': np.random.rand(100)\n",
    "    })\n",
    "\n",
    "    # Introduce NaNs into some features\n",
    "    for feature in ['feature2', 'feature4', 'feature7']:\n",
    "        nan_indices = np.random.choice(year_data.index, size=int(0.2 * len(year_data)), replace=False)\n",
    "        year_data.loc[nan_indices, feature] = np.nan\n",
    "\n",
    "    # Generate target variable based on some function of the features\n",
    "    # For example, let y = 1 when the sum of all features is greater than 5, else y = 0\n",
    "    year_data['target'] = np.where(year_data.iloc[:, 1:-1].sum(axis=1) > 5, 1, 0)\n",
    "\n",
    "    # Concatenate with the main DataFrame\n",
    "    data = pd.concat([data, year_data], ignore_index=True)\n",
    "\n",
    "# Create segmentation\n",
    "segments = ['segment_1', 'segment_2', 'segment_3']\n",
    "data['segment'] = np.random.choice(segments, size=len(data))\n",
    "\n",
    "# Separate features and target variable\n",
    "X = data.drop('target', axis=1)\n",
    "y = data['target']\n",
    "\n",
    "# Split the data into development (training) and test sets\n",
    "X_dev, X_test, y_dev, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5af5828b-6e3c-4972-90f4-ae3fae3550bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom transformer to drop columns with more than 80% missing values\n",
    "class DropMissing(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, threshold=0.8):\n",
    "        self.threshold = threshold\n",
    "        self.missing_table = pd.DataFrame()\n",
    "\n",
    "    def fit(self, X, y=None):\n",
    "        missing_ratio = X.isnull().mean()\n",
    "        self.to_keep = missing_ratio[missing_ratio < self.threshold].index.tolist()\n",
    "\n",
    "        # Store the percentage of missing values per variable in a table\n",
    "        self.missing_table = pd.DataFrame({\n",
    "            'Variable': missing_ratio.index,\n",
    "            'Missing_Percentage': missing_ratio.values * 100\n",
    "        })\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        return X[self.to_keep]\n",
    "\n",
    "# Custom transformer for OptimalBinning\n",
    "class OptimalBinningTransform(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, y):\n",
    "        self.y = y\n",
    "        self.binning_dict = {}\n",
    "        self.iv_table = pd.DataFrame(columns=['Variable', 'Information Value'])\n",
    "\n",
    "    def fit(self, X, y=None):\n",
    "        for col in X.columns:\n",
    "            optb = OptimalBinning(name=col)\n",
    "\n",
    "            # print(7, len(X[col]), len(self.y))\n",
    "            \n",
    "            optb.fit(X[col], self.y)\n",
    "            self.binning_dict[col] = optb\n",
    "            binning_table = optb.binning_table.build()\n",
    "\n",
    "            temp_df = pd.DataFrame({'Variable': [col], 'Information Value': binning_table.loc['Totals', 'IV']})\n",
    "            self.iv_table = pd.concat([self.iv_table, temp_df], ignore_index=True)\n",
    "     \n",
    "            \n",
    "        # Sort by IV and select top variables\n",
    "        self.iv_table.sort_values(by=\"Information Value\", ascending=False, inplace=True)\n",
    "        self.top_cols = self.iv_table.head(5)[\"Variable\"].tolist()\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        X_transformed = pd.DataFrame()\n",
    "        for col in self.top_cols:\n",
    "            optb = self.binning_dict[col]\n",
    "            X_transformed[col] = optb.transform(X[col], metric=\"indices\")\n",
    "\n",
    "        return X_transformed\n",
    "\n",
    "def gini_fn(y_true, y_score):\n",
    "    fpr, tpr, thresholds = roc_curve(y_true, y_score)\n",
    "    roc_auc = auc(fpr, tpr)\n",
    "    return 2 * roc_auc - 1\n",
    "    \n",
    "# Custom transformer for Statsmodels Logistic Regression\n",
    "class StatsmodelsLogit(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self):\n",
    "        self.model = None\n",
    "        self.result = None\n",
    "        self.gini_value = None\n",
    "        self.explanatory_vars = None\n",
    "    \n",
    "    def fit(self, X, y):\n",
    "        # print(222, X)\n",
    "        X = sm.add_constant(X)\n",
    "        self.model = sm.Logit(y, X).fit(disp=0)\n",
    "        self.explanatory_vars = self.model.params.index.tolist()\n",
    "\n",
    "        # Calculate Gini coefficient after fitting the model\n",
    "        y_score = self.model.predict(X)\n",
    "        self.gini_value = gini_fn(y, y_score)\n",
    "        \n",
    "        return self\n",
    "\n",
    "    def predict(self, X):\n",
    "        X = sm.add_constant(X)\n",
    "        return self.model.predict(X)\n",
    "\n",
    "    def get_gini(self):\n",
    "        return self.gini_value\n",
    "\n",
    "def calculate_moc_c(row: pd.Series, years: list) -> float:\n",
    "    values = row[years]\n",
    "    std_dev_mean = np.std(values) / np.sqrt(len(values))\n",
    "    avg_default_rate = np.mean(values)\n",
    "    moc_c = std_dev_mean / avg_default_rate if avg_default_rate != 0 else np.nan\n",
    "\n",
    "    return moc_c\n",
    "\n",
    "\n",
    "def calculate_annual_defaults(df, target_col='target', predictions_col='predictions'):\n",
    "    # Create bins for predicted PD\n",
    "    x = df[predictions_col].values\n",
    "    y = df[target_col].values\n",
    "    \n",
    "    optb = OptimalBinning(name=predictions_col, dtype=\"numerical\", solver=\"cp\")\n",
    "    optb.fit(x, y)\n",
    "    df['bin_number'] = optb.transform(x, metric=\"indices\")\n",
    "\n",
    "    \n",
    "    \n",
    "    # Find columns that look like years\n",
    "\n",
    "    \n",
    "    \n",
    "    # Show default rates per bin\n",
    "    annual_defaults = df.pivot_table(index='bin_number', columns='year', values=target_col, aggfunc='mean', margins=True)\n",
    "\n",
    "\n",
    "    year_col = [col for col in annual_defaults.columns if str(col).isdigit() and len(str(col)) == 4]\n",
    "\n",
    "    # print(22, year_col, annual_defaults.columns)\n",
    "\n",
    "\n",
    "    annual_defaults['annual_mean'] = annual_defaults[year_col].mean(axis=1)\n",
    "\n",
    "    # print(6, annual_defaults.head())\n",
    "\n",
    "    # Compare realized and predicted default rates\n",
    "    annual_defaults = annual_defaults.reset_index()\n",
    "    df = df.merge(annual_defaults[['bin_number', 'annual_mean']], on='bin_number', how='left')\n",
    "    \n",
    "    return df, annual_defaults"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "08bf0fe9-ea99-4222-87c6-16dfe1f197b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create pipelines\n",
    "all_steps = {\n",
    "    'drop_missing': DropMissing,\n",
    "    'optbinning': OptimalBinningTransform,\n",
    "    'logit_model': StatsmodelsLogit,\n",
    "    # 'step4': Step4,\n",
    "    # 'step5': Step5,\n",
    "    # ... add all your 10 steps here\n",
    "}\n",
    "\n",
    "def create_pipeline(step_param_dict):\n",
    "    # Extract the label if it exists; otherwise, set to None\n",
    "    label = step_param_dict.pop('label', None)\n",
    "    steps = [(name, all_steps[name](**params)) for name, params in step_param_dict.items()]\n",
    "    pipeline_instancee = Pipeline(steps)\n",
    "    \n",
    "    return (label, pipeline_instancee)\n",
    "\n",
    "pipeline_config = [\n",
    "    {\n",
    "        'label': 'Pipeline_1',\n",
    "        'drop_missing': {'threshold': 0.9},\n",
    "        'optbinning': {'y': y}, # PROBLEM\n",
    "        'logit_model': {}\n",
    "    },\n",
    "    {\n",
    "        'label': 'Pipeline_2',\n",
    "        'drop_missing': {'threshold': 0.9},\n",
    "        'optbinning': {'y': y},\n",
    "        'logit_model': {}\n",
    "    },\n",
    "    # Add more configurations here...\n",
    "]\n",
    "\n",
    "# pipelines = [create_pipeline(config) for config in pipeline_config]\n",
    "\n",
    "def update_config(option, dynamic_params):\n",
    "    updated_option = option.copy()\n",
    "    for step, params in updated_option.items():\n",
    "        if isinstance(params, dict):\n",
    "            for key, value in params.items():\n",
    "                if key in dynamic_params:\n",
    "                    params[key] = dynamic_params[key]\n",
    "    return updated_option"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bfd27e96-934b-4e04-b876-33d8c8b68da5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>segment</th>\n",
       "      <th>gini</th>\n",
       "      <th>moc_c</th>\n",
       "      <th>explanatory_vars</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Pipeline_1</td>\n",
       "      <td>segment_1</td>\n",
       "      <td>0.818524</td>\n",
       "      <td>0.308072</td>\n",
       "      <td>[const, feature3, feature1, feature8, feature7, feature5]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Pipeline_2</td>\n",
       "      <td>segment_1</td>\n",
       "      <td>0.818524</td>\n",
       "      <td>0.308072</td>\n",
       "      <td>[const, feature3, feature1, feature8, feature7, feature5]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Pipeline_1</td>\n",
       "      <td>segment_2</td>\n",
       "      <td>0.861663</td>\n",
       "      <td>0.430100</td>\n",
       "      <td>[const, feature6, feature4, feature8, feature9, feature2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Pipeline_2</td>\n",
       "      <td>segment_2</td>\n",
       "      <td>0.861663</td>\n",
       "      <td>0.430100</td>\n",
       "      <td>[const, feature6, feature4, feature8, feature9, feature2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Pipeline_1</td>\n",
       "      <td>segment_3</td>\n",
       "      <td>0.844464</td>\n",
       "      <td>0.319135</td>\n",
       "      <td>[const, feature5, feature8, feature7, feature9, feature4]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Pipeline_2</td>\n",
       "      <td>segment_3</td>\n",
       "      <td>0.844464</td>\n",
       "      <td>0.319135</td>\n",
       "      <td>[const, feature5, feature8, feature7, feature9, feature4]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id    segment      gini     moc_c  \\\n",
       "0  Pipeline_1  segment_1  0.818524  0.308072   \n",
       "1  Pipeline_2  segment_1  0.818524  0.308072   \n",
       "2  Pipeline_1  segment_2  0.861663  0.430100   \n",
       "3  Pipeline_2  segment_2  0.861663  0.430100   \n",
       "4  Pipeline_1  segment_3  0.844464  0.319135   \n",
       "5  Pipeline_2  segment_3  0.844464  0.319135   \n",
       "\n",
       "                                            explanatory_vars  \n",
       "0  [const, feature3, feature1, feature8, feature7, feature5]  \n",
       "1  [const, feature3, feature1, feature8, feature7, feature5]  \n",
       "2  [const, feature6, feature4, feature8, feature9, feature2]  \n",
       "3  [const, feature6, feature4, feature8, feature9, feature2]  \n",
       "4  [const, feature5, feature8, feature7, feature9, feature4]  \n",
       "5  [const, feature5, feature8, feature7, feature9, feature4]  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Run pipelines\n",
    "model_data = []\n",
    "final_portfolio = []\n",
    "\n",
    "# dynamic_keys = ['y']\n",
    "\n",
    "for segment in segments:\n",
    "    # Create segment datasets\n",
    "    data_pipeline = data[data['segment'] == segment].reset_index(drop=True).copy()\n",
    "    y = data_pipeline['target']\n",
    "    X = data_pipeline.drop(['segment', 'target'], axis=1) # todo\n",
    "\n",
    "    # Update pipeline parameters\n",
    "    dynamic_params = {\n",
    "        'y': y,\n",
    "        # 'input2': input2,\n",
    "    }\n",
    "\n",
    "    for option in pipeline_config:\n",
    "        \n",
    "         # Update the pipeline configuration dynamically\n",
    "        updated_option = update_config(option, dynamic_params)\n",
    "\n",
    "        # Create and run the pipeline\n",
    "        label, pipeline = create_pipeline(updated_option)\n",
    "        \n",
    "        pipeline.fit(X, y)\n",
    "\n",
    "        # Make prediction\n",
    "        data_pipeline['predictions'] = pipeline.predict(X)\n",
    "        final_portfolio.append(data_pipeline)\n",
    "\n",
    "        # Calibration\n",
    "        data_pipeline, annual_defaults = calculate_annual_defaults(data_pipeline, target_col='target', predictions_col='predictions')\n",
    "        \n",
    "        # Calculate MoC C\n",
    "        year_col = [col for col in annual_defaults.columns if str(col).isdigit() and len(str(col)) == 4]\n",
    "        annual_defaults['moc_c'] = annual_defaults.apply(lambda row: calculate_moc_c(row, year_col), axis=1) #.reset_index()\n",
    "    \n",
    "        # Retrieve data from individual steps\n",
    "        missings = pipeline.named_steps['drop_missing'].missing_table\n",
    "        gini = pipeline.named_steps['logit_model'].gini_value\n",
    "        explanatory_vars = pipeline.named_steps['logit_model'].explanatory_vars\n",
    "    \n",
    "        # Add model-specific tests to a dataframe\n",
    "        model_data.append([label, segment, gini, annual_defaults['moc_c'].mean(), explanatory_vars])\n",
    "\n",
    "# Concatenate portfolios\n",
    "result = pd.concat(final_portfolio, axis=0)\n",
    "    \n",
    "# Create a summary table\n",
    "summary_df = pd.DataFrame(model_data, columns=['id', 'segment', 'gini', 'moc_c', 'explanatory_vars'])\n",
    "\n",
    "summary_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "49de65f4-e4d1-4ad4-924c-502909568bb6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>feature1</th>\n",
       "      <th>feature2</th>\n",
       "      <th>feature3</th>\n",
       "      <th>feature4</th>\n",
       "      <th>feature5</th>\n",
       "      <th>feature6</th>\n",
       "      <th>feature7</th>\n",
       "      <th>feature8</th>\n",
       "      <th>feature9</th>\n",
       "      <th>feature10</th>\n",
       "      <th>target</th>\n",
       "      <th>segment</th>\n",
       "      <th>predictions</th>\n",
       "      <th>bin_number</th>\n",
       "      <th>annual_mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2010</td>\n",
       "      <td>0.537677</td>\n",
       "      <td>0.120459</td>\n",
       "      <td>0.045511</td>\n",
       "      <td>0.803571</td>\n",
       "      <td>0.628494</td>\n",
       "      <td>0.028617</td>\n",
       "      <td>0.278457</td>\n",
       "      <td>0.329940</td>\n",
       "      <td>0.507032</td>\n",
       "      <td>0.004914</td>\n",
       "      <td>0</td>\n",
       "      <td>segment_1</td>\n",
       "      <td>0.015143</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2010</td>\n",
       "      <td>0.191056</td>\n",
       "      <td>0.583340</td>\n",
       "      <td>0.869528</td>\n",
       "      <td>0.348889</td>\n",
       "      <td>0.572634</td>\n",
       "      <td>0.810088</td>\n",
       "      <td>0.295353</td>\n",
       "      <td>0.034755</td>\n",
       "      <td>0.079333</td>\n",
       "      <td>0.127750</td>\n",
       "      <td>0</td>\n",
       "      <td>segment_1</td>\n",
       "      <td>0.024983</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2010</td>\n",
       "      <td>0.674544</td>\n",
       "      <td>0.496049</td>\n",
       "      <td>0.066517</td>\n",
       "      <td>0.385747</td>\n",
       "      <td>0.463567</td>\n",
       "      <td>0.621302</td>\n",
       "      <td>0.046434</td>\n",
       "      <td>0.017110</td>\n",
       "      <td>0.291355</td>\n",
       "      <td>0.292285</td>\n",
       "      <td>0</td>\n",
       "      <td>segment_1</td>\n",
       "      <td>0.001777</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2010</td>\n",
       "      <td>0.233640</td>\n",
       "      <td>0.805590</td>\n",
       "      <td>0.693687</td>\n",
       "      <td>0.498214</td>\n",
       "      <td>0.476364</td>\n",
       "      <td>0.203336</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.389557</td>\n",
       "      <td>0.752468</td>\n",
       "      <td>0.616470</td>\n",
       "      <td>0</td>\n",
       "      <td>segment_1</td>\n",
       "      <td>0.008617</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2010</td>\n",
       "      <td>0.188479</td>\n",
       "      <td>0.631598</td>\n",
       "      <td>0.354367</td>\n",
       "      <td>0.600386</td>\n",
       "      <td>0.090397</td>\n",
       "      <td>0.993964</td>\n",
       "      <td>0.753858</td>\n",
       "      <td>0.331632</td>\n",
       "      <td>0.717383</td>\n",
       "      <td>0.769003</td>\n",
       "      <td>0</td>\n",
       "      <td>segment_1</td>\n",
       "      <td>0.005950</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   year  feature1  feature2  feature3  feature4  feature5  feature6  feature7  \\\n",
       "0  2010  0.537677  0.120459  0.045511  0.803571  0.628494  0.028617  0.278457   \n",
       "1  2010  0.191056  0.583340  0.869528  0.348889  0.572634  0.810088  0.295353   \n",
       "2  2010  0.674544  0.496049  0.066517  0.385747  0.463567  0.621302  0.046434   \n",
       "3  2010  0.233640  0.805590  0.693687  0.498214  0.476364  0.203336       NaN   \n",
       "4  2010  0.188479  0.631598  0.354367  0.600386  0.090397  0.993964  0.753858   \n",
       "\n",
       "   feature8  feature9  feature10  target    segment  predictions  bin_number  \\\n",
       "0  0.329940  0.507032   0.004914       0  segment_1     0.015143           0   \n",
       "1  0.034755  0.079333   0.127750       0  segment_1     0.024983           0   \n",
       "2  0.017110  0.291355   0.292285       0  segment_1     0.001777           0   \n",
       "3  0.389557  0.752468   0.616470       0  segment_1     0.008617           0   \n",
       "4  0.331632  0.717383   0.769003       0  segment_1     0.005950           0   \n",
       "\n",
       "   annual_mean  \n",
       "0          NaN  \n",
       "1          NaN  \n",
       "2          NaN  \n",
       "3          NaN  \n",
       "4          NaN  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8d704e1e-6400-4eff-9b01-240fb0186981",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>segment</th>\n",
       "      <th>gini</th>\n",
       "      <th>moc_c</th>\n",
       "      <th>explanatory_vars</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Pipeline_1</td>\n",
       "      <td>segment_1</td>\n",
       "      <td>0.818524</td>\n",
       "      <td>0.308072</td>\n",
       "      <td>[const, feature3, feature1, feature8, feature7, feature5]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Pipeline_2</td>\n",
       "      <td>segment_1</td>\n",
       "      <td>0.818524</td>\n",
       "      <td>0.308072</td>\n",
       "      <td>[const, feature3, feature1, feature8, feature7, feature5]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Pipeline_1</td>\n",
       "      <td>segment_2</td>\n",
       "      <td>0.861663</td>\n",
       "      <td>0.430100</td>\n",
       "      <td>[const, feature6, feature4, feature8, feature9, feature2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Pipeline_2</td>\n",
       "      <td>segment_2</td>\n",
       "      <td>0.861663</td>\n",
       "      <td>0.430100</td>\n",
       "      <td>[const, feature6, feature4, feature8, feature9, feature2]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Pipeline_1</td>\n",
       "      <td>segment_3</td>\n",
       "      <td>0.844464</td>\n",
       "      <td>0.319135</td>\n",
       "      <td>[const, feature5, feature8, feature7, feature9, feature4]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Pipeline_2</td>\n",
       "      <td>segment_3</td>\n",
       "      <td>0.844464</td>\n",
       "      <td>0.319135</td>\n",
       "      <td>[const, feature5, feature8, feature7, feature9, feature4]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           id    segment      gini     moc_c  \\\n",
       "0  Pipeline_1  segment_1  0.818524  0.308072   \n",
       "1  Pipeline_2  segment_1  0.818524  0.308072   \n",
       "2  Pipeline_1  segment_2  0.861663  0.430100   \n",
       "3  Pipeline_2  segment_2  0.861663  0.430100   \n",
       "4  Pipeline_1  segment_3  0.844464  0.319135   \n",
       "5  Pipeline_2  segment_3  0.844464  0.319135   \n",
       "\n",
       "                                            explanatory_vars  \n",
       "0  [const, feature3, feature1, feature8, feature7, feature5]  \n",
       "1  [const, feature3, feature1, feature8, feature7, feature5]  \n",
       "2  [const, feature6, feature4, feature8, feature9, feature2]  \n",
       "3  [const, feature6, feature4, feature8, feature9, feature2]  \n",
       "4  [const, feature5, feature8, feature7, feature9, feature4]  \n",
       "5  [const, feature5, feature8, feature7, feature9, feature4]  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary_df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "22fb207a-1e1a-44a8-bc69-3b8fb1224fa6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>year</th>\n",
       "      <th>bin_number</th>\n",
       "      <th>2010</th>\n",
       "      <th>2011</th>\n",
       "      <th>2012</th>\n",
       "      <th>2013</th>\n",
       "      <th>2014</th>\n",
       "      <th>2015</th>\n",
       "      <th>2016</th>\n",
       "      <th>2017</th>\n",
       "      <th>2018</th>\n",
       "      <th>2019</th>\n",
       "      <th>All</th>\n",
       "      <th>annual_mean</th>\n",
       "      <th>moc_c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.007407</td>\n",
       "      <td>0.011111</td>\n",
       "      <td>0.948683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.040000</td>\n",
       "      <td>0.632456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.130844</td>\n",
       "      <td>0.237409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.75</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.558824</td>\n",
       "      <td>0.633333</td>\n",
       "      <td>0.160289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.60</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.710417</td>\n",
       "      <td>0.157068</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "year bin_number      2010  2011  2012      2013      2014      2015  2016  \\\n",
       "0             0  0.111111  0.00   0.0  0.000000  0.000000  0.000000  0.00   \n",
       "1             1  0.000000  0.00   0.0  0.200000  0.000000  0.200000  0.00   \n",
       "2             2  0.100000  0.00   0.0  0.142857  0.272727  0.200000  0.25   \n",
       "3             3  0.750000  0.75   1.0  0.333333  0.666667  0.333333  0.50   \n",
       "4             4  0.000000  0.60   1.0       NaN       NaN  1.000000  1.00   \n",
       "\n",
       "year  2017      2018      2019       All  annual_mean     moc_c  \n",
       "0     0.00  0.000000  0.000000  0.007407     0.011111  0.948683  \n",
       "1     0.00  0.000000  0.000000  0.045455     0.040000  0.632456  \n",
       "2     0.20  0.000000  0.142857  0.142857     0.130844  0.237409  \n",
       "3     0.00  1.000000  1.000000  0.558824     0.633333  0.160289  \n",
       "4     0.75  0.333333  1.000000  0.700000     0.710417  0.157068  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annual_defaults.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "008d74db-be8e-420c-a17b-8949e4ce6f43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>feature1</th>\n",
       "      <th>feature2</th>\n",
       "      <th>feature3</th>\n",
       "      <th>feature4</th>\n",
       "      <th>feature5</th>\n",
       "      <th>feature6</th>\n",
       "      <th>feature7</th>\n",
       "      <th>feature8</th>\n",
       "      <th>feature9</th>\n",
       "      <th>feature10</th>\n",
       "      <th>target</th>\n",
       "      <th>segment</th>\n",
       "      <th>predictions</th>\n",
       "      <th>bin_number</th>\n",
       "      <th>annual_mean_x</th>\n",
       "      <th>annual_mean_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2010</td>\n",
       "      <td>0.351130</td>\n",
       "      <td>0.420757</td>\n",
       "      <td>0.654969</td>\n",
       "      <td>0.091013</td>\n",
       "      <td>0.017034</td>\n",
       "      <td>0.065540</td>\n",
       "      <td>0.923099</td>\n",
       "      <td>0.919340</td>\n",
       "      <td>0.828352</td>\n",
       "      <td>0.741384</td>\n",
       "      <td>0</td>\n",
       "      <td>segment_3</td>\n",
       "      <td>0.145456</td>\n",
       "      <td>2</td>\n",
       "      <td>0.130844</td>\n",
       "      <td>0.130844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2010</td>\n",
       "      <td>0.683317</td>\n",
       "      <td>0.886494</td>\n",
       "      <td>0.563958</td>\n",
       "      <td>0.719239</td>\n",
       "      <td>0.907448</td>\n",
       "      <td>0.126884</td>\n",
       "      <td>0.386255</td>\n",
       "      <td>0.352947</td>\n",
       "      <td>0.267354</td>\n",
       "      <td>0.046298</td>\n",
       "      <td>0</td>\n",
       "      <td>segment_3</td>\n",
       "      <td>0.097787</td>\n",
       "      <td>2</td>\n",
       "      <td>0.130844</td>\n",
       "      <td>0.130844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2010</td>\n",
       "      <td>0.650683</td>\n",
       "      <td>0.975524</td>\n",
       "      <td>0.175502</td>\n",
       "      <td>0.082543</td>\n",
       "      <td>0.184788</td>\n",
       "      <td>0.237935</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.540740</td>\n",
       "      <td>0.578799</td>\n",
       "      <td>0.201813</td>\n",
       "      <td>0</td>\n",
       "      <td>segment_3</td>\n",
       "      <td>0.004692</td>\n",
       "      <td>0</td>\n",
       "      <td>0.011111</td>\n",
       "      <td>0.011111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2010</td>\n",
       "      <td>0.516448</td>\n",
       "      <td>0.851496</td>\n",
       "      <td>0.892531</td>\n",
       "      <td>0.928364</td>\n",
       "      <td>0.892888</td>\n",
       "      <td>0.403039</td>\n",
       "      <td>0.677262</td>\n",
       "      <td>0.228250</td>\n",
       "      <td>0.602115</td>\n",
       "      <td>0.453546</td>\n",
       "      <td>1</td>\n",
       "      <td>segment_3</td>\n",
       "      <td>0.374361</td>\n",
       "      <td>3</td>\n",
       "      <td>0.633333</td>\n",
       "      <td>0.633333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2010</td>\n",
       "      <td>0.718342</td>\n",
       "      <td>0.107650</td>\n",
       "      <td>0.212696</td>\n",
       "      <td>0.863827</td>\n",
       "      <td>0.341108</td>\n",
       "      <td>0.137730</td>\n",
       "      <td>0.288029</td>\n",
       "      <td>0.587903</td>\n",
       "      <td>0.688958</td>\n",
       "      <td>0.779187</td>\n",
       "      <td>0</td>\n",
       "      <td>segment_3</td>\n",
       "      <td>0.099541</td>\n",
       "      <td>2</td>\n",
       "      <td>0.130844</td>\n",
       "      <td>0.130844</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   year  feature1  feature2  feature3  feature4  feature5  feature6  feature7  \\\n",
       "0  2010  0.351130  0.420757  0.654969  0.091013  0.017034  0.065540  0.923099   \n",
       "1  2010  0.683317  0.886494  0.563958  0.719239  0.907448  0.126884  0.386255   \n",
       "2  2010  0.650683  0.975524  0.175502  0.082543  0.184788  0.237935       NaN   \n",
       "3  2010  0.516448  0.851496  0.892531  0.928364  0.892888  0.403039  0.677262   \n",
       "4  2010  0.718342  0.107650  0.212696  0.863827  0.341108  0.137730  0.288029   \n",
       "\n",
       "   feature8  feature9  feature10  target    segment  predictions bin_number  \\\n",
       "0  0.919340  0.828352   0.741384       0  segment_3     0.145456          2   \n",
       "1  0.352947  0.267354   0.046298       0  segment_3     0.097787          2   \n",
       "2  0.540740  0.578799   0.201813       0  segment_3     0.004692          0   \n",
       "3  0.228250  0.602115   0.453546       1  segment_3     0.374361          3   \n",
       "4  0.587903  0.688958   0.779187       0  segment_3     0.099541          2   \n",
       "\n",
       "   annual_mean_x  annual_mean_y  \n",
       "0       0.130844       0.130844  \n",
       "1       0.130844       0.130844  \n",
       "2       0.011111       0.011111  \n",
       "3       0.633333       0.633333  \n",
       "4       0.130844       0.130844  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_pipeline.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3180c30b-ee90-45aa-be35-3d3703bd3eeb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}