train_model_Multi_inputs_NN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "065fd370",
   "metadata": {},
   "source": [
    "# Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ddd41789",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T05:32:36.915795Z",
     "start_time": "2021-10-16T05:32:35.300827Z"
    }
   },
   "outputs": [],
   "source": [
    "import joblib\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.preprocessing import LabelBinarizer, MinMaxScaler, RobustScaler, StandardScaler\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cfb42ca2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:09:09.490191Z",
     "start_time": "2021-10-16T06:09:09.479207Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.layers import InputLayer, Dense, Dropout, Conv1D, Conv2D, AveragePooling1D, AveragePooling2D, LeakyReLU\n",
    "from tensorflow.keras.layers import Input, Flatten, Reshape, Lambda, BatchNormalization, Conv2D\n",
    "from tensorflow.keras.layers import MaxPooling2D, GlobalAveragePooling2D, concatenate\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.regularizers import L2\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.applications.efficientnet import EfficientNetB1\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow_addons as tfa\n",
    "import tensorflow.keras.backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fe349301",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T05:45:07.003102Z",
     "start_time": "2021-10-16T05:45:06.735105Z"
    }
   },
   "outputs": [],
   "source": [
    "physical_devices = tf.config.list_physical_devices('GPU')\n",
    "tf.config.experimental.set_memory_growth(physical_devices[0], True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "63114370",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T05:45:07.065100Z",
     "start_time": "2021-10-16T05:45:07.050101Z"
    }
   },
   "outputs": [],
   "source": [
    "def r2_metric(y_true, y_pred):\n",
    "    numerator = K.sum((y_true - y_pred) ** 2, axis=0)\n",
    "    denominator = K.sum((y_true - K.mean(y_pred, axis=0)) ** 2, axis=0)\n",
    "    return 1-(numerator/denominator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9294b995",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T05:45:07.409105Z",
     "start_time": "2021-10-16T05:45:07.406102Z"
    }
   },
   "outputs": [],
   "source": [
    "def LMAE(y_actual, y_pred): \n",
    "    lmae=tf.experimental.numpy.log10(K.mean(K.abs(y_actual - y_pred), axis=0))\n",
    "    return lmae"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c884c60",
   "metadata": {},
   "source": [
    "# Tabular Input head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e4a45da4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T05:57:46.596328Z",
     "start_time": "2021-10-16T05:57:46.516329Z"
    }
   },
   "outputs": [],
   "source": [
    "# model accept input from tabular data\n",
    "tab_model = Sequential([\n",
    "    InputLayer(input_shape=(144,), dtype='float32'),\n",
    "    Dense(128, activation='relu'),\n",
    "    Dense(256, activation='relu'),\n",
    "    Dense(256, activation='relu', kernel_regularizer=L2(0.01)),\n",
    "    Dense(128, activation='relu'),\n",
    "    Dense(64, activation='relu', kernel_regularizer=L2(0.01)),\n",
    "    Dense(16, activation='relu'),\n",
    "    Dense(4, activation='relu', kernel_regularizer=L2(0.01)),\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "911365ec",
   "metadata": {},
   "source": [
    "# DeepInsight Image Input head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "86536c92",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T05:48:20.160571Z",
     "start_time": "2021-10-16T05:48:18.663315Z"
    }
   },
   "outputs": [],
   "source": [
    "# 1st architect with 256 batch size\n",
    "img_model = Sequential([\n",
    "    InputLayer(input_shape=(50,50,3), dtype='float32'),\n",
    "    Conv2D(32, 3, activation='relu', padding='same'),\n",
    "    Conv2D(64, 3, activation='relu', padding='same'),\n",
    "    Conv2D(64, 3, activation='relu', padding='same'),\n",
    "    BatchNormalization(),\n",
    "    MaxPooling2D((2,2)),\n",
    "    Dropout(0.25),\n",
    "    Conv2D(128, 3, activation='relu', padding='same'),\n",
    "    Conv2D(128, 3, activation='relu', padding='same'),\n",
    "    BatchNormalization(),\n",
    "    MaxPooling2D((2,2)),\n",
    "    Dropout(0.25),\n",
    "    Conv2D(256, 3, activation='relu', padding='same'),\n",
    "    Conv2D(256, 3, activation='relu', padding='same'),\n",
    "    BatchNormalization(),\n",
    "    MaxPooling2D((2,2)),\n",
    "    Dropout(0.25),\n",
    "    Flatten(),\n",
    "    Dense(512, activation='relu'),\n",
    "    Dense(256, activation='relu'),\n",
    "    BatchNormalization(),\n",
    "    Dropout(0.4),\n",
    "    Dense(64, activation='relu'),\n",
    "    BatchNormalization(),\n",
    "    Dense(16, activation='relu'),\n",
    "    Dense(4, activation='relu'),\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "879deb41",
   "metadata": {},
   "source": [
    "# Combine model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "91c98295",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:10:58.702285Z",
     "start_time": "2021-10-16T06:10:58.659288Z"
    }
   },
   "outputs": [],
   "source": [
    "x = concatenate([tab_model.output, img_model.output])\n",
    "x = Dense(4, activation='relu')(x)\n",
    "x = Dense(4, activation='relu')(x)\n",
    "x = Dense(1, activation='linear')(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1ac590bb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:10:59.104154Z",
     "start_time": "2021-10-16T06:10:59.086171Z"
    }
   },
   "outputs": [],
   "source": [
    "combined_model = Model(inputs=[tab_model.input, img_model.input], outputs=x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d89a716d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:11:09.648256Z",
     "start_time": "2021-10-16T06:11:09.614259Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 50, 50, 3)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d (Conv2D)                 (None, 50, 50, 32)   896         input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 50, 50, 64)   18496       conv2d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 50, 50, 64)   36928       conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, 50, 50, 64)   256         conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D)    (None, 25, 25, 64)   0           batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 25, 25, 64)   0           max_pooling2d[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 25, 25, 128)  73856       dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 25, 25, 128)  147584      conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 25, 25, 128)  512         conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, 12, 12, 128)  0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 12, 12, 128)  0           max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 12, 12, 256)  295168      dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 12, 12, 256)  590080      conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 12, 12, 256)  1024        conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)  (None, 6, 6, 256)    0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 6, 6, 256)    0           max_pooling2d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 9216)         0           dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "input_3 (InputLayer)            [(None, 144)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 512)          4719104     flatten[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 128)          18560       input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 256)          131328      dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 256)          33024       dense_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 256)          1024        dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 256)          65792       dense_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 256)          0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 128)          32896       dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 64)           16448       dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, 64)           8256        dense_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 64)           256         dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_10 (Dense)                (None, 16)           1040        dense_9[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 16)           1040        batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_11 (Dense)                (None, 4)            68          dense_10[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 4)            68          dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 8)            0           dense_11[0][0]                   \n",
      "                                                                 dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (None, 4)            36          concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_16 (Dense)                (None, 4)            20          dense_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_17 (Dense)                (None, 1)            5           dense_16[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 6,193,765\n",
      "Trainable params: 6,192,229\n",
      "Non-trainable params: 1,536\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "combined_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4eaea7b7",
   "metadata": {},
   "source": [
    "# Load and prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "bb9b511a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:24:11.410236Z",
     "start_time": "2021-10-16T06:24:10.688254Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>volume</th>\n",
       "      <th>volume/g</th>\n",
       "      <th>density</th>\n",
       "      <th>weight</th>\n",
       "      <th>surface_area</th>\n",
       "      <th>void_fraction</th>\n",
       "      <th>void_volume</th>\n",
       "      <th>CO2/N2_selectivity</th>\n",
       "      <th>heat_adsorption</th>\n",
       "      <th>n_atoms</th>\n",
       "      <th>...</th>\n",
       "      <th>topology_1</th>\n",
       "      <th>topology_2</th>\n",
       "      <th>topology_3</th>\n",
       "      <th>topology_4</th>\n",
       "      <th>topology_5</th>\n",
       "      <th>topology_6</th>\n",
       "      <th>topology_7</th>\n",
       "      <th>topology_8</th>\n",
       "      <th>topology_9</th>\n",
       "      <th>topology_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.524386</td>\n",
       "      <td>-0.584699</td>\n",
       "      <td>0.798882</td>\n",
       "      <td>-0.383085</td>\n",
       "      <td>-0.257582</td>\n",
       "      <td>-0.872930</td>\n",
       "      <td>-0.642857</td>\n",
       "      <td>0.157631</td>\n",
       "      <td>0.609280</td>\n",
       "      <td>-0.406977</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.282926</td>\n",
       "      <td>-0.615547</td>\n",
       "      <td>0.856904</td>\n",
       "      <td>0.949117</td>\n",
       "      <td>-0.604123</td>\n",
       "      <td>-0.588382</td>\n",
       "      <td>-0.513449</td>\n",
       "      <td>0.691708</td>\n",
       "      <td>0.845745</td>\n",
       "      <td>0.976744</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.537500</td>\n",
       "      <td>-0.411402</td>\n",
       "      <td>0.509126</td>\n",
       "      <td>-0.484315</td>\n",
       "      <td>-0.491038</td>\n",
       "      <td>-0.536250</td>\n",
       "      <td>-0.447101</td>\n",
       "      <td>-0.021201</td>\n",
       "      <td>0.322523</td>\n",
       "      <td>-0.325581</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.007296</td>\n",
       "      <td>-0.043730</td>\n",
       "      <td>0.045099</td>\n",
       "      <td>0.044946</td>\n",
       "      <td>-0.091623</td>\n",
       "      <td>-0.201260</td>\n",
       "      <td>-0.160789</td>\n",
       "      <td>0.298554</td>\n",
       "      <td>0.219176</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.514063</td>\n",
       "      <td>-0.603040</td>\n",
       "      <td>0.833117</td>\n",
       "      <td>-0.356674</td>\n",
       "      <td>-0.401136</td>\n",
       "      <td>-0.878771</td>\n",
       "      <td>-0.647639</td>\n",
       "      <td>0.512156</td>\n",
       "      <td>0.407682</td>\n",
       "      <td>-0.186047</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68597</th>\n",
       "      <td>-0.489396</td>\n",
       "      <td>-0.701596</td>\n",
       "      <td>1.030967</td>\n",
       "      <td>-0.257028</td>\n",
       "      <td>-1.162504</td>\n",
       "      <td>1.944400</td>\n",
       "      <td>0.590556</td>\n",
       "      <td>0.220592</td>\n",
       "      <td>-2.777374</td>\n",
       "      <td>0.104651</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68598</th>\n",
       "      <td>-0.333898</td>\n",
       "      <td>-0.933076</td>\n",
       "      <td>1.612110</td>\n",
       "      <td>0.233003</td>\n",
       "      <td>-0.411015</td>\n",
       "      <td>-1.200729</td>\n",
       "      <td>-0.804148</td>\n",
       "      <td>-0.676437</td>\n",
       "      <td>-2.777374</td>\n",
       "      <td>0.186047</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68599</th>\n",
       "      <td>-0.075575</td>\n",
       "      <td>-0.894122</td>\n",
       "      <td>1.500426</td>\n",
       "      <td>0.697741</td>\n",
       "      <td>-1.260756</td>\n",
       "      <td>2.398555</td>\n",
       "      <td>0.590556</td>\n",
       "      <td>-0.487479</td>\n",
       "      <td>-2.777374</td>\n",
       "      <td>1.093023</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68600</th>\n",
       "      <td>0.876831</td>\n",
       "      <td>-0.820367</td>\n",
       "      <td>1.305642</td>\n",
       "      <td>2.371556</td>\n",
       "      <td>-1.199915</td>\n",
       "      <td>2.210121</td>\n",
       "      <td>0.590556</td>\n",
       "      <td>-0.715515</td>\n",
       "      <td>-2.777374</td>\n",
       "      <td>2.953488</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68601</th>\n",
       "      <td>-0.292698</td>\n",
       "      <td>-1.247394</td>\n",
       "      <td>2.830765</td>\n",
       "      <td>0.809086</td>\n",
       "      <td>-0.569418</td>\n",
       "      <td>-1.176546</td>\n",
       "      <td>-0.802023</td>\n",
       "      <td>-0.847824</td>\n",
       "      <td>-2.777374</td>\n",
       "      <td>0.069767</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>68602 rows × 144 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         volume  volume/g   density    weight  surface_area  void_fraction  \\\n",
       "0     -0.524386 -0.584699  0.798882 -0.383085     -0.257582      -0.872930   \n",
       "1      0.282926 -0.615547  0.856904  0.949117     -0.604123      -0.588382   \n",
       "2     -0.537500 -0.411402  0.509126 -0.484315     -0.491038      -0.536250   \n",
       "3      0.007296 -0.043730  0.045099  0.044946     -0.091623      -0.201260   \n",
       "4     -0.514063 -0.603040  0.833117 -0.356674     -0.401136      -0.878771   \n",
       "...         ...       ...       ...       ...           ...            ...   \n",
       "68597 -0.489396 -0.701596  1.030967 -0.257028     -1.162504       1.944400   \n",
       "68598 -0.333898 -0.933076  1.612110  0.233003     -0.411015      -1.200729   \n",
       "68599 -0.075575 -0.894122  1.500426  0.697741     -1.260756       2.398555   \n",
       "68600  0.876831 -0.820367  1.305642  2.371556     -1.199915       2.210121   \n",
       "68601 -0.292698 -1.247394  2.830765  0.809086     -0.569418      -1.176546   \n",
       "\n",
       "       void_volume  CO2/N2_selectivity  heat_adsorption   n_atoms  ...  \\\n",
       "0        -0.642857            0.157631         0.609280 -0.406977  ...   \n",
       "1        -0.513449            0.691708         0.845745  0.976744  ...   \n",
       "2        -0.447101           -0.021201         0.322523 -0.325581  ...   \n",
       "3        -0.160789            0.298554         0.219176  0.023256  ...   \n",
       "4        -0.647639            0.512156         0.407682 -0.186047  ...   \n",
       "...            ...                 ...              ...       ...  ...   \n",
       "68597     0.590556            0.220592        -2.777374  0.104651  ...   \n",
       "68598    -0.804148           -0.676437        -2.777374  0.186047  ...   \n",
       "68599     0.590556           -0.487479        -2.777374  1.093023  ...   \n",
       "68600     0.590556           -0.715515        -2.777374  2.953488  ...   \n",
       "68601    -0.802023           -0.847824        -2.777374  0.069767  ...   \n",
       "\n",
       "       topology_1  topology_2  topology_3  topology_4  topology_5  topology_6  \\\n",
       "0               0           0           0           0           1           0   \n",
       "1               0           1           0           0           0           0   \n",
       "2               0           0           0           0           1           0   \n",
       "3               0           0           0           0           0           0   \n",
       "4               0           0           0           0           1           0   \n",
       "...           ...         ...         ...         ...         ...         ...   \n",
       "68597           0           0           0           0           1           0   \n",
       "68598           0           1           0           0           0           0   \n",
       "68599           0           0           0           0           0           0   \n",
       "68600           0           0           0           0           0           0   \n",
       "68601           0           0           0           0           1           0   \n",
       "\n",
       "       topology_7  topology_8  topology_9  topology_10  \n",
       "0               0           0           0            0  \n",
       "1               0           0           0            0  \n",
       "2               0           0           0            0  \n",
       "3               0           1           0            0  \n",
       "4               0           0           0            0  \n",
       "...           ...         ...         ...          ...  \n",
       "68597           0           0           0            0  \n",
       "68598           0           0           0            0  \n",
       "68599           0           0           0            0  \n",
       "68600           0           0           0            0  \n",
       "68601           0           0           0            0  \n",
       "\n",
       "[68602 rows x 144 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tab_train = pd.read_csv('train_extra_for_gan.csv')\n",
    "tab_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b7ccedb4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:25:15.912007Z",
     "start_time": "2021-10-16T06:25:10.610310Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(68602, 50, 50, 3)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img_train = np.load('np_files/train_image_matrix_50_50.npy')\n",
    "img_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3f2fd628",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:27:21.316701Z",
     "start_time": "2021-10-16T06:27:21.279703Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CO2_working_capacity [mL/g]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>105.284502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>101.224774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>118.987011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>187.626004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>79.210001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68597</th>\n",
       "      <td>-12.943652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68598</th>\n",
       "      <td>-12.985582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68599</th>\n",
       "      <td>-13.187635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68600</th>\n",
       "      <td>15.672698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68601</th>\n",
       "      <td>3.144708</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>68602 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       CO2_working_capacity [mL/g]\n",
       "0                       105.284502\n",
       "1                       101.224774\n",
       "2                       118.987011\n",
       "3                       187.626004\n",
       "4                        79.210001\n",
       "...                            ...\n",
       "68597                   -12.943652\n",
       "68598                   -12.985582\n",
       "68599                   -13.187635\n",
       "68600                    15.672698\n",
       "68601                     3.144708\n",
       "\n",
       "[68602 rows x 1 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target = pd.read_csv('target_extra_for_gan.csv')\n",
    "target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "0f51a7f5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:27:21.581147Z",
     "start_time": "2021-10-16T06:27:21.546898Z"
    }
   },
   "outputs": [],
   "source": [
    "target['is_good'] = 0\n",
    "target.loc[target['CO2_working_capacity [mL/g]']>200, 'is_good'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1e8406f5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:27:21.889193Z",
     "start_time": "2021-10-16T06:27:21.881178Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    57143\n",
       "1    11459\n",
       "Name: is_good, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target['is_good'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d76b3eed",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:27:22.244922Z",
     "start_time": "2021-10-16T06:27:22.233885Z"
    }
   },
   "outputs": [],
   "source": [
    "is_good = target['is_good'].values\n",
    "target = target.drop(['is_good'], axis=1).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e6fbb6d2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:30:07.019529Z",
     "start_time": "2021-10-16T06:30:05.101522Z"
    }
   },
   "outputs": [],
   "source": [
    "## train/val split\n",
    "tab_X_train, tab_X_val, img_X_train, img_X_val, y_train, y_val = train_test_split(tab_train, img_train, target, \n",
    "                                                                                  test_size= 0.2, \n",
    "                                                                                  random_state=42,\n",
    "                                                                                  stratify=is_good)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3b02c60a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:30:09.346802Z",
     "start_time": "2021-10-16T06:30:09.327793Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((54881, 144), (13721, 144))"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(tab_X_train.shape, tab_X_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "34feb4dc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:30:22.307085Z",
     "start_time": "2021-10-16T06:30:22.290083Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((54881, 50, 50, 3), (13721, 50, 50, 3))"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(img_X_train.shape, img_X_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "00b51d49",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:30:38.407055Z",
     "start_time": "2021-10-16T06:30:38.387021Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((54881, 1), (13721, 1))"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(y_train.shape, y_val.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3f8b20b",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "9341eda5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:33:49.419762Z",
     "start_time": "2021-10-16T06:33:49.413768Z"
    }
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 256\n",
    "EPOCHS = 100\n",
    "INIT_LR = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "dda00b3a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:33:49.637028Z",
     "start_time": "2021-10-16T06:33:49.629011Z"
    }
   },
   "outputs": [],
   "source": [
    "tf.keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "326ffe48",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:33:49.865621Z",
     "start_time": "2021-10-16T06:33:49.858592Z"
    }
   },
   "outputs": [],
   "source": [
    "optimizer = Adam(learning_rate=INIT_LR, decay=INIT_LR/EPOCHS, beta_1=0.9, beta_2=0.999, epsilon=1e-8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "ed652e0b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:33:50.206629Z",
     "start_time": "2021-10-16T06:33:50.201640Z"
    }
   },
   "outputs": [],
   "source": [
    "# callbacks\n",
    "early_stopping = EarlyStopping(patience=10, restore_best_weights=True, monitor='val_loss', mode='min')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "8af4376e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T06:34:13.396230Z",
     "start_time": "2021-10-16T06:34:13.374203Z"
    }
   },
   "outputs": [],
   "source": [
    "combined_model.compile(optimizer=optimizer, loss=LMAE, metrics=[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "b3fdc9d1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T09:15:10.029961Z",
     "start_time": "2021-10-16T06:34:14.734554Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "215/215 [==============================] - 74s 272ms/step - loss: 2.4348 - val_loss: 1.5916\n",
      "Epoch 2/100\n",
      "215/215 [==============================] - 78s 364ms/step - loss: 1.4316 - val_loss: 1.5218\n",
      "Epoch 3/100\n",
      "215/215 [==============================] - 140s 653ms/step - loss: 1.3878 - val_loss: 1.4173\n",
      "Epoch 4/100\n",
      "215/215 [==============================] - 163s 760ms/step - loss: 1.3644 - val_loss: 1.4334\n",
      "Epoch 5/100\n",
      "215/215 [==============================] - 204s 950ms/step - loss: 1.3569 - val_loss: 1.4262\n",
      "Epoch 6/100\n",
      "215/215 [==============================] - 225s 1s/step - loss: 1.3439 - val_loss: 1.3288\n",
      "Epoch 7/100\n",
      "215/215 [==============================] - 169s 785ms/step - loss: 1.3350 - val_loss: 1.3692\n",
      "Epoch 8/100\n",
      "215/215 [==============================] - 270s 1s/step - loss: 1.3361 - val_loss: 1.3747\n",
      "Epoch 9/100\n",
      "215/215 [==============================] - 322s 1s/step - loss: 1.3280 - val_loss: 1.3230\n",
      "Epoch 10/100\n",
      "215/215 [==============================] - 288s 1s/step - loss: 1.3254 - val_loss: 1.3372\n",
      "Epoch 11/100\n",
      "215/215 [==============================] - 318s 1s/step - loss: 1.3177 - val_loss: 1.3105\n",
      "Epoch 12/100\n",
      "215/215 [==============================] - 323s 2s/step - loss: 1.3147 - val_loss: 1.3499\n",
      "Epoch 13/100\n",
      "215/215 [==============================] - 328s 2s/step - loss: 1.3123 - val_loss: 1.3718\n",
      "Epoch 14/100\n",
      "215/215 [==============================] - 328s 2s/step - loss: 1.3155 - val_loss: 1.3162\n",
      "Epoch 15/100\n",
      "215/215 [==============================] - 349s 2s/step - loss: 1.3059 - val_loss: 1.3140\n",
      "Epoch 16/100\n",
      "215/215 [==============================] - 275s 1s/step - loss: 1.3032 - val_loss: 1.3006\n",
      "Epoch 17/100\n",
      "215/215 [==============================] - 149s 694ms/step - loss: 1.2994 - val_loss: 1.2953\n",
      "Epoch 18/100\n",
      "215/215 [==============================] - 234s 1s/step - loss: 1.3017 - val_loss: 1.3032\n",
      "Epoch 19/100\n",
      "215/215 [==============================] - 270s 1s/step - loss: 1.2988 - val_loss: 1.2948\n",
      "Epoch 20/100\n",
      "215/215 [==============================] - 259s 1s/step - loss: 1.2984 - val_loss: 1.2946\n",
      "Epoch 21/100\n",
      "215/215 [==============================] - 321s 1s/step - loss: 1.2956 - val_loss: 1.2931\n",
      "Epoch 22/100\n",
      "215/215 [==============================] - 317s 1s/step - loss: 1.2890 - val_loss: 1.2924\n",
      "Epoch 23/100\n",
      "215/215 [==============================] - 310s 1s/step - loss: 1.2937 - val_loss: 1.3066\n",
      "Epoch 24/100\n",
      "215/215 [==============================] - 281s 1s/step - loss: 1.2877 - val_loss: 1.2914\n",
      "Epoch 25/100\n",
      "215/215 [==============================] - 193s 896ms/step - loss: 1.2828 - val_loss: 1.2907\n",
      "Epoch 26/100\n",
      "215/215 [==============================] - 267s 1s/step - loss: 1.2855 - val_loss: 1.2991\n",
      "Epoch 27/100\n",
      "215/215 [==============================] - 171s 796ms/step - loss: 1.2881 - val_loss: 1.3120\n",
      "Epoch 28/100\n",
      "215/215 [==============================] - 247s 1s/step - loss: 1.2828 - val_loss: 1.2877\n",
      "Epoch 29/100\n",
      "215/215 [==============================] - 190s 888ms/step - loss: 1.2821 - val_loss: 1.2963\n",
      "Epoch 30/100\n",
      "215/215 [==============================] - 213s 993ms/step - loss: 1.2807 - val_loss: 1.2831\n",
      "Epoch 31/100\n",
      "215/215 [==============================] - 171s 796ms/step - loss: 1.2759 - val_loss: 1.2879\n",
      "Epoch 32/100\n",
      "215/215 [==============================] - 204s 950ms/step - loss: 1.2766 - val_loss: 1.3098\n",
      "Epoch 33/100\n",
      "215/215 [==============================] - 241s 1s/step - loss: 1.2787 - val_loss: 1.2960\n",
      "Epoch 34/100\n",
      "215/215 [==============================] - 133s 619ms/step - loss: 1.2769 - val_loss: 1.2837\n",
      "Epoch 35/100\n",
      "215/215 [==============================] - 185s 863ms/step - loss: 1.2723 - val_loss: 1.2804\n",
      "Epoch 36/100\n",
      "215/215 [==============================] - 194s 904ms/step - loss: 1.2736 - val_loss: 1.2839\n",
      "Epoch 37/100\n",
      "215/215 [==============================] - 128s 594ms/step - loss: 1.2707 - val_loss: 1.2831\n",
      "Epoch 38/100\n",
      "215/215 [==============================] - 127s 593ms/step - loss: 1.2662 - val_loss: 1.2884\n",
      "Epoch 39/100\n",
      "215/215 [==============================] - 135s 628ms/step - loss: 1.2693 - val_loss: 1.3105\n",
      "Epoch 40/100\n",
      "215/215 [==============================] - 136s 634ms/step - loss: 1.2675 - val_loss: 1.2874\n",
      "Epoch 41/100\n",
      "215/215 [==============================] - 151s 705ms/step - loss: 1.2676 - val_loss: 1.2934\n",
      "Epoch 42/100\n",
      "215/215 [==============================] - 133s 621ms/step - loss: 1.2664 - val_loss: 1.2939\n",
      "Epoch 43/100\n",
      "215/215 [==============================] - 132s 612ms/step - loss: 1.2642 - val_loss: 1.2812\n",
      "Epoch 44/100\n",
      "215/215 [==============================] - 132s 615ms/step - loss: 1.2657 - val_loss: 1.2804\n",
      "Epoch 45/100\n",
      "215/215 [==============================] - 176s 820ms/step - loss: 1.2613 - val_loss: 1.2898\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x28a3066ab48>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_model.fit(x = [tab_X_train, img_X_train],\n",
    "          y = y_train,\n",
    "          validation_data=([tab_X_val, img_X_val], y_val),\n",
    "          batch_size=BATCH_SIZE,\n",
    "          epochs=EPOCHS,\n",
    "          verbose=1,\n",
    "          callbacks=[early_stopping]\n",
    "         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "9af4e8d5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T09:18:28.787100Z",
     "start_time": "2021-10-16T09:18:28.425098Z"
    }
   },
   "outputs": [],
   "source": [
    "combined_model.save('models/combined_deepinsight_cnn_50_50.h5', save_format='h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8fa0910",
   "metadata": {},
   "source": [
    "# Test submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "b945b7b4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T09:29:01.778861Z",
     "start_time": "2021-10-16T09:29:01.563825Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>volume</th>\n",
       "      <th>volume/g</th>\n",
       "      <th>density</th>\n",
       "      <th>weight</th>\n",
       "      <th>surface_area</th>\n",
       "      <th>void_fraction</th>\n",
       "      <th>void_volume</th>\n",
       "      <th>CO2/N2_selectivity</th>\n",
       "      <th>heat_adsorption</th>\n",
       "      <th>n_atoms</th>\n",
       "      <th>...</th>\n",
       "      <th>topology_1</th>\n",
       "      <th>topology_2</th>\n",
       "      <th>topology_3</th>\n",
       "      <th>topology_4</th>\n",
       "      <th>topology_5</th>\n",
       "      <th>topology_6</th>\n",
       "      <th>topology_7</th>\n",
       "      <th>topology_8</th>\n",
       "      <th>topology_9</th>\n",
       "      <th>topology_10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.479628</td>\n",
       "      <td>-0.272893</td>\n",
       "      <td>0.314058</td>\n",
       "      <td>-0.460370</td>\n",
       "      <td>-0.614563</td>\n",
       "      <td>-0.704325</td>\n",
       "      <td>-0.513748</td>\n",
       "      <td>0.841859</td>\n",
       "      <td>0.753026</td>\n",
       "      <td>-0.302326</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.945696</td>\n",
       "      <td>-0.807447</td>\n",
       "      <td>1.273569</td>\n",
       "      <td>2.466234</td>\n",
       "      <td>-0.451899</td>\n",
       "      <td>-0.222836</td>\n",
       "      <td>-0.399283</td>\n",
       "      <td>-0.064565</td>\n",
       "      <td>-0.481679</td>\n",
       "      <td>0.558140</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.287257</td>\n",
       "      <td>0.548750</td>\n",
       "      <td>-0.446130</td>\n",
       "      <td>-0.510708</td>\n",
       "      <td>0.956839</td>\n",
       "      <td>0.381305</td>\n",
       "      <td>0.483264</td>\n",
       "      <td>-0.329196</td>\n",
       "      <td>-0.530118</td>\n",
       "      <td>-0.581395</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.120337</td>\n",
       "      <td>0.943906</td>\n",
       "      <td>-0.672449</td>\n",
       "      <td>-0.262907</td>\n",
       "      <td>1.180306</td>\n",
       "      <td>0.698967</td>\n",
       "      <td>0.957860</td>\n",
       "      <td>-0.501135</td>\n",
       "      <td>-0.490294</td>\n",
       "      <td>-0.186047</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.455792</td>\n",
       "      <td>0.798101</td>\n",
       "      <td>-0.595773</td>\n",
       "      <td>0.077741</td>\n",
       "      <td>0.664925</td>\n",
       "      <td>0.626321</td>\n",
       "      <td>0.813210</td>\n",
       "      <td>-0.333562</td>\n",
       "      <td>-0.371552</td>\n",
       "      <td>0.093023</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16995</th>\n",
       "      <td>14.883110</td>\n",
       "      <td>6.775274</td>\n",
       "      <td>-1.708149</td>\n",
       "      <td>3.453108</td>\n",
       "      <td>2.525318</td>\n",
       "      <td>2.492180</td>\n",
       "      <td>8.834429</td>\n",
       "      <td>-0.752689</td>\n",
       "      <td>-1.772980</td>\n",
       "      <td>3.558140</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16996</th>\n",
       "      <td>1.407069</td>\n",
       "      <td>2.165319</td>\n",
       "      <td>-1.115874</td>\n",
       "      <td>0.238821</td>\n",
       "      <td>1.483792</td>\n",
       "      <td>1.602959</td>\n",
       "      <td>2.779438</td>\n",
       "      <td>-0.642975</td>\n",
       "      <td>-1.437719</td>\n",
       "      <td>0.162791</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16997</th>\n",
       "      <td>1.211109</td>\n",
       "      <td>2.360628</td>\n",
       "      <td>-1.164993</td>\n",
       "      <td>0.062301</td>\n",
       "      <td>1.649838</td>\n",
       "      <td>1.398200</td>\n",
       "      <td>2.669157</td>\n",
       "      <td>-0.746253</td>\n",
       "      <td>-1.480803</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16998</th>\n",
       "      <td>1.217125</td>\n",
       "      <td>2.786262</td>\n",
       "      <td>-1.258839</td>\n",
       "      <td>-0.045891</td>\n",
       "      <td>1.676200</td>\n",
       "      <td>1.672998</td>\n",
       "      <td>3.387029</td>\n",
       "      <td>-0.738450</td>\n",
       "      <td>-1.571499</td>\n",
       "      <td>-0.069767</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16999</th>\n",
       "      <td>11.484388</td>\n",
       "      <td>4.749416</td>\n",
       "      <td>-1.543962</td>\n",
       "      <td>3.522421</td>\n",
       "      <td>2.088765</td>\n",
       "      <td>2.208162</td>\n",
       "      <td>6.098326</td>\n",
       "      <td>-0.765526</td>\n",
       "      <td>-1.757809</td>\n",
       "      <td>3.604651</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17000 rows × 144 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          volume  volume/g   density    weight  surface_area  void_fraction  \\\n",
       "0      -0.479628 -0.272893  0.314058 -0.460370     -0.614563      -0.704325   \n",
       "1       0.945696 -0.807447  1.273569  2.466234     -0.451899      -0.222836   \n",
       "2      -0.287257  0.548750 -0.446130 -0.510708      0.956839       0.381305   \n",
       "3       0.120337  0.943906 -0.672449 -0.262907      1.180306       0.698967   \n",
       "4       0.455792  0.798101 -0.595773  0.077741      0.664925       0.626321   \n",
       "...          ...       ...       ...       ...           ...            ...   \n",
       "16995  14.883110  6.775274 -1.708149  3.453108      2.525318       2.492180   \n",
       "16996   1.407069  2.165319 -1.115874  0.238821      1.483792       1.602959   \n",
       "16997   1.211109  2.360628 -1.164993  0.062301      1.649838       1.398200   \n",
       "16998   1.217125  2.786262 -1.258839 -0.045891      1.676200       1.672998   \n",
       "16999  11.484388  4.749416 -1.543962  3.522421      2.088765       2.208162   \n",
       "\n",
       "       void_volume  CO2/N2_selectivity  heat_adsorption   n_atoms  ...  \\\n",
       "0        -0.513748            0.841859         0.753026 -0.302326  ...   \n",
       "1        -0.399283           -0.064565        -0.481679  0.558140  ...   \n",
       "2         0.483264           -0.329196        -0.530118 -0.581395  ...   \n",
       "3         0.957860           -0.501135        -0.490294 -0.186047  ...   \n",
       "4         0.813210           -0.333562        -0.371552  0.093023  ...   \n",
       "...            ...                 ...              ...       ...  ...   \n",
       "16995     8.834429           -0.752689        -1.772980  3.558140  ...   \n",
       "16996     2.779438           -0.642975        -1.437719  0.162791  ...   \n",
       "16997     2.669157           -0.746253        -1.480803  0.023256  ...   \n",
       "16998     3.387029           -0.738450        -1.571499 -0.069767  ...   \n",
       "16999     6.098326           -0.765526        -1.757809  3.604651  ...   \n",
       "\n",
       "       topology_1  topology_2  topology_3  topology_4  topology_5  topology_6  \\\n",
       "0               0           0           0           0           1           0   \n",
       "1               0           0           0           0           0           0   \n",
       "2               0           0           0           0           1           0   \n",
       "3               0           0           0           0           1           0   \n",
       "4               0           0           0           0           1           0   \n",
       "...           ...         ...         ...         ...         ...         ...   \n",
       "16995           0           0           0           1           0           0   \n",
       "16996           0           0           0           1           0           0   \n",
       "16997           0           0           0           0           1           0   \n",
       "16998           0           0           0           0           1           0   \n",
       "16999           0           0           0           0           1           0   \n",
       "\n",
       "       topology_7  topology_8  topology_9  topology_10  \n",
       "0               0           0           0            0  \n",
       "1               0           0           0            0  \n",
       "2               0           0           0            0  \n",
       "3               0           0           0            0  \n",
       "4               0           0           0            0  \n",
       "...           ...         ...         ...          ...  \n",
       "16995           0           0           0            0  \n",
       "16996           0           0           0            0  \n",
       "16997           0           0           0            0  \n",
       "16998           0           0           0            0  \n",
       "16999           0           0           0            0  \n",
       "\n",
       "[17000 rows x 144 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tab_test = pd.read_csv('test_extra_2_cleaned.csv')\n",
    "tab_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "dc14499f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T09:28:50.215734Z",
     "start_time": "2021-10-16T09:28:49.681707Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17000, 50, 50, 3)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mat_test = np.load('np_files/test_image_matrix_50_50.npy')\n",
    "mat_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "8c96562e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T09:29:51.375695Z",
     "start_time": "2021-10-16T09:29:45.047691Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>CO2_working_capacity [mL/g]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>68614</td>\n",
       "      <td>166.052383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>68615</td>\n",
       "      <td>54.064545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>68616</td>\n",
       "      <td>66.692818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>68617</td>\n",
       "      <td>67.646645</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>68618</td>\n",
       "      <td>62.072334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16995</th>\n",
       "      <td>85609</td>\n",
       "      <td>-7.133041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16996</th>\n",
       "      <td>85610</td>\n",
       "      <td>-0.405801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16997</th>\n",
       "      <td>85611</td>\n",
       "      <td>-0.310039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16998</th>\n",
       "      <td>85612</td>\n",
       "      <td>-0.344075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16999</th>\n",
       "      <td>85613</td>\n",
       "      <td>-4.186900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  CO2_working_capacity [mL/g]\n",
       "0      68614                   166.052383\n",
       "1      68615                    54.064545\n",
       "2      68616                    66.692818\n",
       "3      68617                    67.646645\n",
       "4      68618                    62.072334\n",
       "...      ...                          ...\n",
       "16995  85609                    -7.133041\n",
       "16996  85610                    -0.405801\n",
       "16997  85611                    -0.310039\n",
       "16998  85612                    -0.344075\n",
       "16999  85613                    -4.186900\n",
       "\n",
       "[17000 rows x 2 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict = combined_model.predict([tab_test, mat_test]).flatten()\n",
    "\n",
    "predicted_df = pd.DataFrame({\"id\":range(68614,68614+len(test)), \"CO2_working_capacity [mL/g]\":predict})\n",
    "predicted_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "7536e596",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T09:30:05.514658Z",
     "start_time": "2021-10-16T09:30:05.441658Z"
    }
   },
   "outputs": [],
   "source": [
    "predicted_df.to_csv(\"output/phase2/submission.csv\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb341c39",
   "metadata": {},
   "source": [
    "# Turn model to feature extractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "a9c1eebf",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T09:40:40.321800Z",
     "start_time": "2021-10-16T09:40:40.312783Z"
    }
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import load_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "27a3b684",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T09:40:41.401050Z",
     "start_time": "2021-10-16T09:40:40.728050Z"
    }
   },
   "outputs": [],
   "source": [
    "loaded_model = load_model('models/combined_deepinsight_cnn_50_50.h5', custom_objects={'LMAE':LMAE})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "200ba47e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T09:40:41.510056Z",
     "start_time": "2021-10-16T09:40:41.491051Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 50, 50, 3)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d (Conv2D)                 (None, 50, 50, 32)   896         input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 50, 50, 64)   18496       conv2d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 50, 50, 64)   36928       conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, 50, 50, 64)   256         conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D)    (None, 25, 25, 64)   0           batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 25, 25, 64)   0           max_pooling2d[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 25, 25, 128)  73856       dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 25, 25, 128)  147584      conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 25, 25, 128)  512         conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, 12, 12, 128)  0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 12, 12, 128)  0           max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 12, 12, 256)  295168      dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 12, 12, 256)  590080      conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 12, 12, 256)  1024        conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)  (None, 6, 6, 256)    0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 6, 6, 256)    0           max_pooling2d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 9216)         0           dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "input_3 (InputLayer)            [(None, 144)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 512)          4719104     flatten[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 128)          18560       input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 256)          131328      dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 256)          33024       dense_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 256)          1024        dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 256)          65792       dense_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 256)          0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 128)          32896       dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 64)           16448       dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, 64)           8256        dense_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 64)           256         dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_10 (Dense)                (None, 16)           1040        dense_9[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 16)           1040        batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_11 (Dense)                (None, 4)            68          dense_10[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 4)            68          dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 8)            0           dense_11[0][0]                   \n",
      "                                                                 dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (None, 4)            36          concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_16 (Dense)                (None, 4)            20          dense_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_17 (Dense)                (None, 1)            5           dense_16[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 6,193,765\n",
      "Trainable params: 6,192,229\n",
      "Non-trainable params: 1,536\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "loaded_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "1d7e5a37",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T09:40:56.078598Z",
     "start_time": "2021-10-16T09:40:56.068559Z"
    }
   },
   "outputs": [],
   "source": [
    "feats_ext = tf.keras.Model(inputs = loaded_model.input, outputs = loaded_model.layers[-2].output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "b89e94bf",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T09:41:00.766274Z",
     "start_time": "2021-10-16T09:41:00.740274Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 50, 50, 3)]  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d (Conv2D)                 (None, 50, 50, 32)   896         input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 50, 50, 64)   18496       conv2d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 50, 50, 64)   36928       conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, 50, 50, 64)   256         conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D)    (None, 25, 25, 64)   0           batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 25, 25, 64)   0           max_pooling2d[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 25, 25, 128)  73856       dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 25, 25, 128)  147584      conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 25, 25, 128)  512         conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, 12, 12, 128)  0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 12, 12, 128)  0           max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 12, 12, 256)  295168      dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 12, 12, 256)  590080      conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 12, 12, 256)  1024        conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)  (None, 6, 6, 256)    0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 6, 6, 256)    0           max_pooling2d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 9216)         0           dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "input_3 (InputLayer)            [(None, 144)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 512)          4719104     flatten[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 128)          18560       input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 256)          131328      dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 256)          33024       dense_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 256)          1024        dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 256)          65792       dense_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 256)          0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 128)          32896       dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 64)           16448       dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, 64)           8256        dense_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 64)           256         dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_10 (Dense)                (None, 16)           1040        dense_9[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 16)           1040        batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_11 (Dense)                (None, 4)            68          dense_10[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 4)            68          dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 8)            0           dense_11[0][0]                   \n",
      "                                                                 dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (None, 4)            36          concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_16 (Dense)                (None, 4)            20          dense_15[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 6,193,760\n",
      "Trainable params: 6,192,224\n",
      "Non-trainable params: 1,536\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "feats_ext.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "c90e00ec",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T09:41:19.908569Z",
     "start_time": "2021-10-16T09:41:19.752895Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.\n"
     ]
    }
   ],
   "source": [
    "feats_ext.save('models/combined_di_cnn_feats_ext.h5', save_format='h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aad2569c",
   "metadata": {},
   "source": [
    "# Train new feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "f38f6e93",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T10:19:40.286091Z",
     "start_time": "2021-10-16T10:19:39.936094Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.\n"
     ]
    }
   ],
   "source": [
    "feats_ext = load_model('models/combined_di_cnn_feats_ext.h5', custom_objects={'LMAE':LMAE})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "f781721b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T10:20:42.106579Z",
     "start_time": "2021-10-16T10:20:16.851578Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(68602, 4)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_feats = feats_ext.predict([tab_train, img_train])\n",
    "new_feats.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "4f57c3e0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T10:20:45.715591Z",
     "start_time": "2021-10-16T10:20:45.683557Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comb_cnn_0</th>\n",
       "      <th>comb_cnn_1</th>\n",
       "      <th>comb_cnn_2</th>\n",
       "      <th>comb_cnn_3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>58.904251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>69.381874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.915159</td>\n",
       "      <td>82.243233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>117.579170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>61.207539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68597</th>\n",
       "      <td>0.716291</td>\n",
       "      <td>7.768239</td>\n",
       "      <td>5.032241</td>\n",
       "      <td>8.482945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68598</th>\n",
       "      <td>0.974198</td>\n",
       "      <td>8.142911</td>\n",
       "      <td>5.304706</td>\n",
       "      <td>7.986442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68599</th>\n",
       "      <td>0.778554</td>\n",
       "      <td>7.931119</td>\n",
       "      <td>5.137025</td>\n",
       "      <td>8.463214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68600</th>\n",
       "      <td>0.799744</td>\n",
       "      <td>7.906666</td>\n",
       "      <td>5.131299</td>\n",
       "      <td>8.344973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68601</th>\n",
       "      <td>1.005249</td>\n",
       "      <td>6.179784</td>\n",
       "      <td>3.833471</td>\n",
       "      <td>5.429558</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>68602 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       comb_cnn_0  comb_cnn_1  comb_cnn_2  comb_cnn_3\n",
       "0        0.000000    0.000000    0.000000   58.904251\n",
       "1        0.000000    0.000000    0.000000   69.381874\n",
       "2        0.000000    0.000000    7.915159   82.243233\n",
       "3        0.000000    0.000000    0.000000  117.579170\n",
       "4        0.000000    0.000000    0.000000   61.207539\n",
       "...           ...         ...         ...         ...\n",
       "68597    0.716291    7.768239    5.032241    8.482945\n",
       "68598    0.974198    8.142911    5.304706    7.986442\n",
       "68599    0.778554    7.931119    5.137025    8.463214\n",
       "68600    0.799744    7.906666    5.131299    8.344973\n",
       "68601    1.005249    6.179784    3.833471    5.429558\n",
       "\n",
       "[68602 rows x 4 columns]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_feats_cols = [f'comb_cnn_{i}' for i in range(new_feats.shape[1])]\n",
    "new_feats_df = pd.DataFrame(new_feats, columns=new_feats_cols)\n",
    "new_feats_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "ce28cfeb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T10:20:51.606441Z",
     "start_time": "2021-10-16T10:20:51.543441Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comb_cnn_0</th>\n",
       "      <th>comb_cnn_1</th>\n",
       "      <th>comb_cnn_2</th>\n",
       "      <th>comb_cnn_3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>68602.000000</td>\n",
       "      <td>68602.000000</td>\n",
       "      <td>68602.000000</td>\n",
       "      <td>68602.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.135411</td>\n",
       "      <td>1.436154</td>\n",
       "      <td>2.168628</td>\n",
       "      <td>88.425812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.439961</td>\n",
       "      <td>3.232600</td>\n",
       "      <td>4.774739</td>\n",
       "      <td>61.025421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>50.224761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>72.503105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.874144</td>\n",
       "      <td>119.262621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>4.834850</td>\n",
       "      <td>26.855505</td>\n",
       "      <td>56.489670</td>\n",
       "      <td>432.389038</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         comb_cnn_0    comb_cnn_1    comb_cnn_2    comb_cnn_3\n",
       "count  68602.000000  68602.000000  68602.000000  68602.000000\n",
       "mean       0.135411      1.436154      2.168628     88.425812\n",
       "std        0.439961      3.232600      4.774739     61.025421\n",
       "min        0.000000      0.000000      0.000000      0.000000\n",
       "25%        0.000000      0.000000      0.000000     50.224761\n",
       "50%        0.000000      0.000000      0.000000     72.503105\n",
       "75%        0.000000      0.000000      2.874144    119.262621\n",
       "max        4.834850     26.855505     56.489670    432.389038"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_feats_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "d2b2cb5b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T10:21:13.293703Z",
     "start_time": "2021-10-16T10:21:12.951704Z"
    }
   },
   "outputs": [],
   "source": [
    "new_feats_df.to_csv('train_feats_from_comb_cnn.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "465556ef",
   "metadata": {},
   "source": [
    "# Test new feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "a7df3cd0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T10:22:45.895942Z",
     "start_time": "2021-10-16T10:22:39.672959Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17000, 4)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_new_feats = feats_ext.predict([tab_test, mat_test])\n",
    "test_new_feats.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "0b80adb5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T10:22:59.619499Z",
     "start_time": "2021-10-16T10:22:59.604526Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comb_cnn_0</th>\n",
       "      <th>comb_cnn_1</th>\n",
       "      <th>comb_cnn_2</th>\n",
       "      <th>comb_cnn_3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>121.871979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.312899</td>\n",
       "      <td>40.470531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>48.788647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>49.490231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>45.390083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16995</th>\n",
       "      <td>2.446862</td>\n",
       "      <td>9.734859</td>\n",
       "      <td>6.094297</td>\n",
       "      <td>4.706102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16996</th>\n",
       "      <td>1.053595</td>\n",
       "      <td>8.092432</td>\n",
       "      <td>5.274447</td>\n",
       "      <td>7.620755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16997</th>\n",
       "      <td>1.062499</td>\n",
       "      <td>8.433994</td>\n",
       "      <td>5.508307</td>\n",
       "      <td>8.026564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16998</th>\n",
       "      <td>1.148709</td>\n",
       "      <td>9.191586</td>\n",
       "      <td>5.929290</td>\n",
       "      <td>8.741869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16999</th>\n",
       "      <td>1.816106</td>\n",
       "      <td>8.900167</td>\n",
       "      <td>5.707390</td>\n",
       "      <td>5.877586</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       comb_cnn_0  comb_cnn_1  comb_cnn_2  comb_cnn_3\n",
       "0        0.000000    0.000000    0.000000  121.871979\n",
       "1        0.000000    0.000000    3.312899   40.470531\n",
       "2        0.000000    0.000000    0.000000   48.788647\n",
       "3        0.000000    0.000000    0.000000   49.490231\n",
       "4        0.000000    0.000000    0.000000   45.390083\n",
       "...           ...         ...         ...         ...\n",
       "16995    2.446862    9.734859    6.094297    4.706102\n",
       "16996    1.053595    8.092432    5.274447    7.620755\n",
       "16997    1.062499    8.433994    5.508307    8.026564\n",
       "16998    1.148709    9.191586    5.929290    8.741869\n",
       "16999    1.816106    8.900167    5.707390    5.877586\n",
       "\n",
       "[17000 rows x 4 columns]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_new_feats_df = pd.DataFrame(test_new_feats, columns=new_feats_cols)\n",
    "test_new_feats_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "a2ef6617",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T10:23:00.583362Z",
     "start_time": "2021-10-16T10:23:00.544362Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comb_cnn_0</th>\n",
       "      <th>comb_cnn_1</th>\n",
       "      <th>comb_cnn_2</th>\n",
       "      <th>comb_cnn_3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>17000.000000</td>\n",
       "      <td>17000.000000</td>\n",
       "      <td>17000.000000</td>\n",
       "      <td>17000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.151956</td>\n",
       "      <td>1.199031</td>\n",
       "      <td>2.374463</td>\n",
       "      <td>79.928833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.479456</td>\n",
       "      <td>2.945057</td>\n",
       "      <td>4.912333</td>\n",
       "      <td>57.008751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.379038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>47.571190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>64.896301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.388823</td>\n",
       "      <td>103.052229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>4.924204</td>\n",
       "      <td>16.828388</td>\n",
       "      <td>48.217567</td>\n",
       "      <td>421.367798</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         comb_cnn_0    comb_cnn_1    comb_cnn_2    comb_cnn_3\n",
       "count  17000.000000  17000.000000  17000.000000  17000.000000\n",
       "mean       0.151956      1.199031      2.374463     79.928833\n",
       "std        0.479456      2.945057      4.912333     57.008751\n",
       "min        0.000000      0.000000      0.000000      1.379038\n",
       "25%        0.000000      0.000000      0.000000     47.571190\n",
       "50%        0.000000      0.000000      0.000000     64.896301\n",
       "75%        0.000000      0.000000      4.388823    103.052229\n",
       "max        4.924204     16.828388     48.217567    421.367798"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_new_feats_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "be80e8c2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-10-16T10:23:18.870995Z",
     "start_time": "2021-10-16T10:23:18.764997Z"
    }
   },
   "outputs": [],
   "source": [
    "test_new_feats_df.to_csv('test_feats_from_comb_cnn.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "chemistry-ml",
   "language": "python",
   "name": "chemistry-ml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}