main_notebook_for_model_development.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MIT License\n",
    "Copyright (c) 2023 Okyaz Eminaga\n",
    "Permission is hereby granted, free of charge, to any person obtaining a copy\n",
    "of this software and associated documentation files (the \"Software\"), to deal\n",
    "in the Software without restriction, including without limitation the rights\n",
    "to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n",
    "copies of the Software, and to permit persons to whom the Software is\n",
    "furnished to do so, subject to the following conditions:\n",
    "The above copyright notice and this permission notice shall be included in all\n",
    "copies or substantial portions of the Software.\n",
    "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n",
    "IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n",
    "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n",
    "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n",
    "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n",
    "OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n",
    "SOFTWARE."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n",
    "import PIL\n",
    "\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"3\"\n",
    "\n",
    "import numpy as np\n",
    "from tensorflow.compat.v1.keras.backend import set_session\n",
    "import tensorflow as tf\n",
    "config = tf.compat.v1.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "set_session(tf.compat.v1.Session(config=config))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Any, Dict, Iterable, Sequence, Tuple, Optional, Union\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import tensorflow as tf\n",
    "print(\"Using Tensorflow:\", tf.__version__)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Development"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from albumentations import (\n",
    "    Compose, RandomBrightness, JpegCompression, HueSaturationValue, RandomContrast, HorizontalFlip,\n",
    "    Rotate, RandomCrop,RandomSizedCrop,CenterCrop\n",
    ")\n",
    "transforms = Compose([\n",
    "            Rotate(limit=40),\n",
    "            RandomBrightness(limit=0.1),\n",
    "            JpegCompression(quality_lower=85, quality_upper=100, p=0.5),\n",
    "            HueSaturationValue(hue_shift_limit=20, sat_shift_limit=30, val_shift_limit=20, p=0.5),\n",
    "            RandomContrast(limit=0.2, p=0.5),\n",
    "            HorizontalFlip()\n",
    "        ])\n",
    "no_change_transform = Compose([CenterCrop(4096,4096, always_apply=True),RandomSizedCrop([512,586],512,512,p=1.0, always_apply=True)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "\n",
    "\n",
    "class InputFunction(object):\n",
    "    \"\"\"Callable input function that computes the risk set for each batch.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    images : np.ndarray, shape=(n_samples, height, width)\n",
    "        Image data.\n",
    "    time : np.ndarray, shape=(n_samples,)\n",
    "        Observed time.\n",
    "    event : np.ndarray, shape=(n_samples,)\n",
    "        Event indicator.\n",
    "    batch_size : int, optional, default=64\n",
    "        Number of samples per batch.\n",
    "    drop_last : int, optional, default=False\n",
    "        Whether to drop the last incomplete batch.\n",
    "    shuffle : bool, optional, default=False\n",
    "        Whether to shuffle data.\n",
    "    seed : int, optional, default=89\n",
    "        Random number seed.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self,\n",
    "                 x: np.ndarray,\n",
    "                 time: np.ndarray,\n",
    "                 event: np.ndarray,\n",
    "                 augmentation: bool = False,\n",
    "                 input_size: (int, int) = (512, 512),\n",
    "                 channel_number: int = 3,\n",
    "                 batch_size: int = 32,\n",
    "                 drop_last: bool = False,\n",
    "                 shuffle: bool = False,\n",
    "                 k: int = 1,\n",
    "                 read_file: bool = False,\n",
    "                 repeat: int = 1,\n",
    "                 resize_img: bool = False,\n",
    "                 seed: int = 89) -> None:\n",
    "        self.x = x\n",
    "        self.time = time\n",
    "        self.input_size = input_size\n",
    "        self.augmentation = augmentation\n",
    "        self.event = event\n",
    "        self.batch_size = batch_size\n",
    "        self.drop_last = drop_last\n",
    "        self.shuffle = shuffle\n",
    "        self.seed = seed\n",
    "        self.repeat = repeat\n",
    "        self.k = k\n",
    "        self.resize_img = resize_img\n",
    "        self.read_file = read_file\n",
    "        self.channel_number = channel_number\n",
    "\n",
    "    def size(self) -> int:\n",
    "        \"\"\"Total number of samples.\"\"\"\n",
    "        return len(self.x)\n",
    "\n",
    "    def steps_per_epoch(self) -> int:\n",
    "        \"\"\"Number of batches for one epoch.\"\"\"\n",
    "        return int(np.floor(len(self.x) / self.batch_size))\n",
    "\n",
    "    def _get_data_batch(self, index: np.ndarray) -> Tuple[np.ndarray, Dict[str, np.ndarray]]:\n",
    "        \"\"\"Compute risk set for samples in batch.\"\"\"\n",
    "        time = self.time[index].copy()\n",
    "        event = self.event[index].copy()\n",
    "        x = self.x[index].copy()\n",
    "        if self.read_file:\n",
    "            images = []\n",
    "            for fl in x:\n",
    "                img = Image.open(fl)\n",
    "                img = img.resize((5120//self.k, 5120//self.k))\n",
    "                img = np.array(img)\n",
    "                # img = cv2.resize(img, (5120//self.k, 5120//self.k))\n",
    "                data = {\"image\": img}\n",
    "                if self.augmentation:\n",
    "                    aug_data = transforms(**data)\n",
    "                else:\n",
    "                    aug_data = no_change_transform(**data)\n",
    "                aug_img = aug_data[\"image\"]\n",
    "                images.append(aug_img)\n",
    "            x = np.array(images)\n",
    "        else:\n",
    "            if self.resize_img:\n",
    "                x_tmp = []\n",
    "                for j in range(x.shape[0]):\n",
    "                    x_tmp.append(\n",
    "                        resize(x[j], self.input_size, preserve_range=True).astype(np.uint8))\n",
    "                x = np.array(x_tmp)\n",
    "            if self.augmentation:\n",
    "                for i in range(x.shape[0]):\n",
    "                    data = {\"image\": x[i]}\n",
    "                    aug_data = transforms(**data)\n",
    "                    x[i] = aug_data[\"image\"]\n",
    "\n",
    "        return x, event.astype(np.int32)\n",
    "\n",
    "    def _iter_data(self) -> Iterable[Tuple[np.ndarray, Dict[str, np.ndarray]]]:\n",
    "        \"\"\"Generator that yields one batch at a time.\"\"\"\n",
    "        index = np.arange(self.size())\n",
    "        rnd = np.random.RandomState(self.seed)\n",
    "\n",
    "        if self.shuffle:\n",
    "            rnd.shuffle(index)\n",
    "        for b in range(self.steps_per_epoch()):\n",
    "            start = b * self.batch_size\n",
    "            idx = index[start:(start + self.batch_size)]\n",
    "            yield self._get_data_batch(idx)\n",
    "\n",
    "        if not self.drop_last:\n",
    "            start = self.steps_per_epoch() * self.batch_size\n",
    "            idx = index[start:]\n",
    "            yield self._get_data_batch(idx)\n",
    "\n",
    "    def _get_shapes(self) -> Tuple[tf.TensorShape, Dict[str, tf.TensorShape]]:\n",
    "        \"\"\"Return shapes of data returned by `self._iter_data`.\"\"\"\n",
    "        batch_size = self.batch_size if self.drop_last else None\n",
    "        h, w = self.input_size\n",
    "        c = self.channel_number\n",
    "        images = tf.TensorShape([batch_size, h, w, c])\n",
    "        return images, tf.TensorShape((batch_size,))\n",
    "\n",
    "    def _get_dtypes(self) -> Tuple[tf.DType, Dict[str, tf.DType]]:\n",
    "        \"\"\"Return dtypes of data returned by `self._iter_data`.\"\"\"\n",
    "        return tf.float32, tf.int32\n",
    "\n",
    "    def _make_dataset(self) -> tf.data.Dataset:\n",
    "        \"\"\"Create dataset from generator.\"\"\"\n",
    "        options = tf.data.Options()\n",
    "        options.experimental_optimization.noop_elimination = True\n",
    "        # options.experimental_optimization.map_vectorization.enabled = True\n",
    "        # options.experimental_optimization.autotune = True\n",
    "        # options.experimental_optimization.apply_default_optimizations=True\n",
    "        options.experimental_optimization.map_parallelization = True\n",
    "        ds = tf.data.Dataset.from_generator(\n",
    "            self._iter_data,\n",
    "            self._get_dtypes(),\n",
    "            self._get_shapes()\n",
    "        )\n",
    "        ds = ds.with_options(options)\n",
    "        if self.repeat > 1:\n",
    "            return ds.repeat(self.repeat)\n",
    "        else:\n",
    "            return ds\n",
    "\n",
    "    def __call__(self) -> tf.data.Dataset:\n",
    "        return self._make_dataset()\n",
    "\n",
    "\n",
    "'''\n",
    "#USED FOR PLEXUSNET ATTENTION POOLING AND 2 CLASS#\n",
    "\n",
    "class InputFunction(object):\n",
    "    \"\"\"Callable input function that computes the risk set for each batch.\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    images : np.ndarray, shape=(n_samples, height, width)\n",
    "        Image data.\n",
    "    time : np.ndarray, shape=(n_samples,)\n",
    "        Observed time.\n",
    "    event : np.ndarray, shape=(n_samples,)\n",
    "        Event indicator.\n",
    "    batch_size : int, optional, default=64\n",
    "        Number of samples per batch.\n",
    "    drop_last : int, optional, default=False\n",
    "        Whether to drop the last incomplete batch.\n",
    "    shuffle : bool, optional, default=False\n",
    "        Whether to shuffle data.\n",
    "    seed : int, optional, default=89\n",
    "        Random number seed.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self,\n",
    "                 x: np.ndarray,\n",
    "                 time: np.ndarray,\n",
    "                 event: np.ndarray,\n",
    "                 augmentation : bool = False,\n",
    "                 input_size : (int, int) = (512,512),\n",
    "                 channel_number : int = 3, \n",
    "                 batch_size: int = 32,\n",
    "                 drop_last: bool = False,\n",
    "                 shuffle: bool = False,\n",
    "                 k : int = 1,\n",
    "                 read_file: bool = False,\n",
    "                 repeat : int = 1,\n",
    "                 resize_img : bool = False,\n",
    "                 seed: int = 89) -> None:\n",
    "        self.x = x\n",
    "        self.time = time\n",
    "        self.input_size = input_size\n",
    "        self.augmentation=augmentation\n",
    "        self.event = event\n",
    "        self.batch_size = batch_size\n",
    "        self.drop_last = drop_last\n",
    "        self.shuffle = shuffle\n",
    "        self.seed = seed\n",
    "        self.repeat=repeat\n",
    "        self.k= k\n",
    "        self.resize_img =resize_img\n",
    "        self.read_file=read_file\n",
    "        self.channel_number= channel_number\n",
    "\n",
    "    def size(self) -> int:\n",
    "        \"\"\"Total number of samples.\"\"\"\n",
    "        return len(self.x)\n",
    "\n",
    "    def steps_per_epoch(self) -> int:\n",
    "        \"\"\"Number of batches for one epoch.\"\"\"\n",
    "        return int(np.floor(len(self.x) / self.batch_size))\n",
    "\n",
    "    def _get_data_batch(self, index: np.ndarray) -> Tuple[np.ndarray, Dict[str, np.ndarray]]:\n",
    "        \"\"\"Compute risk set for samples in batch.\"\"\"\n",
    "        time = self.time[index].copy()\n",
    "        event = self.event[index].copy()\n",
    "        x = self.x[index].copy()\n",
    "        if self.read_file:\n",
    "            images = []\n",
    "            for fl in x:\n",
    "                img=Image.open(fl)\n",
    "                img =img.resize((5120//self.k, 5120//self.k))\n",
    "                img = np.array(img)\n",
    "                #img = cv2.resize(img, (5120//self.k, 5120//self.k))\n",
    "                data = {\"image\":img}\n",
    "                if self.augmentation:\n",
    "                    aug_data = transforms(**data)\n",
    "                else:\n",
    "                    aug_data = no_change_transform(**data)\n",
    "                aug_img = aug_data[\"image\"]\n",
    "                images.append(aug_img)\n",
    "            x = np.array(images)\n",
    "        else:\n",
    "            if self.resize_img:\n",
    "                x_tmp=[]\n",
    "                for j in range(x.shape[0]):\n",
    "                    x_tmp.append(resize(x[j],self.input_size, preserve_range=True).astype(np.uint8))\n",
    "                x=np.array(x_tmp)\n",
    "            if self.augmentation:\n",
    "                for i in range(x.shape[0]):\n",
    "                    data = {\"image\":x[i]}\n",
    "                    aug_data = transforms(**data)\n",
    "                    x[i]=aug_data[\"image\"]\n",
    "    \n",
    "        return x, event.astype(np.int32)\n",
    "\n",
    "    def _iter_data(self) -> Iterable[Tuple[np.ndarray, Dict[str, np.ndarray]]]:\n",
    "        \"\"\"Generator that yields one batch at a time.\"\"\"\n",
    "        index = np.arange(self.size())\n",
    "        rnd = np.random.RandomState(self.seed)\n",
    "\n",
    "        if self.shuffle:\n",
    "            rnd.shuffle(index)\n",
    "        for b in range(self.steps_per_epoch()):\n",
    "            start = b * self.batch_size\n",
    "            idx = index[start:(start + self.batch_size)]\n",
    "            yield self._get_data_batch(idx)\n",
    "\n",
    "        if not self.drop_last:\n",
    "            start = self.steps_per_epoch() * self.batch_size\n",
    "            idx = index[start:]\n",
    "            yield self._get_data_batch(idx)\n",
    "\n",
    "    def _get_shapes(self) -> Tuple[tf.TensorShape, Dict[str, tf.TensorShape]]:\n",
    "        \"\"\"Return shapes of data returned by `self._iter_data`.\"\"\"\n",
    "        batch_size = self.batch_size if self.drop_last else None\n",
    "        h, w= self.input_size\n",
    "        c= self.channel_number\n",
    "        images = tf.TensorShape([batch_size, h, w, c])\n",
    "        return images, tf.TensorShape((batch_size,2))\n",
    "\n",
    "    def _get_dtypes(self) -> Tuple[tf.DType, Dict[str, tf.DType]]:\n",
    "        \"\"\"Return dtypes of data returned by `self._iter_data`.\"\"\"\n",
    "        return tf.float32, tf.int32\n",
    "\n",
    "    def _make_dataset(self) -> tf.data.Dataset:\n",
    "        \"\"\"Create dataset from generator.\"\"\"\n",
    "        options = tf.data.Options()\n",
    "        options.experimental_optimization.noop_elimination = True\n",
    "        #options.experimental_optimization.map_vectorization.enabled = True\n",
    "        #options.experimental_optimization.autotune = True\n",
    "        #options.experimental_optimization.apply_default_optimizations=True\n",
    "        options.experimental_optimization.map_parallelization=True\n",
    "        ds = tf.data.Dataset.from_generator(\n",
    "            self._iter_data,\n",
    "            self._get_dtypes(),\n",
    "            self._get_shapes()\n",
    "        )\n",
    "        ds = ds.with_options(options)\n",
    "        if self.repeat>1: \n",
    "            return ds.repeat(self.repeat)\n",
    "        else:\n",
    "            return ds\n",
    "\n",
    "    def __call__(self) -> tf.data.Dataset:\n",
    "        return self._make_dataset()\n",
    "'''\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "time_train=np.load(\"./time_train_shuffled_10x.npy\")\n",
    "event_train=np.load(\"./event_train_shuffled_10x.npy\")\n",
    "image_train=np.load(\"./image_train_shuffled_10x.npy\",mmap_mode=\"r\")\n",
    "\n",
    "time_valid=np.load(\"./time_valid_shuffled_10x.npy\")\n",
    "event_valid=np.load(\"./event_valid_shuffled_10x.npy\")\n",
    "image_valid=np.load(\"./image_valid_shuffled_10x.npy\",mmap_mode=\"r\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_fn = InputFunction(image_train, time_train, event_train,\n",
    "                  drop_last=True,\n",
    "                  augmentation=True,\n",
    "                  repeat=50,\n",
    "                  shuffle=True,\n",
    "                resize_img=False,\n",
    "                         input_size=(512,512),\n",
    "                        batch_size=16)\n",
    "eval_fn = InputFunction(image_valid, time_valid, event=event_valid, resize_img=False,\n",
    "                         input_size=(512,512))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from plexusnet.architecture import PlexusNet, LoadModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.keras.backend.clear_session()\n",
    "# PLEXUSNET WITH AVG POOLING\n",
    "model=PlexusNet(depth=5, length=2, junction=3, n_class=1, final_activation=\"sigmoid\",initial_filter=6,filter_num_for_first_convlayer=4, input_shape=(512,512),ApplyLayerNormalization=True,run_all_BN=False,type_of_block=\"soft_att\",GlobalPooling=\"avg\").model \n",
    "log_file = './log_PlexusNET_BCR_10x.txt'\n",
    "weight_folder = \"./PlexusNET_BCR_10x/\"\n",
    "EPOCHS=200\n",
    "\n",
    "# PLEXUSNET WITH ATTENTION POOLING\n",
    "'''\n",
    "# YOU NEED TO CHANGE THE INPUT FUNCTION\n",
    "model=PlexusNet(depth=6, length=2, junction=3, n_class=2, final_activation=\"softmax\",initial_filter=2,filter_num_for_first_convlayer=4, input_shape=(512,512),ApplyLayerNormalization=True,run_all_BN=False,type_of_block=\"soft_att\",get_last_conv=True,ApplyAttentionPooling=True).model \n",
    "log_file = './log_PlexusNETAttentionPooling_BCR_10x.txt'\n",
    "weight_folder = \"./PlexusNETAttentionPooling_BCR_10x/\"\n",
    "EPOCHS=200\n",
    "'''\n",
    "#ResNetRS50\n",
    "'''\n",
    "model_base=tf.keras.applications.ResNetRS50(include_top=False,classes=1,pooling=\"avg\")\n",
    "y=tf.keras.layers.Dense(1,activation=\"sigmoid\")(model_base.output)\n",
    "model=tf.keras.Model(model_base.input, y)\n",
    "log_file = './log_ResNetRS50_BCR_10x.txt'\n",
    "weight_folder = \"./ResNetRS50_BCR_10x/\"\n",
    "EPOCHS=50\n",
    "'''\n",
    "\n",
    "# EfficientNetB1\n",
    "'''\n",
    "model_base=tf.keras.applications.EfficientNetB1(include_top=False,classes=1,pooling=\"avg\")\n",
    "y=tf.keras.layers.Dense(1,activation=\"sigmoid\")(model_base.output)\n",
    "model=tf.keras.Model(model_base.input, y)\n",
    "log_file = './log_EfficientNetB1_BCR_10x.txt'\n",
    "weight_folder = \"./EfficientNetB1_BCR_10x/\"\n",
    "EPOCHS=50\n",
    "'''\n",
    "\n",
    "#VGG16\n",
    "'''\n",
    "model_base=tf.keras.applications.VGG16(include_top=False,classes=1,pooling=\"avg\")\n",
    "y=tf.keras.layers.Dense(1,activation=\"sigmoid\")(model_base.output)\n",
    "model=tf.keras.Model(model_base.input, y)\n",
    "log_file = './log_EVGG16_BCR_10x.txt'\n",
    "weight_folder = \"./VGG16_BCR_10x/\"\n",
    "EPOCHS=50\n",
    "'''\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow_addons as tfa\n",
    "from tensorflow.keras import optimizers\n",
    "lr_sh=tfa.optimizers.CyclicalLearningRate(initial_learning_rate=1e-6,\n",
    "    maximal_learning_rate=1e-3,\n",
    "    step_size=train_fn.steps_per_epoch()*4,\n",
    "    scale_fn=lambda x: 1.,\n",
    "    scale_mode=\"cycle\",\n",
    "    name=\"MyCyclicScheduler\")\n",
    "model.compile(optimizer=optimizers.Adam(lr_sh), loss=tf.keras.losses.binary_crossentropy, metrics=[tf.keras.metrics.AUC(),tf.keras.metrics.BinaryAccuracy()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_check=tf.keras.callbacks.ModelCheckpoint(\n",
    "   weight_folder +\"weight_{epoch:02d}.h5\",\n",
    "    monitor=\"val_loss\",\n",
    "    verbose=0,\n",
    "    save_best_only=False,\n",
    "    save_weights_only=False,\n",
    "    mode=\"auto\",\n",
    "    save_freq=\"epoch\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#train_fn.steps_per_epoch(),eval_fn.steps_per_epoch()\n",
    "hist=model.fit(train_fn(),steps_per_epoch=train_fn.steps_per_epoch(),epochs=EPOCHS,  \n",
    "               validation_steps=eval_fn.steps_per_epoch(), validation_data=eval_fn(),\n",
    "               callbacks=[tf.keras.callbacks.CSVLogger(log_file), \n",
    "                          model_check])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### IN-TRAINING VALIDATION - CASE LEVEL PER EPOCH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "valid_set=pd.read_csv(\"valid_set.csv\")\n",
    "valid_set=valid_set[valid_set.Filename.str.contains(\"/B/\")==False] #Exclude benign samples\n",
    "valid_set[\"X1st.BCR.Type\"].value_counts()\n",
    "valid_set[\"BCR_status\"]=1-valid_set[\"X1st.BCR.Type\"].str.contains(\"-\")\n",
    "print(valid_set[\"X1st.BCR.Type\"].value_counts())\n",
    "print(valid_set[\"BCR_status\"].value_counts())\n",
    "from lifelines.utils.concordance import concordance_index\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "from collections import defaultdict\n",
    "from tqdm import tqdm\n",
    "def GetResult(results):\n",
    "    case_lst = defaultdict(list)\n",
    "    y_true_case =defaultdict(list)\n",
    "    Gls_case = defaultdict(list)\n",
    "    time_case =defaultdict(list)\n",
    "    for i,fl in enumerate(results):\n",
    "        case_id=list(results.keys())[i].split(\"-\")[2]\n",
    "        case_lst[case_id].extend(results[fl])\n",
    "        time_case[case_id].append(valid_set[\"Interval.RP.to.BCR.or.last.contact.death\"].iloc[i])\n",
    "        y_true_case[case_id].append(valid_set.BCR_status.iloc[i])\n",
    "        Gls_case[case_id].append(list(results.keys())[i].split(\"/\")[2])\n",
    "    y_true_lst = []\n",
    "    y_pred_lst = []\n",
    "    y_time_lst = []\n",
    "    for key in y_true_case:\n",
    "        y_true_lst.append(y_true_case[key][0])\n",
    "        y_=np.histogram(case_lst[key])\n",
    "        b=np.where(y_[0]>=2)\n",
    "        _m=np.max(y_[1][b])\n",
    "        y_pred_lst.append(_m)\n",
    "        y_time_lst.append(time_case[key][0])\n",
    "    print(roc_auc_score(y_true_lst,y_pred_lst),concordance_index(y_time_lst,1-np.array(y_pred_lst),y_true_lst))\n",
    "    return {'roc':roc_auc_score(y_true_lst,y_pred_lst),\n",
    "            'cindex': concordance_index(y_time_lst,1-np.array(y_pred_lst),y_true_lst)}\n",
    "\n",
    "def RunAnalyses(model_best):\n",
    "    results=defaultdict(list)\n",
    "    heatmaps = defaultdict(list)\n",
    "    for fl in tqdm(valid_set.Filename):\n",
    "        img=np.array(PIL.Image.open(fl).resize((5120//4, 5120//4)))\n",
    "        img=img[128:-128,128:-128]\n",
    "        heatmap= np.zeros((3,3),dtype=np.float)\n",
    "        patch = []\n",
    "        for j in range(0,img.shape[0]-256,256):\n",
    "            for i in range(0, img.shape[1]-256,256):\n",
    "                patch.append(img[j:j+512,i:i+512])\n",
    "        pr=model_best.predict(np.array(patch))\n",
    "        k=0\n",
    "        for j in range(0,3):\n",
    "            for i in range(0, 3):\n",
    "                heatmap[j,i]=pr[k]\n",
    "                k+=1\n",
    "        heatmaps[fl]=heatmap\n",
    "        results[fl]=pr\n",
    "    return heatmaps,results\n",
    "\n",
    "#Run Analyze and \n",
    "heatmaps_model = {}\n",
    "results_model = {}\n",
    "cindex_model = {}\n",
    "roc_model = {}\n",
    "for epoch in range(1,201): #CHANGE HERE FOR EPOCH RANGE 201 or 51 depending on the model or the number of epochs you have trained\n",
    "    print(epoch)\n",
    "    model_best = LoadModel(f\"{weight_folder}weight_{epoch:02d}.h5\") #Remove h5 when using TF >2.9\n",
    "    heatmaps,results = RunAnalyses(model_best)\n",
    "    v=GetResult(results)\n",
    "    heatmaps_model[epoch]=heatmaps\n",
    "    results_model[epoch]=results\n",
    "    cindex_model[epoch]=v[\"cindex\"]\n",
    "    roc_model[epoch]=v[\"roc\"]\n",
    "### SELEC BEST EPOCH\n",
    "ind_x=np.argmax(list(cindex_model.values()))\n",
    "print(ind_x+1)\n",
    "print(list(cindex_model.values())[ind_x])\n",
    "print(list(roc_model.values())[ind_x])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GENERATE AND STORE PREDICTION AT PATCH LEVEL FOR DEVELOPMENT and TEST SETS (10x) \n",
    "#### CAN RUN INDEPENDENT FROM PREVIOUS CODES ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "import os\n",
    "gc.collect()\n",
    "\n",
    "\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\"\n",
    "import numpy as np\n",
    "from tensorflow.compat.v1.keras.backend import set_session\n",
    "\n",
    "import tensorflow as tf\n",
    "config = tf.compat.v1.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "set_session(tf.compat.v1.Session(config=config))\n",
    "from plexusnet.architecture import *\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "from tqdm import tqdm\n",
    "import PIL\n",
    "tf.keras.backend.clear_session()\n",
    "from plexusnet.architecture import LoadModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelx=weight_folder[2:-1]\n",
    "weight_file=\"weight.h5\" #CHANGE HERE. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_set=pd.read_csv(\"./test_set_OnlyTumor.csv\")\n",
    "development_set=pd.read_csv(\"./development_set_OnlyTumor.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path=\"./dataset/\"\n",
    "#\n",
    "# resolution_factor\n",
    "# 1 := ~40x\n",
    "# 2 := ~20x\n",
    "# 4 := ~10x\n",
    "resolution_factor = 4\n",
    "def RunAnalyses(model_best, dataset):\n",
    "    results=defaultdict(list)\n",
    "    heatmaps = defaultdict(list)\n",
    "    for fl_ in tqdm(dataset.Filename):\n",
    "        fl=data_path+fl_[1:]\n",
    "        img=np.array(PIL.Image.open(fl).resize((5120//resolution_factor, 5120//resolution_factor))) #To achieve 3x3 grid spliting (a grid cell 512x512) with overlap 50% after excluding the white non-informative area\n",
    "        img=img[128:-128,128:-128] #To reduce the white non-informative area and to achieve an 1024x1024 image\n",
    "        heatmap= np.zeros((3,3),dtype=np.float)\n",
    "        patch = []\n",
    "        for j in range(0,img.shape[0]-256,256):\n",
    "            for i in range(0, img.shape[1]-256,256):\n",
    "                img_C=np.array(PIL.Image.fromarray(img[j:j+512,i:i+512]),dtype=np.uint8)\n",
    "                patch.append(img_C)\n",
    "        pr=model_best.predict(np.array(patch), verbose=0)\n",
    "        k=0\n",
    "        if resolution_factor==4: #Out-of-Memory error issues for 20x and 40x resolution\n",
    "            for j in range(0,3):\n",
    "                for i in range(0, 3):\n",
    "                    heatmap[j,i]=pr[k]\n",
    "                    k+=1\n",
    "        heatmaps[fl]=heatmap\n",
    "        results[fl]=pr\n",
    "    return heatmaps,results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.keras.backend.clear_session\n",
    "model_best = LoadModel(f\"./{modelx}/{weight_file}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_,results_development_set = RunAnalyses(model_best, development_set)\n",
    "columns = defaultdict(list)\n",
    "for k in results_development_set:\n",
    "    for j in range(9):\n",
    "        columns[j].append(float(results_development_set[k][j][0].flatten()))\n",
    "for col in columns:\n",
    "    valid_set[col]=columns[col]\n",
    "development_set.to_csv(f\"./{modelx}_development_set.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_,results_testset = RunAnalyses(model_best, test_set)\n",
    "columns = defaultdict(list)\n",
    "for k in results_testset:\n",
    "    for j in range(9):\n",
    "        columns[j].append(float(results_testset[k][j][0].flatten()))\n",
    "for col in columns:\n",
    "    valid_set[col]=columns[col]\n",
    "test_set.to_csv(f\"{modelx}_test_set.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}