1.Data preparing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b4578561-49c0-4a1b-9506-f36983458da3",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "167afc15-3fb2-4629-b0ed-755218e436b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e02c213-2f6d-4184-9937-91612b8531d4",
   "metadata": {},
   "source": [
    "## Clean data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "be14468b-acba-4541-97c1-d0e37cc9727e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#Patient Identifier</th>\n",
       "      <th>Lymph nodes examined positive</th>\n",
       "      <th>Nottingham prognostic index</th>\n",
       "      <th>Cellularity</th>\n",
       "      <th>Chemotherapy</th>\n",
       "      <th>Cohort</th>\n",
       "      <th>ER status measured by IHC</th>\n",
       "      <th>HER2 status measured by SNP6</th>\n",
       "      <th>Hormone Therapy</th>\n",
       "      <th>Inferred Menopausal State</th>\n",
       "      <th>...</th>\n",
       "      <th>Overall Survival Status</th>\n",
       "      <th>Pam50 + Claudin-low subtype</th>\n",
       "      <th>3-Gene classifier subtype</th>\n",
       "      <th>Patient's Vital Status</th>\n",
       "      <th>Primary Tumor Laterality</th>\n",
       "      <th>Radio Therapy</th>\n",
       "      <th>Tumor Other Histologic Subtype</th>\n",
       "      <th>Type of Breast Surgery</th>\n",
       "      <th>Relapse Free Status</th>\n",
       "      <th>Relapse Free Status (Months)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>#Identifier to uniquely specify a patient.</td>\n",
       "      <td>Number of lymphnodes positive</td>\n",
       "      <td>Nottingham prognostic index</td>\n",
       "      <td>Tumor Content</td>\n",
       "      <td>Chemotherapy.</td>\n",
       "      <td>Cohort.</td>\n",
       "      <td>ER status measured by IHC</td>\n",
       "      <td>HER2 status measured by SNP6</td>\n",
       "      <td>Hormone Therapy</td>\n",
       "      <td>Inferred Menopausal State</td>\n",
       "      <td>...</td>\n",
       "      <td>Overall patient survival status.</td>\n",
       "      <td>Pam50 + Claudin-low subtype</td>\n",
       "      <td>3-Gene classifier subtype</td>\n",
       "      <td>The survival state of the person.</td>\n",
       "      <td>For tumors in paired organs, designates the si...</td>\n",
       "      <td>Radio Therapy</td>\n",
       "      <td>Text to describe a tumor's histologic subtype ...</td>\n",
       "      <td>Type of Breast Surgery</td>\n",
       "      <td>Relapse Free Status (Recurred = cases where th...</td>\n",
       "      <td>Time until last follow-up or relapse (in months)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>#STRING</td>\n",
       "      <td>STRING</td>\n",
       "      <td>NUMBER</td>\n",
       "      <td>STRING</td>\n",
       "      <td>STRING</td>\n",
       "      <td>STRING</td>\n",
       "      <td>STRING</td>\n",
       "      <td>STRING</td>\n",
       "      <td>STRING</td>\n",
       "      <td>STRING</td>\n",
       "      <td>...</td>\n",
       "      <td>STRING</td>\n",
       "      <td>STRING</td>\n",
       "      <td>STRING</td>\n",
       "      <td>STRING</td>\n",
       "      <td>STRING</td>\n",
       "      <td>STRING</td>\n",
       "      <td>STRING</td>\n",
       "      <td>STRING</td>\n",
       "      <td>STRING</td>\n",
       "      <td>NUMBER</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>#1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PATIENT_ID</td>\n",
       "      <td>LYMPH_NODES_EXAMINED_POSITIVE</td>\n",
       "      <td>NPI</td>\n",
       "      <td>CELLULARITY</td>\n",
       "      <td>CHEMOTHERAPY</td>\n",
       "      <td>COHORT</td>\n",
       "      <td>ER_IHC</td>\n",
       "      <td>HER2_SNP6</td>\n",
       "      <td>HORMONE_THERAPY</td>\n",
       "      <td>INFERRED_MENOPAUSAL_STATE</td>\n",
       "      <td>...</td>\n",
       "      <td>OS_STATUS</td>\n",
       "      <td>CLAUDIN_SUBTYPE</td>\n",
       "      <td>THREEGENE</td>\n",
       "      <td>VITAL_STATUS</td>\n",
       "      <td>LATERALITY</td>\n",
       "      <td>RADIO_THERAPY</td>\n",
       "      <td>HISTOLOGICAL_SUBTYPE</td>\n",
       "      <td>BREAST_SURGERY</td>\n",
       "      <td>RFS_STATUS</td>\n",
       "      <td>RFS_MONTHS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>MB-0000</td>\n",
       "      <td>10</td>\n",
       "      <td>6.044</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NO</td>\n",
       "      <td>1</td>\n",
       "      <td>Positve</td>\n",
       "      <td>NEUTRAL</td>\n",
       "      <td>YES</td>\n",
       "      <td>Post</td>\n",
       "      <td>...</td>\n",
       "      <td>0:LIVING</td>\n",
       "      <td>claudin-low</td>\n",
       "      <td>ER-/HER2-</td>\n",
       "      <td>Living</td>\n",
       "      <td>Right</td>\n",
       "      <td>YES</td>\n",
       "      <td>Ductal/NST</td>\n",
       "      <td>MASTECTOMY</td>\n",
       "      <td>0:Not Recurred</td>\n",
       "      <td>138.65</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          #Patient Identifier  Lymph nodes examined positive  \\\n",
       "0  #Identifier to uniquely specify a patient.  Number of lymphnodes positive   \n",
       "1                                     #STRING                         STRING   \n",
       "2                                          #1                              1   \n",
       "3                                  PATIENT_ID  LYMPH_NODES_EXAMINED_POSITIVE   \n",
       "4                                     MB-0000                             10   \n",
       "\n",
       "   Nottingham prognostic index    Cellularity   Chemotherapy   Cohort  \\\n",
       "0  Nottingham prognostic index  Tumor Content  Chemotherapy.  Cohort.   \n",
       "1                       NUMBER         STRING         STRING   STRING   \n",
       "2                            1              1              1        1   \n",
       "3                          NPI    CELLULARITY   CHEMOTHERAPY   COHORT   \n",
       "4                        6.044            NaN             NO        1   \n",
       "\n",
       "   ER status measured by IHC  HER2 status measured by SNP6  Hormone Therapy  \\\n",
       "0  ER status measured by IHC  HER2 status measured by SNP6  Hormone Therapy   \n",
       "1                     STRING                        STRING           STRING   \n",
       "2                          1                             1                1   \n",
       "3                     ER_IHC                     HER2_SNP6  HORMONE_THERAPY   \n",
       "4                    Positve                       NEUTRAL              YES   \n",
       "\n",
       "   Inferred Menopausal State  ...           Overall Survival Status  \\\n",
       "0  Inferred Menopausal State  ...  Overall patient survival status.   \n",
       "1                     STRING  ...                            STRING   \n",
       "2                          1  ...                                 1   \n",
       "3  INFERRED_MENOPAUSAL_STATE  ...                         OS_STATUS   \n",
       "4                       Post  ...                          0:LIVING   \n",
       "\n",
       "   Pam50 + Claudin-low subtype  3-Gene classifier subtype  \\\n",
       "0  Pam50 + Claudin-low subtype  3-Gene classifier subtype   \n",
       "1                       STRING                     STRING   \n",
       "2                            1                          1   \n",
       "3              CLAUDIN_SUBTYPE                  THREEGENE   \n",
       "4                  claudin-low                  ER-/HER2-   \n",
       "\n",
       "              Patient's Vital Status  \\\n",
       "0  The survival state of the person.   \n",
       "1                             STRING   \n",
       "2                                  1   \n",
       "3                       VITAL_STATUS   \n",
       "4                             Living   \n",
       "\n",
       "                            Primary Tumor Laterality  Radio Therapy  \\\n",
       "0  For tumors in paired organs, designates the si...  Radio Therapy   \n",
       "1                                             STRING         STRING   \n",
       "2                                                  1              1   \n",
       "3                                         LATERALITY  RADIO_THERAPY   \n",
       "4                                              Right            YES   \n",
       "\n",
       "                      Tumor Other Histologic Subtype  Type of Breast Surgery  \\\n",
       "0  Text to describe a tumor's histologic subtype ...  Type of Breast Surgery   \n",
       "1                                             STRING                  STRING   \n",
       "2                                                  1                       1   \n",
       "3                               HISTOLOGICAL_SUBTYPE          BREAST_SURGERY   \n",
       "4                                         Ductal/NST              MASTECTOMY   \n",
       "\n",
       "                                 Relapse Free Status  \\\n",
       "0  Relapse Free Status (Recurred = cases where th...   \n",
       "1                                             STRING   \n",
       "2                                                  1   \n",
       "3                                         RFS_STATUS   \n",
       "4                                     0:Not Recurred   \n",
       "\n",
       "                       Relapse Free Status (Months)  \n",
       "0  Time until last follow-up or relapse (in months)  \n",
       "1                                            NUMBER  \n",
       "2                                                 1  \n",
       "3                                        RFS_MONTHS  \n",
       "4                                            138.65  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clinical_data = pd.read_table('D:\\\\MyProjects\\\\Cancer detection\\\\data review\\\\brca_metabric\\\\data_clinical_patient.txt')\n",
    "clinical_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f78e620b-0dfc-4527-ad18-5409a8752da3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lymph nodes examined positive</th>\n",
       "      <th>Nottingham prognostic index</th>\n",
       "      <th>Cellularity</th>\n",
       "      <th>Chemotherapy</th>\n",
       "      <th>Cohort</th>\n",
       "      <th>ER status measured by IHC</th>\n",
       "      <th>HER2 status measured by SNP6</th>\n",
       "      <th>Hormone Therapy</th>\n",
       "      <th>Inferred Menopausal State</th>\n",
       "      <th>Sex</th>\n",
       "      <th>...</th>\n",
       "      <th>Overall Survival Status</th>\n",
       "      <th>Pam50 + Claudin-low subtype</th>\n",
       "      <th>3-Gene classifier subtype</th>\n",
       "      <th>Patient's Vital Status</th>\n",
       "      <th>Primary Tumor Laterality</th>\n",
       "      <th>Radio Therapy</th>\n",
       "      <th>Tumor Other Histologic Subtype</th>\n",
       "      <th>Type of Breast Surgery</th>\n",
       "      <th>Relapse Free Status</th>\n",
       "      <th>Relapse Free Status (Months)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Identifier</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MB-0000</th>\n",
       "      <td>10.0</td>\n",
       "      <td>6.044</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NO</td>\n",
       "      <td>1</td>\n",
       "      <td>Positve</td>\n",
       "      <td>NEUTRAL</td>\n",
       "      <td>YES</td>\n",
       "      <td>Post</td>\n",
       "      <td>Female</td>\n",
       "      <td>...</td>\n",
       "      <td>0:LIVING</td>\n",
       "      <td>claudin-low</td>\n",
       "      <td>ER-/HER2-</td>\n",
       "      <td>Living</td>\n",
       "      <td>Right</td>\n",
       "      <td>YES</td>\n",
       "      <td>Ductal/NST</td>\n",
       "      <td>MASTECTOMY</td>\n",
       "      <td>0:Not Recurred</td>\n",
       "      <td>138.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0002</th>\n",
       "      <td>0.0</td>\n",
       "      <td>4.02</td>\n",
       "      <td>High</td>\n",
       "      <td>NO</td>\n",
       "      <td>1</td>\n",
       "      <td>Positve</td>\n",
       "      <td>NEUTRAL</td>\n",
       "      <td>YES</td>\n",
       "      <td>Pre</td>\n",
       "      <td>Female</td>\n",
       "      <td>...</td>\n",
       "      <td>0:LIVING</td>\n",
       "      <td>LumA</td>\n",
       "      <td>ER+/HER2- High Prolif</td>\n",
       "      <td>Living</td>\n",
       "      <td>Right</td>\n",
       "      <td>YES</td>\n",
       "      <td>Ductal/NST</td>\n",
       "      <td>BREAST CONSERVING</td>\n",
       "      <td>0:Not Recurred</td>\n",
       "      <td>83.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0005</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4.03</td>\n",
       "      <td>High</td>\n",
       "      <td>YES</td>\n",
       "      <td>1</td>\n",
       "      <td>Positve</td>\n",
       "      <td>NEUTRAL</td>\n",
       "      <td>YES</td>\n",
       "      <td>Pre</td>\n",
       "      <td>Female</td>\n",
       "      <td>...</td>\n",
       "      <td>1:DECEASED</td>\n",
       "      <td>LumB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Died of Disease</td>\n",
       "      <td>Right</td>\n",
       "      <td>NO</td>\n",
       "      <td>Ductal/NST</td>\n",
       "      <td>MASTECTOMY</td>\n",
       "      <td>1:Recurred</td>\n",
       "      <td>151.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0006</th>\n",
       "      <td>3.0</td>\n",
       "      <td>4.05</td>\n",
       "      <td>Moderate</td>\n",
       "      <td>YES</td>\n",
       "      <td>1</td>\n",
       "      <td>Positve</td>\n",
       "      <td>NEUTRAL</td>\n",
       "      <td>YES</td>\n",
       "      <td>Pre</td>\n",
       "      <td>Female</td>\n",
       "      <td>...</td>\n",
       "      <td>0:LIVING</td>\n",
       "      <td>LumB</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Living</td>\n",
       "      <td>Right</td>\n",
       "      <td>YES</td>\n",
       "      <td>Mixed</td>\n",
       "      <td>MASTECTOMY</td>\n",
       "      <td>0:Not Recurred</td>\n",
       "      <td>162.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0008</th>\n",
       "      <td>8.0</td>\n",
       "      <td>6.08</td>\n",
       "      <td>High</td>\n",
       "      <td>YES</td>\n",
       "      <td>1</td>\n",
       "      <td>Positve</td>\n",
       "      <td>NEUTRAL</td>\n",
       "      <td>YES</td>\n",
       "      <td>Post</td>\n",
       "      <td>Female</td>\n",
       "      <td>...</td>\n",
       "      <td>1:DECEASED</td>\n",
       "      <td>LumB</td>\n",
       "      <td>ER+/HER2- High Prolif</td>\n",
       "      <td>Died of Disease</td>\n",
       "      <td>Right</td>\n",
       "      <td>YES</td>\n",
       "      <td>Mixed</td>\n",
       "      <td>MASTECTOMY</td>\n",
       "      <td>1:Recurred</td>\n",
       "      <td>18.55</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Lymph nodes examined positive Nottingham prognostic index  \\\n",
       "Identifier                                                              \n",
       "MB-0000                              10.0                       6.044   \n",
       "MB-0002                               0.0                        4.02   \n",
       "MB-0005                               1.0                        4.03   \n",
       "MB-0006                               3.0                        4.05   \n",
       "MB-0008                               8.0                        6.08   \n",
       "\n",
       "           Cellularity Chemotherapy Cohort ER status measured by IHC  \\\n",
       "Identifier                                                             \n",
       "MB-0000            NaN           NO      1                   Positve   \n",
       "MB-0002           High           NO      1                   Positve   \n",
       "MB-0005           High          YES      1                   Positve   \n",
       "MB-0006       Moderate          YES      1                   Positve   \n",
       "MB-0008           High          YES      1                   Positve   \n",
       "\n",
       "           HER2 status measured by SNP6 Hormone Therapy  \\\n",
       "Identifier                                                \n",
       "MB-0000                         NEUTRAL             YES   \n",
       "MB-0002                         NEUTRAL             YES   \n",
       "MB-0005                         NEUTRAL             YES   \n",
       "MB-0006                         NEUTRAL             YES   \n",
       "MB-0008                         NEUTRAL             YES   \n",
       "\n",
       "           Inferred Menopausal State     Sex  ... Overall Survival Status  \\\n",
       "Identifier                                    ...                           \n",
       "MB-0000                         Post  Female  ...                0:LIVING   \n",
       "MB-0002                          Pre  Female  ...                0:LIVING   \n",
       "MB-0005                          Pre  Female  ...              1:DECEASED   \n",
       "MB-0006                          Pre  Female  ...                0:LIVING   \n",
       "MB-0008                         Post  Female  ...              1:DECEASED   \n",
       "\n",
       "           Pam50 + Claudin-low subtype 3-Gene classifier subtype  \\\n",
       "Identifier                                                         \n",
       "MB-0000                    claudin-low                 ER-/HER2-   \n",
       "MB-0002                           LumA     ER+/HER2- High Prolif   \n",
       "MB-0005                           LumB                       NaN   \n",
       "MB-0006                           LumB                       NaN   \n",
       "MB-0008                           LumB     ER+/HER2- High Prolif   \n",
       "\n",
       "           Patient's Vital Status Primary Tumor Laterality Radio Therapy  \\\n",
       "Identifier                                                                 \n",
       "MB-0000                    Living                    Right           YES   \n",
       "MB-0002                    Living                    Right           YES   \n",
       "MB-0005           Died of Disease                    Right            NO   \n",
       "MB-0006                    Living                    Right           YES   \n",
       "MB-0008           Died of Disease                    Right           YES   \n",
       "\n",
       "           Tumor Other Histologic Subtype Type of Breast Surgery  \\\n",
       "Identifier                                                         \n",
       "MB-0000                        Ductal/NST             MASTECTOMY   \n",
       "MB-0002                        Ductal/NST      BREAST CONSERVING   \n",
       "MB-0005                        Ductal/NST             MASTECTOMY   \n",
       "MB-0006                             Mixed             MASTECTOMY   \n",
       "MB-0008                             Mixed             MASTECTOMY   \n",
       "\n",
       "           Relapse Free Status Relapse Free Status (Months)  \n",
       "Identifier                                                   \n",
       "MB-0000         0:Not Recurred                       138.65  \n",
       "MB-0002         0:Not Recurred                        83.52  \n",
       "MB-0005             1:Recurred                       151.28  \n",
       "MB-0006         0:Not Recurred                       162.76  \n",
       "MB-0008             1:Recurred                        18.55  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clinical_data.drop([0,1,2,3], axis=0,inplace=True)\n",
    "new_idx = clinical_data[['#Patient Identifier']]\n",
    "new_idx = new_idx.rename({'#Patient Identifier':'Identifier'}, axis=1)\n",
    "clinical_data.drop(['#Patient Identifier'], axis=1, inplace=True)\n",
    "clinical_data.index = new_idx['Identifier']\n",
    "clinical_data['Lymph nodes examined positive'] = clinical_data['Lymph nodes examined positive'].apply(lambda x: float(x))\n",
    "clinical_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "eeba5a96-a949-4604-8b42-6c68d8f8d935",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Hugo_Symbol</th>\n",
       "      <th>Entrez_Gene_Id</th>\n",
       "      <th>MB-0362</th>\n",
       "      <th>MB-0346</th>\n",
       "      <th>MB-0386</th>\n",
       "      <th>MB-0574</th>\n",
       "      <th>MB-0503</th>\n",
       "      <th>MB-0641</th>\n",
       "      <th>MB-0201</th>\n",
       "      <th>MB-0218</th>\n",
       "      <th>...</th>\n",
       "      <th>MB-6122</th>\n",
       "      <th>MB-6192</th>\n",
       "      <th>MB-4820</th>\n",
       "      <th>MB-5527</th>\n",
       "      <th>MB-5167</th>\n",
       "      <th>MB-5465</th>\n",
       "      <th>MB-5453</th>\n",
       "      <th>MB-5471</th>\n",
       "      <th>MB-5127</th>\n",
       "      <th>MB-4313</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RERE</td>\n",
       "      <td>473.0</td>\n",
       "      <td>8.676978</td>\n",
       "      <td>9.653589</td>\n",
       "      <td>9.033589</td>\n",
       "      <td>8.814855</td>\n",
       "      <td>9.274265</td>\n",
       "      <td>9.286585</td>\n",
       "      <td>8.437347</td>\n",
       "      <td>8.569973</td>\n",
       "      <td>...</td>\n",
       "      <td>8.756024</td>\n",
       "      <td>8.804947</td>\n",
       "      <td>9.991215</td>\n",
       "      <td>9.595923</td>\n",
       "      <td>9.637249</td>\n",
       "      <td>8.131637</td>\n",
       "      <td>9.606915</td>\n",
       "      <td>9.049296</td>\n",
       "      <td>8.858622</td>\n",
       "      <td>8.415867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RNF165</td>\n",
       "      <td>494470.0</td>\n",
       "      <td>6.075331</td>\n",
       "      <td>6.687887</td>\n",
       "      <td>5.910885</td>\n",
       "      <td>5.628740</td>\n",
       "      <td>5.908698</td>\n",
       "      <td>6.206729</td>\n",
       "      <td>6.095592</td>\n",
       "      <td>6.383530</td>\n",
       "      <td>...</td>\n",
       "      <td>7.101087</td>\n",
       "      <td>5.601472</td>\n",
       "      <td>7.103160</td>\n",
       "      <td>6.418987</td>\n",
       "      <td>6.203370</td>\n",
       "      <td>9.101942</td>\n",
       "      <td>7.427494</td>\n",
       "      <td>6.850000</td>\n",
       "      <td>6.550450</td>\n",
       "      <td>6.831722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CD049690</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.453928</td>\n",
       "      <td>5.454185</td>\n",
       "      <td>5.501577</td>\n",
       "      <td>5.471941</td>\n",
       "      <td>5.531743</td>\n",
       "      <td>5.372668</td>\n",
       "      <td>5.693519</td>\n",
       "      <td>5.401276</td>\n",
       "      <td>...</td>\n",
       "      <td>5.344674</td>\n",
       "      <td>5.364348</td>\n",
       "      <td>5.504866</td>\n",
       "      <td>5.277314</td>\n",
       "      <td>5.439207</td>\n",
       "      <td>5.423027</td>\n",
       "      <td>5.534115</td>\n",
       "      <td>5.339346</td>\n",
       "      <td>5.566071</td>\n",
       "      <td>5.541395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BC033982</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.994525</td>\n",
       "      <td>5.346010</td>\n",
       "      <td>5.247467</td>\n",
       "      <td>5.316523</td>\n",
       "      <td>5.244094</td>\n",
       "      <td>5.167365</td>\n",
       "      <td>5.189106</td>\n",
       "      <td>5.416517</td>\n",
       "      <td>...</td>\n",
       "      <td>5.300466</td>\n",
       "      <td>4.861101</td>\n",
       "      <td>4.980904</td>\n",
       "      <td>5.436855</td>\n",
       "      <td>5.346776</td>\n",
       "      <td>4.939292</td>\n",
       "      <td>5.062191</td>\n",
       "      <td>5.166765</td>\n",
       "      <td>5.140141</td>\n",
       "      <td>5.266802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PHF7</td>\n",
       "      <td>51533.0</td>\n",
       "      <td>5.838270</td>\n",
       "      <td>5.600876</td>\n",
       "      <td>6.030718</td>\n",
       "      <td>5.849428</td>\n",
       "      <td>5.964661</td>\n",
       "      <td>5.783374</td>\n",
       "      <td>5.737572</td>\n",
       "      <td>5.923928</td>\n",
       "      <td>...</td>\n",
       "      <td>5.653481</td>\n",
       "      <td>5.922894</td>\n",
       "      <td>6.181493</td>\n",
       "      <td>5.992153</td>\n",
       "      <td>6.230316</td>\n",
       "      <td>5.644587</td>\n",
       "      <td>5.927409</td>\n",
       "      <td>6.117095</td>\n",
       "      <td>5.936371</td>\n",
       "      <td>7.408960</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1906 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Hugo_Symbol  Entrez_Gene_Id   MB-0362   MB-0346   MB-0386   MB-0574  \\\n",
       "0        RERE           473.0  8.676978  9.653589  9.033589  8.814855   \n",
       "1      RNF165        494470.0  6.075331  6.687887  5.910885  5.628740   \n",
       "2    CD049690             NaN  5.453928  5.454185  5.501577  5.471941   \n",
       "3    BC033982             NaN  4.994525  5.346010  5.247467  5.316523   \n",
       "4        PHF7         51533.0  5.838270  5.600876  6.030718  5.849428   \n",
       "\n",
       "    MB-0503   MB-0641   MB-0201   MB-0218  ...   MB-6122   MB-6192   MB-4820  \\\n",
       "0  9.274265  9.286585  8.437347  8.569973  ...  8.756024  8.804947  9.991215   \n",
       "1  5.908698  6.206729  6.095592  6.383530  ...  7.101087  5.601472  7.103160   \n",
       "2  5.531743  5.372668  5.693519  5.401276  ...  5.344674  5.364348  5.504866   \n",
       "3  5.244094  5.167365  5.189106  5.416517  ...  5.300466  4.861101  4.980904   \n",
       "4  5.964661  5.783374  5.737572  5.923928  ...  5.653481  5.922894  6.181493   \n",
       "\n",
       "    MB-5527   MB-5167   MB-5465   MB-5453   MB-5471   MB-5127   MB-4313  \n",
       "0  9.595923  9.637249  8.131637  9.606915  9.049296  8.858622  8.415867  \n",
       "1  6.418987  6.203370  9.101942  7.427494  6.850000  6.550450  6.831722  \n",
       "2  5.277314  5.439207  5.423027  5.534115  5.339346  5.566071  5.541395  \n",
       "3  5.436855  5.346776  4.939292  5.062191  5.166765  5.140141  5.266802  \n",
       "4  5.992153  6.230316  5.644587  5.927409  6.117095  5.936371  7.408960  \n",
       "\n",
       "[5 rows x 1906 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene_data = pd.read_table('D:\\\\MyProjects\\\\Cancer detection\\\\data review\\\\brca_metabric\\\\data_mrna_agilent_microarray.txt')\n",
    "gene_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6f6b1b08-30ef-461a-9305-c16d0d03749e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Identifier</th>\n",
       "      <th>RERE</th>\n",
       "      <th>RNF165</th>\n",
       "      <th>CD049690</th>\n",
       "      <th>BC033982</th>\n",
       "      <th>PHF7</th>\n",
       "      <th>CIDEA</th>\n",
       "      <th>PAPD4</th>\n",
       "      <th>AI082173</th>\n",
       "      <th>SLC17A3</th>\n",
       "      <th>SDS</th>\n",
       "      <th>...</th>\n",
       "      <th>BX115874</th>\n",
       "      <th>BX107598</th>\n",
       "      <th>UGCGL1</th>\n",
       "      <th>VPS72</th>\n",
       "      <th>CSMD3</th>\n",
       "      <th>CC2D1A</th>\n",
       "      <th>CB986545</th>\n",
       "      <th>IGSF9</th>\n",
       "      <th>DA110839</th>\n",
       "      <th>FAM71A</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MB-0362</th>\n",
       "      <td>8.676978</td>\n",
       "      <td>6.075331</td>\n",
       "      <td>5.453928</td>\n",
       "      <td>4.994525</td>\n",
       "      <td>5.838270</td>\n",
       "      <td>6.397503</td>\n",
       "      <td>7.906217</td>\n",
       "      <td>5.259461</td>\n",
       "      <td>5.702379</td>\n",
       "      <td>6.930741</td>\n",
       "      <td>...</td>\n",
       "      <td>5.271343</td>\n",
       "      <td>5.680321</td>\n",
       "      <td>7.688492</td>\n",
       "      <td>8.084979</td>\n",
       "      <td>5.161796</td>\n",
       "      <td>6.353215</td>\n",
       "      <td>4.836483</td>\n",
       "      <td>7.304643</td>\n",
       "      <td>5.251843</td>\n",
       "      <td>5.049591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0346</th>\n",
       "      <td>9.653589</td>\n",
       "      <td>6.687887</td>\n",
       "      <td>5.454185</td>\n",
       "      <td>5.346010</td>\n",
       "      <td>5.600876</td>\n",
       "      <td>5.246319</td>\n",
       "      <td>8.267256</td>\n",
       "      <td>5.380069</td>\n",
       "      <td>5.521794</td>\n",
       "      <td>6.141689</td>\n",
       "      <td>...</td>\n",
       "      <td>5.942887</td>\n",
       "      <td>5.461069</td>\n",
       "      <td>7.804165</td>\n",
       "      <td>8.349115</td>\n",
       "      <td>5.197392</td>\n",
       "      <td>6.132355</td>\n",
       "      <td>5.316819</td>\n",
       "      <td>7.933324</td>\n",
       "      <td>5.450611</td>\n",
       "      <td>5.316790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0386</th>\n",
       "      <td>9.033589</td>\n",
       "      <td>5.910885</td>\n",
       "      <td>5.501577</td>\n",
       "      <td>5.247467</td>\n",
       "      <td>6.030718</td>\n",
       "      <td>10.111816</td>\n",
       "      <td>7.959291</td>\n",
       "      <td>5.262024</td>\n",
       "      <td>5.689533</td>\n",
       "      <td>6.529312</td>\n",
       "      <td>...</td>\n",
       "      <td>5.174498</td>\n",
       "      <td>5.304030</td>\n",
       "      <td>7.934309</td>\n",
       "      <td>8.406332</td>\n",
       "      <td>8.087722</td>\n",
       "      <td>6.366335</td>\n",
       "      <td>5.466419</td>\n",
       "      <td>7.580336</td>\n",
       "      <td>5.235394</td>\n",
       "      <td>5.461617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0574</th>\n",
       "      <td>8.814855</td>\n",
       "      <td>5.628740</td>\n",
       "      <td>5.471941</td>\n",
       "      <td>5.316523</td>\n",
       "      <td>5.849428</td>\n",
       "      <td>6.116868</td>\n",
       "      <td>9.206376</td>\n",
       "      <td>5.396576</td>\n",
       "      <td>5.439130</td>\n",
       "      <td>6.430102</td>\n",
       "      <td>...</td>\n",
       "      <td>5.116749</td>\n",
       "      <td>5.632249</td>\n",
       "      <td>7.744562</td>\n",
       "      <td>8.310019</td>\n",
       "      <td>5.780062</td>\n",
       "      <td>6.424048</td>\n",
       "      <td>5.193150</td>\n",
       "      <td>6.903654</td>\n",
       "      <td>5.091927</td>\n",
       "      <td>5.227130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0503</th>\n",
       "      <td>9.274265</td>\n",
       "      <td>5.908698</td>\n",
       "      <td>5.531743</td>\n",
       "      <td>5.244094</td>\n",
       "      <td>5.964661</td>\n",
       "      <td>7.828171</td>\n",
       "      <td>8.706646</td>\n",
       "      <td>5.167213</td>\n",
       "      <td>5.417484</td>\n",
       "      <td>6.684893</td>\n",
       "      <td>...</td>\n",
       "      <td>5.402314</td>\n",
       "      <td>5.472185</td>\n",
       "      <td>7.701394</td>\n",
       "      <td>8.137014</td>\n",
       "      <td>5.498185</td>\n",
       "      <td>6.214301</td>\n",
       "      <td>5.274600</td>\n",
       "      <td>6.839417</td>\n",
       "      <td>5.315224</td>\n",
       "      <td>5.027476</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24368 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Identifier      RERE    RNF165  CD049690  BC033982      PHF7      CIDEA  \\\n",
       "MB-0362     8.676978  6.075331  5.453928  4.994525  5.838270   6.397503   \n",
       "MB-0346     9.653589  6.687887  5.454185  5.346010  5.600876   5.246319   \n",
       "MB-0386     9.033589  5.910885  5.501577  5.247467  6.030718  10.111816   \n",
       "MB-0574     8.814855  5.628740  5.471941  5.316523  5.849428   6.116868   \n",
       "MB-0503     9.274265  5.908698  5.531743  5.244094  5.964661   7.828171   \n",
       "\n",
       "Identifier     PAPD4  AI082173   SLC17A3       SDS  ...  BX115874  BX107598  \\\n",
       "MB-0362     7.906217  5.259461  5.702379  6.930741  ...  5.271343  5.680321   \n",
       "MB-0346     8.267256  5.380069  5.521794  6.141689  ...  5.942887  5.461069   \n",
       "MB-0386     7.959291  5.262024  5.689533  6.529312  ...  5.174498  5.304030   \n",
       "MB-0574     9.206376  5.396576  5.439130  6.430102  ...  5.116749  5.632249   \n",
       "MB-0503     8.706646  5.167213  5.417484  6.684893  ...  5.402314  5.472185   \n",
       "\n",
       "Identifier    UGCGL1     VPS72     CSMD3    CC2D1A  CB986545     IGSF9  \\\n",
       "MB-0362     7.688492  8.084979  5.161796  6.353215  4.836483  7.304643   \n",
       "MB-0346     7.804165  8.349115  5.197392  6.132355  5.316819  7.933324   \n",
       "MB-0386     7.934309  8.406332  8.087722  6.366335  5.466419  7.580336   \n",
       "MB-0574     7.744562  8.310019  5.780062  6.424048  5.193150  6.903654   \n",
       "MB-0503     7.701394  8.137014  5.498185  6.214301  5.274600  6.839417   \n",
       "\n",
       "Identifier  DA110839    FAM71A  \n",
       "MB-0362     5.251843  5.049591  \n",
       "MB-0346     5.450611  5.316790  \n",
       "MB-0386     5.235394  5.461617  \n",
       "MB-0574     5.091927  5.227130  \n",
       "MB-0503     5.315224  5.027476  \n",
       "\n",
       "[5 rows x 24368 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene_data.drop(\"Entrez_Gene_Id\", axis=1, inplace=True)\n",
    "new_idx = gene_data[['Hugo_Symbol']]\n",
    "new_idx = new_idx.rename({'Hugo_Symbol':'Identifier'}, axis=1)\n",
    "gene_data.drop(['Hugo_Symbol'], axis=1, inplace=True)\n",
    "gene_data.index = new_idx['Identifier']\n",
    "gene_data = gene_data.transpose()\n",
    "gene_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "06f9d2d2-a402-422a-abd1-1b6830c2cdf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "genes_set = set(gene_data.index)\n",
    "clinical_set = set(clinical_data.index)\n",
    "to_del_from_clinical = list(clinical_set - genes_set)\n",
    "clinical_data.drop(to_del_from_clinical, axis=0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "31cad092-4652-41d3-b4ad-0d1e9417edba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Lymph nodes examined positive</th>\n",
       "      <th>Pam50 + Claudin-low subtype</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Identifier</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MB-0000</th>\n",
       "      <td>10.0</td>\n",
       "      <td>claudin-low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0002</th>\n",
       "      <td>0.0</td>\n",
       "      <td>LumA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0005</th>\n",
       "      <td>1.0</td>\n",
       "      <td>LumB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0006</th>\n",
       "      <td>3.0</td>\n",
       "      <td>LumB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0008</th>\n",
       "      <td>8.0</td>\n",
       "      <td>LumB</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Lymph nodes examined positive Pam50 + Claudin-low subtype\n",
       "Identifier                                                           \n",
       "MB-0000                              10.0                 claudin-low\n",
       "MB-0002                               0.0                        LumA\n",
       "MB-0005                               1.0                        LumB\n",
       "MB-0006                               3.0                        LumB\n",
       "MB-0008                               8.0                        LumB"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clinical_to_drop = ['Nottingham prognostic index',\n",
    "       'Cellularity', 'Chemotherapy', 'Cohort', 'ER status measured by IHC',\n",
    "       'HER2 status measured by SNP6', 'Hormone Therapy',\n",
    "       'Inferred Menopausal State', 'Sex', 'Integrative Cluster',\n",
    "       'Age at Diagnosis', 'Overall Survival (Months)',\n",
    "       'Overall Survival Status','3-Gene classifier subtype', \"Patient's Vital Status\",\n",
    "       'Primary Tumor Laterality', 'Radio Therapy',\n",
    "       'Tumor Other Histologic Subtype', 'Type of Breast Surgery',\n",
    "       'Relapse Free Status', 'Relapse Free Status (Months)']\n",
    "clinical_data_useful_only = clinical_data.drop(clinical_to_drop, axis=1)\n",
    "clinical_data_useful_only.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7576aef2-d2da-4f32-9a63-52b15ebd8bbc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1904, 23), (1904, 24368))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clinical_data.shape, gene_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2e1bd0db-c922-4475-8a93-4edc28072624",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['LumA', 'Her2', 'claudin-low', 'Basal', 'LumB', 'Normal', 'NC'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = gene_data.join(clinical_data_useful_only) \n",
    "data['Pam50 + Claudin-low subtype'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "084abed1-eae1-4c95-9375-ccd725e5451d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RERE</th>\n",
       "      <th>RNF165</th>\n",
       "      <th>CD049690</th>\n",
       "      <th>BC033982</th>\n",
       "      <th>PHF7</th>\n",
       "      <th>CIDEA</th>\n",
       "      <th>PAPD4</th>\n",
       "      <th>AI082173</th>\n",
       "      <th>SLC17A3</th>\n",
       "      <th>SDS</th>\n",
       "      <th>...</th>\n",
       "      <th>UGCGL1</th>\n",
       "      <th>VPS72</th>\n",
       "      <th>CSMD3</th>\n",
       "      <th>CC2D1A</th>\n",
       "      <th>CB986545</th>\n",
       "      <th>IGSF9</th>\n",
       "      <th>DA110839</th>\n",
       "      <th>FAM71A</th>\n",
       "      <th>Lymph nodes examined positive</th>\n",
       "      <th>Pam50 + Claudin-low subtype</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MB-0362</th>\n",
       "      <td>8.676978</td>\n",
       "      <td>6.075331</td>\n",
       "      <td>5.453928</td>\n",
       "      <td>4.994525</td>\n",
       "      <td>5.838270</td>\n",
       "      <td>6.397503</td>\n",
       "      <td>7.906217</td>\n",
       "      <td>5.259461</td>\n",
       "      <td>5.702379</td>\n",
       "      <td>6.930741</td>\n",
       "      <td>...</td>\n",
       "      <td>7.688492</td>\n",
       "      <td>8.084979</td>\n",
       "      <td>5.161796</td>\n",
       "      <td>6.353215</td>\n",
       "      <td>4.836483</td>\n",
       "      <td>7.304643</td>\n",
       "      <td>5.251843</td>\n",
       "      <td>5.049591</td>\n",
       "      <td>1.0</td>\n",
       "      <td>LumA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0346</th>\n",
       "      <td>9.653589</td>\n",
       "      <td>6.687887</td>\n",
       "      <td>5.454185</td>\n",
       "      <td>5.346010</td>\n",
       "      <td>5.600876</td>\n",
       "      <td>5.246319</td>\n",
       "      <td>8.267256</td>\n",
       "      <td>5.380069</td>\n",
       "      <td>5.521794</td>\n",
       "      <td>6.141689</td>\n",
       "      <td>...</td>\n",
       "      <td>7.804165</td>\n",
       "      <td>8.349115</td>\n",
       "      <td>5.197392</td>\n",
       "      <td>6.132355</td>\n",
       "      <td>5.316819</td>\n",
       "      <td>7.933324</td>\n",
       "      <td>5.450611</td>\n",
       "      <td>5.316790</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Her2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0386</th>\n",
       "      <td>9.033589</td>\n",
       "      <td>5.910885</td>\n",
       "      <td>5.501577</td>\n",
       "      <td>5.247467</td>\n",
       "      <td>6.030718</td>\n",
       "      <td>10.111816</td>\n",
       "      <td>7.959291</td>\n",
       "      <td>5.262024</td>\n",
       "      <td>5.689533</td>\n",
       "      <td>6.529312</td>\n",
       "      <td>...</td>\n",
       "      <td>7.934309</td>\n",
       "      <td>8.406332</td>\n",
       "      <td>8.087722</td>\n",
       "      <td>6.366335</td>\n",
       "      <td>5.466419</td>\n",
       "      <td>7.580336</td>\n",
       "      <td>5.235394</td>\n",
       "      <td>5.461617</td>\n",
       "      <td>8.0</td>\n",
       "      <td>LumA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0574</th>\n",
       "      <td>8.814855</td>\n",
       "      <td>5.628740</td>\n",
       "      <td>5.471941</td>\n",
       "      <td>5.316523</td>\n",
       "      <td>5.849428</td>\n",
       "      <td>6.116868</td>\n",
       "      <td>9.206376</td>\n",
       "      <td>5.396576</td>\n",
       "      <td>5.439130</td>\n",
       "      <td>6.430102</td>\n",
       "      <td>...</td>\n",
       "      <td>7.744562</td>\n",
       "      <td>8.310019</td>\n",
       "      <td>5.780062</td>\n",
       "      <td>6.424048</td>\n",
       "      <td>5.193150</td>\n",
       "      <td>6.903654</td>\n",
       "      <td>5.091927</td>\n",
       "      <td>5.227130</td>\n",
       "      <td>1.0</td>\n",
       "      <td>LumA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0503</th>\n",
       "      <td>9.274265</td>\n",
       "      <td>5.908698</td>\n",
       "      <td>5.531743</td>\n",
       "      <td>5.244094</td>\n",
       "      <td>5.964661</td>\n",
       "      <td>7.828171</td>\n",
       "      <td>8.706646</td>\n",
       "      <td>5.167213</td>\n",
       "      <td>5.417484</td>\n",
       "      <td>6.684893</td>\n",
       "      <td>...</td>\n",
       "      <td>7.701394</td>\n",
       "      <td>8.137014</td>\n",
       "      <td>5.498185</td>\n",
       "      <td>6.214301</td>\n",
       "      <td>5.274600</td>\n",
       "      <td>6.839417</td>\n",
       "      <td>5.315224</td>\n",
       "      <td>5.027476</td>\n",
       "      <td>0.0</td>\n",
       "      <td>LumA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24370 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             RERE    RNF165  CD049690  BC033982      PHF7      CIDEA  \\\n",
       "MB-0362  8.676978  6.075331  5.453928  4.994525  5.838270   6.397503   \n",
       "MB-0346  9.653589  6.687887  5.454185  5.346010  5.600876   5.246319   \n",
       "MB-0386  9.033589  5.910885  5.501577  5.247467  6.030718  10.111816   \n",
       "MB-0574  8.814855  5.628740  5.471941  5.316523  5.849428   6.116868   \n",
       "MB-0503  9.274265  5.908698  5.531743  5.244094  5.964661   7.828171   \n",
       "\n",
       "            PAPD4  AI082173   SLC17A3       SDS  ...    UGCGL1     VPS72  \\\n",
       "MB-0362  7.906217  5.259461  5.702379  6.930741  ...  7.688492  8.084979   \n",
       "MB-0346  8.267256  5.380069  5.521794  6.141689  ...  7.804165  8.349115   \n",
       "MB-0386  7.959291  5.262024  5.689533  6.529312  ...  7.934309  8.406332   \n",
       "MB-0574  9.206376  5.396576  5.439130  6.430102  ...  7.744562  8.310019   \n",
       "MB-0503  8.706646  5.167213  5.417484  6.684893  ...  7.701394  8.137014   \n",
       "\n",
       "            CSMD3    CC2D1A  CB986545     IGSF9  DA110839    FAM71A  \\\n",
       "MB-0362  5.161796  6.353215  4.836483  7.304643  5.251843  5.049591   \n",
       "MB-0346  5.197392  6.132355  5.316819  7.933324  5.450611  5.316790   \n",
       "MB-0386  8.087722  6.366335  5.466419  7.580336  5.235394  5.461617   \n",
       "MB-0574  5.780062  6.424048  5.193150  6.903654  5.091927  5.227130   \n",
       "MB-0503  5.498185  6.214301  5.274600  6.839417  5.315224  5.027476   \n",
       "\n",
       "         Lymph nodes examined positive  Pam50 + Claudin-low subtype  \n",
       "MB-0362                            1.0                         LumA  \n",
       "MB-0346                            5.0                         Her2  \n",
       "MB-0386                            8.0                         LumA  \n",
       "MB-0574                            1.0                         LumA  \n",
       "MB-0503                            0.0                         LumA  \n",
       "\n",
       "[5 rows x 24370 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normal_idx = list(data[data['Pam50 + Claudin-low subtype'] == 'Normal'].index)\n",
    "NC_idx = list(data[data['Pam50 + Claudin-low subtype'] == 'NC'].index)\n",
    "claudin_low_idx = list(data[data['Pam50 + Claudin-low subtype'] == 'claudin-low'].index)\n",
    "indices_to_drop = normal_idx + NC_idx + claudin_low_idx\n",
    "data.drop(indices_to_drop, axis=0, inplace=True)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cdb35a86-1894-44c1-a30d-7655d9c6d1d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data.dropna()\n",
    "data.isnull().values.any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3e795ff0-9d66-47af-a32e-2274c36da40a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Subtype</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MB-0362</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0346</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0386</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0574</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MB-0503</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Subtype\n",
       "MB-0362        2\n",
       "MB-0346        1\n",
       "MB-0386        2\n",
       "MB-0574        2\n",
       "MB-0503        2"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y1 = data.loc[:,'Pam50 + Claudin-low subtype']\n",
    "le = LabelEncoder()\n",
    "y = le.fit_transform(y1)\n",
    "\n",
    "labels = le.classes_\n",
    "\n",
    "X = data.drop(\"Pam50 + Claudin-low subtype\", axis=1)\n",
    "y = pd.DataFrame(y)\n",
    "y.index = X.index\n",
    "y.columns = ['Subtype']\n",
    "y.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1555ce1c-1027-488c-9406-84f943a07fbc",
   "metadata": {},
   "source": [
    "## Split data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4d6b697a-29ff-43cc-9a1b-2761047bb8d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state = 0, stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1016525a-b32f-4f26-ac43-175d17d0ba7b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1395, 24369), (156, 24369))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e87205c8-ffd8-4e24-a4c8-ba5ae5eb8321",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Subtype\n",
       "2          606\n",
       "3          414\n",
       "1          196\n",
       "0          179\n",
       "dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "dd8412ab-9b47-4e9c-9aba-4730d497123b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Basal', 'Her2', 'LumA', 'LumB'], dtype=object)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22b8766d-1326-438b-bb25-097d1c33e593",
   "metadata": {},
   "source": [
    "## Scale data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d8e19294-068b-4732-b0a9-61608fc5a922",
   "metadata": {},
   "outputs": [],
   "source": [
    "ss = StandardScaler()\n",
    "X_train_scaled = ss.fit_transform(X_train)\n",
    "X_test_scaled = ss.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9019a50-f012-4224-9f0a-9e8465d29ce9",
   "metadata": {},
   "source": [
    "## Save data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b5c52105-a204-4ea3-967a-0a42f8b925d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('Saved data/X_train.pickle', 'wb') as output:\n",
    "    pickle.dump(X_train, output)\n",
    "    \n",
    "with open('Saved data/X_test.pickle', 'wb') as output:\n",
    "    pickle.dump(X_test, output)\n",
    "\n",
    "with open('Saved data/X_train_scaled.pickle', 'wb') as output:\n",
    "    pickle.dump(X_train_scaled, output)\n",
    "    \n",
    "with open('Saved data/X_test_scaled.pickle', 'wb') as output:\n",
    "    pickle.dump(X_test_scaled, output)\n",
    "        \n",
    "with open('Saved data/y_train.pickle', 'wb') as output:\n",
    "    pickle.dump(y_train, output)\n",
    "    \n",
    "with open('Saved data/y_test.pickle', 'wb') as output:\n",
    "    pickle.dump(y_test, output)\n",
    "    \n",
    "with open('Saved data/labels.pickle', 'wb') as output:\n",
    "    pickle.dump(labels, output)\n",
    "    \n",
    "with open('Saved models & utils/standardScaler.pickle', 'wb') as output:\n",
    "    pickle.dump(ss, output)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cancer",
   "language": "python",
   "name": "cancer"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}