Split_between_PanCancer_and_Lung_Cancer_Data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09a18106",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "pan_gene_expre=pd.read_hdf('data/Preprocessed_Data/PanCancer.h5',key='gene_expression')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57056bfc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pan_gene_expre.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee25fba2",
   "metadata": {},
   "outputs": [],
   "source": [
    "pan_gene_expre.isnull().values.any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3288f271",
   "metadata": {},
   "outputs": [],
   "source": [
    " pan_clinical=pd.read_hdf('data/Preprocessed_Data/PanCancer.h5',key='clinical')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c529e1fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "pan_clinical.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70df38d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "pan_clinical.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a36478ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "pan_clinical.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89b7df1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "pan_clinical.index.duplicated().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61b7a3a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "pan_clinical['cancer type abbreviation'].value_counts(normalize=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "051a30fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "lung_index=pan_clinical[pan_clinical['cancer type abbreviation'].apply(lambda x: x in ['LUAD','LUSC'])].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "447b8b1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "lung_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b725741",
   "metadata": {},
   "outputs": [],
   "source": [
    "pan_sample_types=pd.read_hdf('data/Preprocessed_Data/PanCancer.h5',key='sample_type')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6399e5e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "pan_sample_types.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73682aec",
   "metadata": {},
   "outputs": [],
   "source": [
    "pan_sample_types.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "742d8d18",
   "metadata": {},
   "outputs": [],
   "source": [
    "pan_sample_types.index.duplicated().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70411661",
   "metadata": {},
   "outputs": [],
   "source": [
    "pan_sample_types.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77a54dfb",
   "metadata": {},
   "outputs": [],
   "source": [
    "pan_sample_types.index.intersection(lung_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "802a50c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "lung_gene_expre=pan_gene_expre.loc[lung_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86a1e21a",
   "metadata": {},
   "outputs": [],
   "source": [
    "lung_gene_expre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50f9a262",
   "metadata": {},
   "outputs": [],
   "source": [
    "lung_sample_types=pan_sample_types.loc[lung_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7bc9baa",
   "metadata": {},
   "outputs": [],
   "source": [
    "lung_sample_types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e1a2b2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "lung_gene_expre.index.equals(lung_sample_types.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "660df92c",
   "metadata": {},
   "outputs": [],
   "source": [
    "lung_survival=pan_clinical.loc[lung_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d002a69a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "lung_survival"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16b108fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "lung_gene_expre.index.equals(lung_survival.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a33c69e",
   "metadata": {},
   "outputs": [],
   "source": [
    "variable='pathology_status'\n",
    "print('number of samples with this information:',sum(lung_sample_types[variable].value_counts(normalize=False,dropna=False)))\n",
    "lung_sample_types[variable].value_counts(normalize=False,dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42b5dce2",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('number of samples with this information:',sum(lung_survival.OS.value_counts(normalize=False,dropna=False)))\n",
    "lung_survival.OS.value_counts(normalize=False,dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "605b3e2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('number of samples with this information:',sum(lung_survival.PFI.value_counts(normalize=False,dropna=False)))\n",
    "lung_survival.PFI.value_counts(normalize=False,dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10103f45",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('number of samples with this information:',sum(lung_survival.DSS.value_counts(normalize=False,dropna=False)))\n",
    "lung_survival.DSS.value_counts(normalize=False,dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "686913e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('number of samples with this information:',sum(lung_survival.DFI.value_counts(normalize=False,dropna=False)))\n",
    "lung_survival.DFI.value_counts(normalize=False,dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff2e1acc",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "with pd.HDFStore('data/Preprocessed_Data/Lung.h5','w') as store:\n",
    "    store['gene_expression']=lung_gene_expre\n",
    "    store['sample_type']=lung_sample_types\n",
    "    store['clinical']=lung_survival"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ed6b89b",
   "metadata": {},
   "outputs": [],
   "source": [
    "non_lung_index=pan_clinical.index.difference(lung_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25aec562",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(non_lung_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2cbbe62",
   "metadata": {},
   "outputs": [],
   "source": [
    "pan_clinical.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a6808e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "lung_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aea76868",
   "metadata": {},
   "outputs": [],
   "source": [
    "10496-1122"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89aad82e",
   "metadata": {},
   "outputs": [],
   "source": [
    "non_lung_gene_expre=pan_gene_expre.loc[non_lung_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73ba5d9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "non_lung_gene_expre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51b012fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "non_lung_index=non_lung_gene_expre.index.intersection(pan_sample_types.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea574800",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(non_lung_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00fed28d",
   "metadata": {},
   "outputs": [],
   "source": [
    "non_lung_sample_types=pan_sample_types.loc[non_lung_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "874ef254",
   "metadata": {},
   "outputs": [],
   "source": [
    "non_lung_sample_types.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8131025",
   "metadata": {},
   "outputs": [],
   "source": [
    "non_lung_sample_types.pathology_status.value_counts(normalize=False,dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c877e5f5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "non_lung_sample_types.sample_type.value_counts(normalize=False,dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1638b28",
   "metadata": {},
   "outputs": [],
   "source": [
    "non_lung_clinical_index=non_lung_gene_expre.index.intersection(pan_clinical.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2522aadc",
   "metadata": {},
   "outputs": [],
   "source": [
    "non_lung_clinical_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9d90e9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "non_lung_clinical=pan_clinical.loc[non_lung_clinical_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfcb91a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "non_lung_clinical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3ec86ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "with pd.HDFStore('data/Preprocessed_Data/non_lung.h5','w') as store:\n",
    "    store['gene_expression']=non_lung_gene_expre\n",
    "    store['sample_type']=non_lung_sample_types\n",
    "    store['clinical']=non_lung_clinical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e04497d5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}