Pathway_Gene_Expre.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f3478993",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import itertools\n",
    "from sklearn import preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9755630a",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "#load kegg data\n",
    "hsa = pd.read_csv('F:/MyData/deepSurvival/data/KEGG/KEGG_gene_hsa.csv', sep='\\t', \n",
    "                  engine='python', header=None, index_col=None, names=['keggId', 'geneName'])\n",
    "hsa_brite = pd.read_csv('F:/MyData/deepSurvival/data/KEGG/KEGG_hsa_brite.csv', \n",
    "                        sep='\\t', engine='python', header=None, index_col=None, names=['keggBriteId', 'keggId'])\n",
    "brite = pd.read_csv('F:/MyData/deepSurvival/data/KEGG/KEGG_brite.csv', \n",
    "                    engine='python', header=0, index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ff470af",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "#load gene expression data\n",
    "tcga_expre = pd.read_hdf(\"F:/MyData/deepSurvival/data/GSE37745/tcga_geo_expre.h5\", \n",
    "                      key = \"tcga\")\n",
    "geo_expre = pd.read_hdf(\"F:/MyData/deepSurvival/data/GSE37745/tcga_geo_expre.h5\", \n",
    "                      key = \"geo\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c29ad497",
   "metadata": {},
   "outputs": [],
   "source": [
    "tcga_expre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28e8afd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "geo_expre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0af381b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#replace <0 values\n",
    "def replace_negative(df):\n",
    "    # checking the element is < 0\n",
    "    df[df < 0] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ba248c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "tcga_expre=tcga_expre.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f38d4cb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "tcga_expre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a50e0ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "geo_expre=geo_expre.T\n",
    "geo_expre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b168a6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#from multiprocessing import Pool\n",
    "#import time\n",
    "#P=Pool(processes=8)\n",
    "for i in list(tcga_expre.columns):\n",
    "   # get the max and min values\n",
    "    Max = np.max(tcga_expre[i])\n",
    "    Min = np.min(tcga_expre[i])\n",
    "    tcga_expre[i] = (tcga_expre[i] - Min)/(Max - Min)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a55d5f6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "tcga_expre=tcga_expre.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7bc33775",
   "metadata": {},
   "outputs": [],
   "source": [
    "tcga_expre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4de41f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in list(geo_expre.columns):\n",
    "    Max = np.max(geo_expre[i])\n",
    "    Min = np.min(geo_expre[i])\n",
    "    geo_expre[i] = (geo_expre[i] - Min)/(Max - Min)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b22880d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "geo_expre=geo_expre.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "583e8fb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "geo_expre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28d79d0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "tcga_clinical=pd.read_hdf('E:/MyData/deepSurvival/data/Preprocessed_Data/Lung.h5',key='clinical')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "686aa03e",
   "metadata": {},
   "outputs": [],
   "source": [
    "tcga_clinical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd859685",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(tcga_clinical.index.intersection(list(map(lambda x: x.split('_')[0], tcga_expre.index))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b6d8c07",
   "metadata": {},
   "outputs": [],
   "source": [
    "ens_ids=tcga_expre.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d066ea2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "ens_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f8b316b",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"The number of genes: {}\".format(len(ens_ids)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d4f9ba1",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Total number of samples (patients): {}\".format(tcga_expre.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "240fc128",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# load ensembol-hugo dict\n",
    "hugo_to_ens = pd.read_csv('E:/MyData/deepSurvival/data/PanCancer/hugo_to_ens.tsv', \n",
    "                       sep='\\t', engine='python', index_col=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2694eb01",
   "metadata": {},
   "outputs": [],
   "source": [
    "hugo_to_ens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a598ad2",
   "metadata": {},
   "outputs": [],
   "source": [
    "geneId=hugo_to_ens['geneId']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70b92d77",
   "metadata": {},
   "outputs": [],
   "source": [
    "ensembl=[]\n",
    "for ids in geneId:\n",
    "    ensembl.append(ids[0:15])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17275627",
   "metadata": {},
   "outputs": [],
   "source": [
    "ensembl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d7936dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "hugo_to_ens['ensembl']=ensembl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90dfa814",
   "metadata": {},
   "outputs": [],
   "source": [
    "hugo_to_ens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ff9b281",
   "metadata": {},
   "outputs": [],
   "source": [
    "hugo_to_ens.drop_duplicates(subset='ensembl',keep='first')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19382d18",
   "metadata": {},
   "outputs": [],
   "source": [
    "hugo_to_ens=hugo_to_ens.drop_duplicates(subset='ensembl',keep='first')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54f6c06c",
   "metadata": {},
   "outputs": [],
   "source": [
    "hugo_to_ens = hugo_to_ens[hugo_to_ens['ensembl'].isin(ens_ids)]\n",
    "hugo_to_ens.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0507ab84",
   "metadata": {},
   "outputs": [],
   "source": [
    "hugo_to_ens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0367d740",
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_mapping = hugo_to_ens[['ensembl', 'geneName']]\n",
    "gene_mapping.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9524380e",
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(gene_mapping.duplicated())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24c8a6cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(set(gene_mapping['geneName']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c10714f",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(set(gene_mapping['ensembl']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc79b93e",
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(map(lambda x: ',' in x or ';' in x, gene_mapping.geneName))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3923d26b",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "tcga_gene_map = tcga_expre.T\n",
    "tcga_gene_map['ensembl'] = tcga_gene_map.index\n",
    "tcga_gene_map['order'] = tcga_expre.mean(axis=0) # sort genes by mean expression values \n",
    "tcga_gene_map = pd.merge(gene_mapping, tcga_gene_map, on='ensembl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f02f4f82",
   "metadata": {},
   "outputs": [],
   "source": [
    "tcga_gene_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ee51658",
   "metadata": {},
   "outputs": [],
   "source": [
    "geo_clinical=pd.read_hdf('E:/MyData/deepSurvival/data/GSE37745/GSE37745.h5',key='clinical')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23c7e395",
   "metadata": {},
   "outputs": [],
   "source": [
    "geo_clinical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4965aa26",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(geo_clinical.index.intersection(list(map(lambda x: x.split('_')[0], geo_expre.index))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2197a098",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 1.36 s\n",
      "Wall time: 1.51 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#load ensembol-hugo dict\n",
    "hugo_to_ens = pd.read_csv('F:/MyData/deepSurvival/data/PanCancer/hugo_to_ens.tsv', \n",
    "                       sep='\\t', engine='python', index_col=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af6a59fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "geneId=hugo_to_ens['geneId']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "858b26f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "ensembl=[]\n",
    "for ids in geneId:\n",
    "    ensembl.append(ids[0:15])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55f5ef3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "hugo_to_ens['ensembl']=ensembl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60dd76f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "hugo_to_ens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6fba0a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "hugo_to_ens=hugo_to_ens.drop_duplicates(subset='ensembl',keep='first')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "421cf7d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "hugo_to_ens = hugo_to_ens[hugo_to_ens['ensembl'].isin(ens_ids)]\n",
    "hugo_to_ens.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61b79fd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_mapping = hugo_to_ens[['ensembl', 'geneName']]\n",
    "gene_mapping.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c35a574a",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "geo_gene_map = geo_expre.T\n",
    "geo_gene_map['ensembl'] = geo_gene_map.index\n",
    "geo_gene_map['order'] = geo_expre.mean(axis=0) # sort genes by mean expression values \n",
    "geo_gene_map = pd.merge(gene_mapping, geo_gene_map, on='ensembl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5d73ca0",
   "metadata": {},
   "outputs": [],
   "source": [
    "geo_gene_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05463b89",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "# Save dataset\n",
    "with pd.HDFStore('data/hugo_ensembl_geneExpre_map.h5','w') as store:\n",
    "    store['tcga_gene_map']=tcga_gene_map\n",
    "    #store['non_lung_gene_map_log2tpm']=non_lung_gene_map\n",
    "    store['geo_gene_map']=geo_gene_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58540003",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}