all-features/4-matrixfy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Combine all features into a single matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import bz2\n",
    "import os\n",
    "import configparser\n",
    "import itertools\n",
    "\n",
    "import pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "unperm_name = 'rephetio-v2.0'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read partitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hetnet</th>\n",
       "      <th>compound_id</th>\n",
       "      <th>disease_id</th>\n",
       "      <th>status</th>\n",
       "      <th>primary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>38860</th>\n",
       "      <td>rephetio-v2.0_perm-4</td>\n",
       "      <td>DB08906</td>\n",
       "      <td>DOID:986</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38861</th>\n",
       "      <td>rephetio-v2.0_perm-5</td>\n",
       "      <td>DB08906</td>\n",
       "      <td>DOID:986</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     hetnet compound_id disease_id  status  primary\n",
       "38860  rephetio-v2.0_perm-4     DB08906   DOID:986       0        0\n",
       "38861  rephetio-v2.0_perm-5     DB08906   DOID:986       0        0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read partition information\n",
    "part_df = pandas.read_table('data/partitions.tsv')\n",
    "part_df.tail(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create DWPC matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hetnet</th>\n",
       "      <th>compound_id</th>\n",
       "      <th>disease_id</th>\n",
       "      <th>metapath</th>\n",
       "      <th>PC</th>\n",
       "      <th>w</th>\n",
       "      <th>DWPC</th>\n",
       "      <th>seconds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>46867570</th>\n",
       "      <td>rephetio-v2.0</td>\n",
       "      <td>DB08881</td>\n",
       "      <td>DOID:1909</td>\n",
       "      <td>CdGeAeGaD</td>\n",
       "      <td>1813210</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.236651</td>\n",
       "      <td>303.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46867571</th>\n",
       "      <td>rephetio-v2.0</td>\n",
       "      <td>DB08881</td>\n",
       "      <td>DOID:1612</td>\n",
       "      <td>CdGeAeGaD</td>\n",
       "      <td>3085218</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.336426</td>\n",
       "      <td>377.3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 hetnet compound_id disease_id   metapath       PC    w  \\\n",
       "46867570  rephetio-v2.0     DB08881  DOID:1909  CdGeAeGaD  1813210  0.4   \n",
       "46867571  rephetio-v2.0     DB08881  DOID:1612  CdGeAeGaD  3085218  0.4   \n",
       "\n",
       "              DWPC  seconds  \n",
       "46867570  0.236651    303.3  \n",
       "46867571  0.336426    377.3  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read DWPC results\n",
    "dwpc_df = pandas.read_table('data/dwpc.tsv.bz2')\n",
    "dwpc_df.tail(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>metapath</th>\n",
       "      <th>hetnet</th>\n",
       "      <th>compound_id</th>\n",
       "      <th>disease_id</th>\n",
       "      <th>CbG&lt;rG&lt;rGaD</th>\n",
       "      <th>CbG&lt;rG&lt;rGdD</th>\n",
       "      <th>CbG&lt;rG&lt;rGuD</th>\n",
       "      <th>CbG&lt;rGaD</th>\n",
       "      <th>CbG&lt;rGaDrD</th>\n",
       "      <th>CbG&lt;rGbCpD</th>\n",
       "      <th>CbG&lt;rGbCtD</th>\n",
       "      <th>...</th>\n",
       "      <th>CuGuDpCpD</th>\n",
       "      <th>CuGuDpCtD</th>\n",
       "      <th>CuGuDpSpD</th>\n",
       "      <th>CuGuDrD</th>\n",
       "      <th>CuGuDrDrD</th>\n",
       "      <th>CuGuDtCpD</th>\n",
       "      <th>CuGuDtCtD</th>\n",
       "      <th>CuGuDuGaD</th>\n",
       "      <th>CuGuDuGdD</th>\n",
       "      <th>CuGuDuGuD</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>rephetio-v2.0</td>\n",
       "      <td>DB00014</td>\n",
       "      <td>DOID:10283</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000736</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.002448</td>\n",
       "      <td>0.000273</td>\n",
       "      <td>0.001891</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>rephetio-v2.0</td>\n",
       "      <td>DB00014</td>\n",
       "      <td>DOID:10534</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001628</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00123</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001286</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 1209 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "metapath         hetnet compound_id  disease_id  CbG<rG<rGaD  CbG<rG<rGdD  \\\n",
       "0         rephetio-v2.0     DB00014  DOID:10283          0.0          0.0   \n",
       "1         rephetio-v2.0     DB00014  DOID:10534          0.0          0.0   \n",
       "\n",
       "metapath  CbG<rG<rGuD  CbG<rGaD  CbG<rGaDrD  CbG<rGbCpD  CbG<rGbCtD  \\\n",
       "0                 0.0       0.0         0.0         0.0         0.0   \n",
       "1                 0.0       0.0         0.0         0.0         0.0   \n",
       "\n",
       "metapath    ...      CuGuDpCpD  CuGuDpCtD  CuGuDpSpD  CuGuDrD  CuGuDrDrD  \\\n",
       "0           ...            0.0        0.0   0.000736      0.0    0.00000   \n",
       "1           ...            0.0        0.0   0.001628      0.0    0.00123   \n",
       "\n",
       "metapath  CuGuDtCpD  CuGuDtCtD  CuGuDuGaD  CuGuDuGdD  CuGuDuGuD  \n",
       "0          0.002448   0.000273   0.001891        0.0   0.000196  \n",
       "1          0.000000   0.000000   0.001286        0.0   0.000000  \n",
       "\n",
       "[2 rows x 1209 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dwpc_spread_df = pandas.pivot_table(dwpc_df, values='DWPC', index=['hetnet', 'compound_id', 'disease_id'], columns='metapath')\n",
    "dwpc_spread_df = dwpc_spread_df.reset_index()\n",
    "dwpc_spread_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove metapaths with missing DWPCs\n",
    "# See https://github.com/dhimmel/learn/issues/1 for potential cause\n",
    "is_complete = pandas.isnull(dwpc_spread_df).sum() == 0\n",
    "\n",
    "# Remove columns with missing data\n",
    "dwpc_spread_df = dwpc_spread_df.loc[:, is_complete]\n",
    "\n",
    "# The following columns have missing values and were removed\n",
    "is_complete.index[-is_complete].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "path = 'data/matrix/dwpc.tsv.bz2'\n",
    "with bz2.open(path, 'wt') as wf:\n",
    "    dwpc_spread_df.to_csv(wf, index=False, sep='\\t', float_format='%.5g')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate Degree features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "config = configparser.ConfigParser()\n",
    "config.read('../config.ini')\n",
    "commit = config['hetnet']['integrate_commit']\n",
    "\n",
    "url = 'https://github.com/dhimmel/integrate/raw/{}/data/summary/metaedge-styles.tsv'.format(commit)\n",
    "metaedge_style_df = pandas.read_table(url)\n",
    "metaedge_to_abbreviation = dict(zip(metaedge_style_df.metaedge, metaedge_style_df.abbreviation))\n",
    "\n",
    "url = 'https://github.com/dhimmel/integrate/raw/{}/data/summary/degrees.xlsx'.format(commit)\n",
    "\n",
    "disease_degree_df = pandas.read_excel(url, sheetname='Disease')\n",
    "disease_degree_df = disease_degree_df.rename(columns={'node_id': 'disease_id'}).drop('node_name', axis='columns')\n",
    "disease_degree_df = disease_degree_df.rename(columns=metaedge_to_abbreviation)\n",
    "\n",
    "compound_degree_df = pandas.read_excel(url, sheetname='Compound')\n",
    "compound_degree_df = compound_degree_df.rename(columns={'node_id': 'compound_id'}).drop('node_name', axis='columns')\n",
    "compound_degree_df = compound_degree_df.rename(columns=metaedge_to_abbreviation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>compound_id</th>\n",
       "      <th>CbG</th>\n",
       "      <th>CcSE</th>\n",
       "      <th>CdG</th>\n",
       "      <th>CiPC</th>\n",
       "      <th>CpD</th>\n",
       "      <th>CrC</th>\n",
       "      <th>CtD</th>\n",
       "      <th>CuG</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DB00014</td>\n",
       "      <td>2</td>\n",
       "      <td>249</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DB00035</td>\n",
       "      <td>5</td>\n",
       "      <td>142</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  compound_id  CbG  CcSE  CdG  CiPC  CpD  CrC  CtD  CuG\n",
       "0     DB00014    2   249    0     1    0    7    2    1\n",
       "1     DB00035    5   142    0     3    0   11    0    0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compound_degree_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>disease_id</th>\n",
       "      <th>DaG</th>\n",
       "      <th>DdG</th>\n",
       "      <th>DlA</th>\n",
       "      <th>DpC</th>\n",
       "      <th>DpS</th>\n",
       "      <th>DrD</th>\n",
       "      <th>DtC</th>\n",
       "      <th>DuG</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DOID:0050156</td>\n",
       "      <td>18</td>\n",
       "      <td>250</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DOID:0050425</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>21</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     disease_id  DaG  DdG  DlA  DpC  DpS  DrD  DtC  DuG\n",
       "0  DOID:0050156   18  250    4    1    8    2    0  250\n",
       "1  DOID:0050425   12    0   16   10   21    6    0    0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "disease_degree_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "compound_degree_df.to_csv('data/matrix/compound_degree.tsv', index=False, sep='\\t')\n",
    "disease_degree_df.to_csv('data/matrix/disease_degree.tsv', index=False, sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute prior dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(209168, 29799)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read compound and disease degrees\n",
    "compound_df = pandas.read_table('../summary/compounds.tsv')\n",
    "disease_df = pandas.read_table('../summary/diseases.tsv')\n",
    "total_pairs = len(compound_df) * len(disease_df)\n",
    "nonzero_prior_pairs = sum(compound_df.treats > 0) * sum(disease_df.treats > 0)\n",
    "total_pairs, nonzero_prior_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>compound_id</th>\n",
       "      <th>disease_id</th>\n",
       "      <th>prior_prob</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DB01048</td>\n",
       "      <td>DOID:10652</td>\n",
       "      <td>0.004753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>DB01048</td>\n",
       "      <td>DOID:9206</td>\n",
       "      <td>0.002341</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  compound_id  disease_id  prior_prob\n",
       "0     DB01048  DOID:10652    0.004753\n",
       "1     DB01048   DOID:9206    0.002341"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rows = list(itertools.product(compound_df.compound_id, disease_df.disease_id))\n",
    "prior_df = pandas.DataFrame(rows, columns=['compound_id', 'disease_id']).merge(\n",
    "  pandas.read_table('../prior/data/observation-prior.tsv')[['compound_id', 'disease_id', 'prior_perm']], how='left'\n",
    ").fillna(0)\n",
    "prior_df = prior_df.rename(columns={'prior_perm': 'prior_prob'})\n",
    "prior_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "755.00004861999798"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(prior_df.prior_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    0.857536\n",
       "True     0.142464\n",
       "Name: prior_prob, dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(prior_df.prior_prob > 0).value_counts(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "prior_df.to_csv('data/matrix/prior.tsv', index=False, sep='\\t', float_format='%.5g')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a single matrix-like dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "matrix_df = part_df.merge(disease_df.iloc[:, :2]).merge(compound_df.iloc[:, :2])\n",
    "matrix_df = matrix_df.merge(prior_df)\n",
    "matrix_df = matrix_df.merge(compound_degree_df).merge(disease_degree_df)\n",
    "matrix_df = matrix_df.merge(dwpc_spread_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hetnet</th>\n",
       "      <th>compound_id</th>\n",
       "      <th>disease_id</th>\n",
       "      <th>status</th>\n",
       "      <th>primary</th>\n",
       "      <th>disease_name</th>\n",
       "      <th>compound_name</th>\n",
       "      <th>prior_prob</th>\n",
       "      <th>CbG</th>\n",
       "      <th>CcSE</th>\n",
       "      <th>...</th>\n",
       "      <th>CuGuDpCpD</th>\n",
       "      <th>CuGuDpCtD</th>\n",
       "      <th>CuGuDpSpD</th>\n",
       "      <th>CuGuDrD</th>\n",
       "      <th>CuGuDrDrD</th>\n",
       "      <th>CuGuDtCpD</th>\n",
       "      <th>CuGuDtCtD</th>\n",
       "      <th>CuGuDuGaD</th>\n",
       "      <th>CuGuDuGdD</th>\n",
       "      <th>CuGuDuGuD</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>rephetio-v2.0_perm-5</td>\n",
       "      <td>DB00014</td>\n",
       "      <td>DOID:0060073</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>lymphatic system cancer</td>\n",
       "      <td>Goserelin</td>\n",
       "      <td>0.022675</td>\n",
       "      <td>2</td>\n",
       "      <td>249</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>rephetio-v2.0</td>\n",
       "      <td>DB00091</td>\n",
       "      <td>DOID:0060073</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>lymphatic system cancer</td>\n",
       "      <td>Cyclosporine</td>\n",
       "      <td>0.047417</td>\n",
       "      <td>30</td>\n",
       "      <td>344</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.028591</td>\n",
       "      <td>0.004754</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 1230 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 hetnet compound_id    disease_id  status  primary  \\\n",
       "0  rephetio-v2.0_perm-5     DB00014  DOID:0060073       0        1   \n",
       "1         rephetio-v2.0     DB00091  DOID:0060073       0        1   \n",
       "\n",
       "              disease_name compound_name  prior_prob  CbG  CcSE    ...      \\\n",
       "0  lymphatic system cancer     Goserelin    0.022675    2   249    ...       \n",
       "1  lymphatic system cancer  Cyclosporine    0.047417   30   344    ...       \n",
       "\n",
       "   CuGuDpCpD  CuGuDpCtD  CuGuDpSpD  CuGuDrD  CuGuDrDrD  CuGuDtCpD  CuGuDtCtD  \\\n",
       "0        0.0        0.0        0.0      0.0        0.0        0.0   0.000000   \n",
       "1        0.0        0.0        0.0      0.0        0.0        0.0   0.028591   \n",
       "\n",
       "   CuGuDuGaD  CuGuDuGdD  CuGuDuGuD  \n",
       "0   0.000000        0.0        0.0  \n",
       "1   0.004754        0.0        0.0  \n",
       "\n",
       "[2 rows x 1230 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_creators = [\n",
    "    {'feature_type': 'prior', 'feature': ['prior_prob']},\n",
    "    {'feature_type': 'degree', 'feature': compound_degree_df.columns[1:]},\n",
    "    {'feature_type': 'degree', 'feature': disease_degree_df.columns[1:]},\n",
    "    {'feature_type': 'dwpc', 'feature': dwpc_spread_df.columns[3:]},\n",
    "]\n",
    "feature_df = pandas.concat(map(pandas.DataFrame, df_creators))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>feature_type</th>\n",
       "      <th>unperm_mean</th>\n",
       "      <th>unperm_sd</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>prior_prob</td>\n",
       "      <td>prior</td>\n",
       "      <td>0.040476</td>\n",
       "      <td>0.073232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CbG</td>\n",
       "      <td>degree</td>\n",
       "      <td>11.311258</td>\n",
       "      <td>12.443931</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      feature feature_type  unperm_mean  unperm_sd\n",
       "0  prior_prob        prior     0.040476   0.073232\n",
       "0         CbG       degree    11.311258  12.443931"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unperm_matrix_df = matrix_df.query(\"hetnet == @unperm_name\").drop('hetnet', axis='columns')\n",
    "feature_df['unperm_mean'] = list(unperm_matrix_df[feature_df.feature].mean())\n",
    "feature_df['unperm_sd'] = list(unperm_matrix_df[feature_df.feature].std())\n",
    "feature_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_df.to_csv('data/matrix/feature-type.tsv', index=False, sep='\\t', float_format='%.5g')\n",
    "\n",
    "path = 'data/matrix/features.tsv.bz2'\n",
    "with bz2.open(path, 'wt') as wf:\n",
    "    matrix_df.to_csv(wf, index=False, sep='\\t', float_format='%.5g')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Save hetnet specific feature files\n",
    "directory = os.path.join('data', 'matrix', unperm_name)\n",
    "if not os.path.exists(directory):\n",
    "    os.mkdir(directory)\n",
    "path = os.path.join(directory, 'features.tsv.bz2')\n",
    "with bz2.open(path, 'wt') as wf:\n",
    "    unperm_matrix_df.to_csv(wf, index=False, sep='\\t', float_format='%.5g')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}