02_abs_matrix.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating matrices representing the difference in branch A and B"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### done before (s. 01_Trajectories_processing): \n",
    "Dataframes as csv for all windows of the two branches with each entry being a transcription factor, a list of genes and the corresponding list of weights representing the interaction between the tf and the genes. This gives a network of tfs, genes and connections for each branch in each window. Next we will look for interactions that strongly vary in branch A and branch B by taking the absolute values between the weights."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH_TO_DATAFRAMES='data_files/dataframes/'\n",
    "PATH_TO_ABS_MATRICES='data_files/abs/'\n",
    "PATH_TO_ABS_MATRICES_WO_0='data_files/abs_wo0/' # removing all entries with difference 0 from the dataframes\n",
    "NUM_WINDOWS=5  # check with before"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import glob\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for all windows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "branchA_csvs=glob.glob(f\"{PATH_TO_DATAFRAMES}A_*\")\n",
    "branchA_csvs=[os.path.basename(csv) for csv in branchA_csvs]\n",
    "\n",
    "branchB_csvs=glob.glob(f\"{PATH_TO_DATAFRAMES}B_*.csv\")\n",
    "branchB_csvs=[os.path.basename(csv) for csv in branchB_csvs]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create those matrices with containing absolute values 0, which makes the result files very large. Below is a version where those entries are ignored."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current window: 0\n",
      "current window: 1\n",
      "current window: 2\n",
      "current window: 3\n",
      "current window: 4\n"
     ]
    }
   ],
   "source": [
    "for k in range(NUM_WINDOWS):\n",
    "    print(f'current window: {k}')\n",
    "    # get the files for the current window\n",
    "    for branchA_csv in branchA_csvs:\n",
    "        if (int)(branchA_csv.split('_')[1])==k:\n",
    "            current_A=branchA_csv\n",
    "            df_a = pd.read_csv(f\"{PATH_TO_DATAFRAMES}{current_A}\")  \n",
    "            break\n",
    "        else: \n",
    "            current_A=None\n",
    "            df_a=pd.DataFrame(columns=['TF', 'Genes', 'Weights'])\n",
    "    \n",
    "    for branchB_csv in branchB_csvs:\n",
    "        if (int)(branchB_csv.split('_')[1])==k:\n",
    "            current_B=branchB_csv\n",
    "            df_b = pd.read_csv(f\"{PATH_TO_DATAFRAMES}{current_B}\")\n",
    "            break\n",
    "        else:\n",
    "            current_B=None\n",
    "            df_b=pd.DataFrame(columns=['TF', 'Genes', 'Weights'])\n",
    "            \n",
    "    # continue with next window if both branches have no entries in this window  \n",
    "    if current_A is None and current_B is None:\n",
    "        continue\n",
    "          \n",
    "    \n",
    "    # get the tfs for the current window\n",
    "    tfs = set(df_a['TF'])\n",
    "    tfs = tfs.union(set(df_b['TF']))\n",
    "    \n",
    "    # get the genes for the current window\n",
    "    genes = set()\n",
    "    \n",
    "\n",
    "    # make sure the genes are saved as a list\n",
    "    for i in range(len(df_a['TF'])):\n",
    "        curr_genes = df_a['Genes'][i].split(\"'\")\n",
    "        curr_set = set(curr_genes)\n",
    "        genes = genes.union(curr_set)\n",
    "    for i in range(len(df_b['TF'])):\n",
    "        curr_genes = df_b['Genes'][i].split(\"'\")\n",
    "        curr_set = set(curr_genes)\n",
    "        genes = genes.union(curr_set)\n",
    "    # remove nonsense from list\n",
    "    genes.remove('[')\n",
    "    genes.remove(']')\n",
    "    genes.remove(', ')\n",
    "    \n",
    "    \n",
    "    # define tfs, genes and weights for current a and b\n",
    "    a_tfs = list(df_a['TF'])\n",
    "    a_genes = list(df_a['Genes'])\n",
    "    a_weights = list(df_a['Weights'])\n",
    "    b_tfs = list(df_b['TF'])\n",
    "    b_genes = list(df_b['Genes'])\n",
    "    b_weights = list(df_b['Weights'])\n",
    "    \n",
    "    \n",
    "    # insert a null value for the first column\n",
    "    genes = sorted(genes) # alphabtical order\n",
    "    genes.insert(0,'NULL') # as column for TFs\n",
    "    \n",
    "    \n",
    "    # create adjacency matrix for a and b\n",
    "    a_rows = []\n",
    "    a_rows.append(genes)\n",
    "    b_rows = []\n",
    "    b_rows.append(genes)\n",
    "    for tf in tfs:\n",
    "        adjacencies = [tf]\n",
    "        \n",
    "        # read weights from dataframe for branch A and add 0.0 if connection between tf and gene does not exist here\n",
    "        if tf in a_tfs:\n",
    "            tf_index = a_tfs.index(tf)\n",
    "\n",
    "            curr_genes = a_genes[tf_index].split(\"'\")\n",
    "            curr_genes.remove('[')\n",
    "            curr_genes.remove(']')\n",
    "            while ', ' in curr_genes:\n",
    "                curr_genes.remove(', ')\n",
    "\n",
    "            for gene in genes: \n",
    "                weight = 0.0\n",
    "                if gene in curr_genes:\n",
    "                    gene_index = curr_genes.index(gene)\n",
    "                    curr_weights = a_weights[tf_index].split(\",\")    \n",
    "                    weight = curr_weights[gene_index]\n",
    "                    # strip [ or ] from number if 1st or last weight\n",
    "                    if (gene_index == len(curr_genes)-1):\n",
    "                        weight = weight.split(\"]\")[0]\n",
    "                    elif gene_index == 0:\n",
    "                        weight = weight.split(\"[\")[1]\n",
    "                    weight = float(weight)\n",
    "\n",
    "                adjacencies.append(weight)\n",
    "        else:\n",
    "            for gene in genes:\n",
    "                adjacencies.append(0.0)\n",
    "        a_rows.append(adjacencies) \n",
    "    \n",
    "        # read weights from dataframe for branch B and add 0.0 if connection between tf and gene does not exist here\n",
    "        adjacencies = [tf]\n",
    "        if tf in b_tfs:\n",
    "            tf_index = b_tfs.index(tf)\n",
    "\n",
    "            curr_genes = b_genes[tf_index].split(\"'\")\n",
    "            curr_genes.remove('[')\n",
    "            curr_genes.remove(']')\n",
    "            while ', ' in curr_genes:\n",
    "                curr_genes.remove(', ')\n",
    "\n",
    "            for gene in genes:\n",
    "                weight=0.0\n",
    "                if gene in curr_genes:\n",
    "                    gene_index = curr_genes.index(gene)\n",
    "                    curr_weights = b_weights[tf_index].split(\",\")               \n",
    "                    weight = curr_weights[gene_index]\n",
    "                    # strip [ or ] from number if 1st or last weight\n",
    "                    if (gene_index == len(curr_genes)-1):\n",
    "                        weight = weight.split(\"]\")[0]\n",
    "                    elif gene_index == 0:\n",
    "                        weight = weight.split(\"[\")[1]\n",
    "                    weight = float(weight)\n",
    "                    \n",
    "                adjacencies.append(weight)\n",
    "        else:\n",
    "            for gene in genes:\n",
    "                adjacencies.append(0.0)\n",
    "        b_rows.append(adjacencies)    \n",
    "        \n",
    "        \n",
    "    # build the matrix with the absolut values for each window\n",
    "    abs_rows = []\n",
    "    #abs_rows.append(genes)\n",
    "\n",
    "    for i in range(1,len(a_rows)):\n",
    "        for j in range(1, len(genes)):\n",
    "            row = []\n",
    "            row.append(a_rows[i][0])\n",
    "            row.append(genes[j])\n",
    "            value = abs(a_rows[i][j+1]-b_rows[i][j+1])\n",
    "            row.append(value)\n",
    "            abs_rows.append(row)\n",
    "    \n",
    "    # save the results\n",
    "    win_dataframe = pd.DataFrame(abs_rows)\n",
    "    win_dataframe.to_csv(f\"{PATH_TO_ABS_MATRICES}win{k}_abs.csv\", header = ['TF', 'GENE', 'abs'], index=False)\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Better version: Create those matrices but leave out absolute values that are 0. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH_TO_WINDOW_WEIGHTS = 'data_files/windows_weights/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current window: 0\n",
      "current window: 1\n",
      "current window: 2\n",
      "current window: 3\n",
      "current window: 4\n"
     ]
    }
   ],
   "source": [
    "for k in range(NUM_WINDOWS):\n",
    "    print(f'current window: {k}')\n",
    "    # get the files for the current window\n",
    "    for branchA_csv in branchA_csvs:\n",
    "        if (int)(branchA_csv.split('_')[1])==k:\n",
    "            current_A=branchA_csv\n",
    "            df_a = pd.read_csv(f\"{PATH_TO_DATAFRAMES}{current_A}\")  \n",
    "            break\n",
    "        else: \n",
    "            current_A=None\n",
    "            df_a=pd.DataFrame(columns=['TF', 'Genes', 'Weights'])\n",
    "    \n",
    "    for branchB_csv in branchB_csvs:\n",
    "        if (int)(branchB_csv.split('_')[1])==k:\n",
    "            current_B=branchB_csv\n",
    "            df_b = pd.read_csv(f\"{PATH_TO_DATAFRAMES}{current_B}\")\n",
    "            break\n",
    "        else:\n",
    "            current_B=None\n",
    "            df_b=pd.DataFrame(columns=['TF', 'Genes', 'Weights'])\n",
    "            \n",
    "    # continue with next window if both branches have no entries in this window  \n",
    "    if current_A is None and current_B is None:\n",
    "        continue\n",
    "          \n",
    "    \n",
    "    # get the tfs for the current window\n",
    "    tfs = set(df_a['TF'])\n",
    "    tfs = tfs.union(set(df_b['TF']))\n",
    "    \n",
    "    # get the genes for the current window\n",
    "    genes = set()\n",
    "    \n",
    "\n",
    "    # make sure the genes are saved as a list\n",
    "    for i in range(len(df_a['TF'])):\n",
    "        curr_genes = df_a['Genes'][i].split(\"'\")\n",
    "        curr_set = set(curr_genes)\n",
    "        genes = genes.union(curr_set)\n",
    "    for i in range(len(df_b['TF'])):\n",
    "        curr_genes = df_b['Genes'][i].split(\"'\")\n",
    "        curr_set = set(curr_genes)\n",
    "        genes = genes.union(curr_set)\n",
    "    # remove nonsense from list\n",
    "    genes.remove('[')\n",
    "    genes.remove(']')\n",
    "    genes.remove(', ')\n",
    "    \n",
    "    \n",
    "    # define tfs, genes and weights for current a and b\n",
    "    a_tfs = list(df_a['TF'])\n",
    "    a_genes = list(df_a['Genes'])\n",
    "    a_weights = list(df_a['Weights'])\n",
    "    b_tfs = list(df_b['TF'])\n",
    "    b_genes = list(df_b['Genes'])\n",
    "    b_weights = list(df_b['Weights'])\n",
    "    \n",
    "    \n",
    "    # insert a null value for the first column\n",
    "    genes = sorted(genes) # alphabtical order\n",
    "    genes.insert(0,'NULL') # as column for TFs\n",
    "    \n",
    "    \n",
    "    # create adjacency matrix for a and b\n",
    "    a_rows = []\n",
    "    a_rows.append(genes)\n",
    "    b_rows = []\n",
    "    b_rows.append(genes)\n",
    "    for tf in tfs:\n",
    "        adjacencies = [tf]\n",
    "        \n",
    "        # for a\n",
    "        if tf in a_tfs:\n",
    "            tf_index = a_tfs.index(tf)\n",
    "\n",
    "            curr_genes = a_genes[tf_index].split(\"'\")\n",
    "            curr_genes.remove('[')\n",
    "            curr_genes.remove(']')\n",
    "            while ', ' in curr_genes:\n",
    "                curr_genes.remove(', ')\n",
    "\n",
    "            for gene in genes: \n",
    "                weight = 0.0\n",
    "                if gene in curr_genes:\n",
    "                    gene_index = curr_genes.index(gene)\n",
    "                    curr_weights = a_weights[tf_index].split(\",\")    \n",
    "                    weight = curr_weights[gene_index]\n",
    "                    # strip [ or ] from number if 1st or last weight\n",
    "                    if (gene_index == len(curr_genes)-1):\n",
    "                        weight = weight.split(\"]\")[0]\n",
    "                    elif gene_index == 0:\n",
    "                        weight = weight.split(\"[\")[1]\n",
    "                    weight = float(weight)\n",
    "\n",
    "                adjacencies.append(weight)\n",
    "        else:\n",
    "            for gene in genes:\n",
    "                adjacencies.append(0.0)\n",
    "        a_rows.append(adjacencies) \n",
    "        \n",
    "        # for b\n",
    "        adjacencies = [tf]\n",
    "        if tf in b_tfs:\n",
    "            tf_index = b_tfs.index(tf)\n",
    "\n",
    "            curr_genes = b_genes[tf_index].split(\"'\")\n",
    "            curr_genes.remove('[')\n",
    "            curr_genes.remove(']')\n",
    "            while ', ' in curr_genes:\n",
    "                curr_genes.remove(', ')\n",
    "\n",
    "            for gene in genes:\n",
    "                weight=0.0\n",
    "                if gene in curr_genes:\n",
    "                    gene_index = curr_genes.index(gene)\n",
    "                    curr_weights = b_weights[tf_index].split(\",\")               \n",
    "                    weight = curr_weights[gene_index]\n",
    "                    # strip [ or ] from number if 1st or last weight\n",
    "                    if (gene_index == len(curr_genes)-1):\n",
    "                        weight = weight.split(\"]\")[0]\n",
    "                    elif gene_index == 0:\n",
    "                        weight = weight.split(\"[\")[1]\n",
    "                    weight = float(weight)\n",
    "                    \n",
    "                adjacencies.append(weight)\n",
    "        else:\n",
    "            for gene in genes:\n",
    "                adjacencies.append(0.0)\n",
    "        b_rows.append(adjacencies)    \n",
    "        \n",
    "        \n",
    "    # build the matrix with the absolut values for each window\n",
    "    abs_rows = []\n",
    "    \n",
    "    win_a_rows = []\n",
    "    win_b_rows = []\n",
    "    #abs_rows.append(genes)\n",
    "\n",
    "    for i in range(1,len(a_rows)):\n",
    "        for j in range(1, len(genes)):\n",
    "            row = []\n",
    "            row.append(a_rows[i][0])\n",
    "            row.append(genes[j])\n",
    "            value = abs(a_rows[i][j+1]-b_rows[i][j+1])\n",
    "            row.append(value)\n",
    "            \n",
    "            # here is the difference! Only entries where the abs-value is not 0 are saved\n",
    "            if (value != 0.0):\n",
    "                abs_rows.append(row)\n",
    "            \n",
    "            win_a_row = []\n",
    "            win_b_row = []\n",
    "            a_weight = a_rows[i][j+1]\n",
    "            b_weight = b_rows[i][j+1]\n",
    "            if a_weight != 0.0:\n",
    "                win_a_row =[a_rows[i][0], genes[j], a_weight]\n",
    "                win_a_rows.append(win_a_row)\n",
    "            if b_weight != 0.0:\n",
    "                win_b_row =[b_rows[i][0], genes[j], b_weight]\n",
    "                win_b_rows.append(win_b_row)\n",
    "    \n",
    "    # save the results\n",
    "    win_dataframe = pd.DataFrame(abs_rows)\n",
    "    win_dataframe.to_csv(f\"{PATH_TO_ABS_MATRICES_WO0}win{k}_abs.csv\", header = ['TF', 'GENE', 'abs'], index=False)\n",
    "    \n",
    "    # save the weights in a different file for better readability (TF, GENE, weight) and not as lists of genes and weights\n",
    "    a_win_df = pd.DataFrame(win_a_rows)\n",
    "    b_win_df = pd.DataFrame(win_b_rows)\n",
    "    a_win_df.to_csv(f\"{PATH_TO_WINDOW_WEIGHTS}A_win{k}_weights.csv\", header = ['TF', 'GENE', 'weight'], index=False)\n",
    "    b_win_df.to_csv(f\"{PATH_TO_WINDOW_WEIGHTS}B_win{k}_weights.csv\", header = ['TF', 'GENE', 'weight'], index=False)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### done: \n",
    "Matrices with absolute vaules for each window, representing the difference between the two branches. The larger the abs value the larger the difference in expression between branch A and B."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}