OpSeF_IV_Run_001.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Requirements:\n",
    "\n",
    "Tested with opsef003.yml (see attached file)\n",
    "opsef002 + n2v = opsef003\n",
    "\n",
    "on a GeForce RTX 2080 with 8GB RAM\n",
    "on ubuntu/18.04.3 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### adaped from:\n",
    "\n",
    "https://github.com/MouseLand/cellpose\n",
    "\n",
    "https://github.com/CellProfiler/CellProfiler\n",
    "\n",
    "https://github.com/mpicbg-csbd/stardist\n",
    "\n",
    "https://github.com/scikit-image/scikit-image\n",
    "\n",
    "https://github.com/VolkerH/unet-nuclei/\n",
    "\n",
    "Thanks to:\n",
    "\n",
    "All developer of the above mentioned repositories. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# basic libs\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import datetime\n",
    "import inspect\n",
    "from glob import glob\n",
    "\n",
    "import tifffile as tif\n",
    "\n",
    "import cv2 as cv\n",
    "import pandas as pd\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import collections\n",
    "import math\n",
    "\n",
    "import pickle\n",
    "\n",
    "import networkx as nx\n",
    "%matplotlib inline\n",
    "\n",
    "# for lif\n",
    "import readlif\n",
    "from readlif.reader import LifFile\n",
    "\n",
    "# skimage\n",
    "import skimage\n",
    "from skimage import transform, io, filters, measure, morphology,img_as_float  \n",
    "from skimage.color import label2rgb,gray2rgb\n",
    "from skimage.filters import gaussian, rank, threshold_otsu\n",
    "from skimage.io import imread, imsave\n",
    "from skimage.measure import label, regionprops, regionprops_table\n",
    "from skimage.morphology import disk, watershed\n",
    "\n",
    "# scipy\n",
    "from scipy.signal import medfilt\n",
    "from scipy.ndimage import generate_binary_structure, binary_dilation\n",
    "\n",
    "# for cellpose\n",
    "from cellpose import models as cp_models\n",
    "from cellpose import utils as cp_utils\n",
    "from cellpose import plot, transforms\n",
    "from cellpose import plot, transforms\n",
    "\n",
    "# other\n",
    "import mxnet as mx\n",
    "\n",
    "# for cluster analysis\n",
    "\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.cluster import AgglomerativeClustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/trasse/anaconda3/envs/opsef003/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:523: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/trasse/anaconda3/envs/opsef003/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:524: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/trasse/anaconda3/envs/opsef003/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/trasse/anaconda3/envs/opsef003/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/trasse/anaconda3/envs/opsef003/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/trasse/anaconda3/envs/opsef003/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:532: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "main_folder = os.path.dirname(os.path.abspath(inspect.stack()[0][1]))\n",
    "import_path = os.path.join(main_folder,\"Utils_and_Configs\")\n",
    "if import_path not in sys.path:\n",
    "    sys.path.append(import_path)\n",
    "\n",
    "# import from import_path\n",
    "from Tools_002 import *\n",
    "from UNet_CP01 import *\n",
    "from Segmentation_Func_06 import *\n",
    "from Pre_Post_Process002 import *\n",
    "from N2V_DataGeneratorTR001 import *\n",
    "from opsef_core_002 import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from https://github.com/mpicbg-csbd/stardist / 3_prediction (2D)\n",
    "from __future__ import print_function, unicode_literals, absolute_import, division\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "from csbdeep.utils import Path, normalize\n",
    "from stardist import random_label_cmap, _draw_polygons\n",
    "from stardist.models import StarDist2D\n",
    "\n",
    "# other\n",
    "import pkg_resources\n",
    "import keras\n",
    "\n",
    "# We import all our dependencies.\n",
    "from n2v.models import N2VConfig, N2V\n",
    "from n2v.utils.n2v_utils import manipulate_val_data\n",
    "# from n2v.internals.N2V_DataGenerator2 import N2V_DataGenerator2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load parameter\n",
    "the parameter for processing need to be defined in the notebook.\n",
    "Opsef_Setup_000X\n",
    "this notebook will print in the end a file_path.\n",
    "Please cut and paste it below!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading processing pipeline from ./Demo_Notebooks/my_runs/Parameter_muscle_mask_twoCH_Run_000.pkl\n"
     ]
    }
   ],
   "source": [
    "file_path = \"./Demo_Notebooks/my_runs/Parameter_muscle_mask_twoCH_Run_000.pkl\"\n",
    "\n",
    "infile = open(file_path,'rb')\n",
    "parameter = pickle.load(infile)\n",
    "print(\"Loading processing pipeline from\",file_path)\n",
    "infile.close()\n",
    "pc,input_def,run_def,initModelSettings = parameter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "def rewrite_fiji_tiff(input_d):\n",
    "    search_path = os.path.join(input_d[\"root\"],\"tiff_swap_ch\")\n",
    "    tiff_to_split = glob(\"{}/*tif\".format(search_path))\n",
    "    for file in tiff_to_split:\n",
    "        print(file)\n",
    "        fn = os.path.split(file)[1]\n",
    "        img = tif.imread(file)\n",
    "        print(img.shape)\n",
    "        img_new = np.swapaxes(img.copy(),0,2)\n",
    "        print(img_new.shape)\n",
    "        tif.imsave(os.path.join(input_d[\"root\"],\"tiff\",fn),img_new)\n",
    "    return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "input_def[\"rearrange_ch_from_fiji\"] = True\n",
    "if input_def[\"rearrange_ch_from_fiji\"]:\n",
    "        rewrite_fiji_tiff(input_def)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating folder.../home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/00_InputRaw\n",
      "Creating folder.../home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/01_Input\n",
      "Creating folder.../home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/02_SegMasks\n",
      "Creating folder.../home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/03_SegOverlays\n",
      "Creating folder.../home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/04_BasicQuantification\n",
      "Creating folder.../home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/05_AdditionalChannel\n",
      "Creating folder.../home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/06_AdditionalMask\n",
      "Creating folder.../home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/07_ClassifiedSegMasks\n",
      "Creating folder.../home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/08_ClassifiedSegOverlays\n",
      "Creating folder.../home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/09_ClassifiedQuantification\n",
      "Creating folder.../home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/10_ForFiji\n",
      "Creating folder.../home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/11_FromFiji\n",
      "['/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0009.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0004.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0012.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0005.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0008.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0002.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0006.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0010.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0007.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0003.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0011.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/Train_muscle_RGB0001.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/Train_muscle_RGB0000.tif']\n",
      "Analysing ['/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/Train_muscle_RGB0001.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/Train_muscle_RGB0000.tif']\n",
      "Saving.. /home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/00_InputRaw/Sum_Train_muscle_RGB0001.tif\n",
      "Saving.. /home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/00_InputRaw/8bitSum_Train_muscle_RGB0001.tif\n",
      "Saving.. /home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/00_InputRaw/Sum_Train_muscle_RGB0000.tif\n",
      "Saving.. /home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/00_InputRaw/8bitSum_Train_muscle_RGB0000.tif\n",
      "Saving.. /home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/00_InputRaw/CleanSum_000_Train_muscle_RGB0001.tif\n",
      "Saving.. /home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/01_Input/Input_000_Train_muscle_RGB0001.tif\n",
      "Saving.. /home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/00_InputRaw/CleanSum_000_Train_muscle_RGB0000.tif\n",
      "Saving.. /home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/01_Input/Input_000_Train_muscle_RGB0000.tif\n",
      "run_now Cellpose\n",
      "Run Cellpose Model:  CP_nuclei\n",
      "GPU found\n",
      "Model_keys dict_keys(['CP_nuclei'])\n",
      "Running now:  ['RGBInput_000_Train_muscle_RGB0001.tif', 'RGBInput_000_Train_muscle_RGB0000.tif']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:33<00:00, 16.93s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving.. /home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/02_SegMasks/000_CP_Mask_0.6_RGBInput_000_Train_muscle_RGB0001.tif\n",
      "Saving.. /home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/03_SegOverlays/000_CP_MaskOutline_0.6_RGBInput_000_Train_muscle_RGB0001.tif\n",
      "(2048, 2048, 3)\n",
      "Saving.. /home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/02_SegMasks/000_CP_Mask_0.6_RGBInput_000_Train_muscle_RGB0000.tif\n",
      "Saving.. /home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/03_SegOverlays/000_CP_MaskOutline_0.6_RGBInput_000_Train_muscle_RGB0000.tif\n",
      "(2048, 2048, 3)\n",
      "2\n",
      "The segmentatio took overall: 0:00:52.320136\n"
     ]
    }
   ],
   "source": [
    "# process for all\n",
    "    # create subfolder\n",
    "make_folder_structure(pc,input_def,run_def)\n",
    "\n",
    "# process for lif\n",
    "if input_def[\"input_type\"] == \".lif\":\n",
    "    lifobject,input_def = define_lif_pipeline(input_def)\n",
    "    preprocess_1_for_lif(lifobject,input_def,pc,run_def)\n",
    "    preprocess_2_for_lif(lifobject,input_def,pc,run_def)\n",
    "# process for tif\n",
    "if input_def[\"input_type\"] == \".tif\":\n",
    "    fpath_list = define_tif_pipeline(input_def)\n",
    "    preprocess_1_for_tif(fpath_list,input_def,pc,run_def)\n",
    "    preprocess_2_for_tif(fpath_list,input_def,pc,run_def)\n",
    "\n",
    "# Segment\n",
    "start_time = datetime.datetime.now()\n",
    "segment(input_def,pc,run_def,initModelSettings)\n",
    "end_time = datetime.datetime.now()\n",
    "time_delta = end_time - start_time\n",
    "print(\"The segmentatio took overall:\", time_delta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export annditional channel & Quantify Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "if pc[\"Export_to_CSV\"]:\n",
    "    all_combined = [] # used for quantifications of more than one intensity channel\n",
    "    # get a list of the masks that were produced by segmentation\n",
    "    mask_files = glob(os.path.join(input_def[\"root\"],\"Processed_{}\".format(run_def[\"run_ID\"]),pc[\"sub_f\"][2])+\"/*.tif\")\n",
    "    mask_to_img_dic, mask_to_8bitimg_dic = make_mask_to_img_dic(mask_files,pc,input_def,run_def,0,pc[\"Intensity_Ch\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "if pc[\"toFiji\"]:\n",
    "    if not pc[\"Export_to_CSV\"]:\n",
    "        mask_files = glob(os.path.join(input_def[\"root\"],\"Processed_{}\".format(run_def[\"run_ID\"]),pc[\"sub_f\"][2])+\"/*.tif\")\n",
    "        mask_to_img_dic, mask_to_8bitimg_dic = make_mask_to_img_dic(mask_files,pc,input_def,run_def,0,pc[\"Intensity_Ch\"])\n",
    "    root_plus = os.path.join(input_def[\"root\"],\"Processed_{}\".format(run_def[\"run_ID\"]))\n",
    "    txt_fn = os.path.join(root_plus,pc[\"sub_f\"][10],\"FilePairList_{}_{}.txt\".format(input_def[\"dataset\"],run_def[\"run_ID\"]))\n",
    "    with open(txt_fn,\"w\") as f:\n",
    "        for mask_fn,image_fn in mask_to_8bitimg_dic.items():\n",
    "            f.write(\"{};{}{}\".format(image_fn.replace(root_plus,\"\"),mask_fn.replace(root_plus,\"\"),\"\\n\"))\n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Export from tiff\n",
      "['/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0009.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0004.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0012.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0005.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0008.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0002.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0006.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0010.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0007.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0003.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/muscle_RGB0011.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/Train_muscle_RGB0001.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/Train_muscle_RGB0000.tif']\n",
      "Analysing ['/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/Train_muscle_RGB0001.tif', '/home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/tiff/Train_muscle_RGB0000.tif']\n",
      "Saving.. /home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/05_AdditionalChannel/Sum_Train_muscle_RGB0001.tif\n",
      "Saving.. /home/trasse/Desktop/MLTestData/SlideScanner_muscle_twoCh/Processed_000/05_AdditionalChannel/Sum_Train_muscle_RGB0000.tif\n"
     ]
    }
   ],
   "source": [
    "# export additional channel\n",
    "if pc[\"export_another_channel\"]:\n",
    "    if input_def[\"input_type\"] == \".lif\":\n",
    "        exported_file_list = export_second_channel_for_mask(lifobject,pc,input_def,run_def)\n",
    "    if input_def[\"input_type\"] == \".tif\":\n",
    "        exported_file_list = export_second_channel_for_mask(\"NoneIsTiFF\",pc,input_def,run_def)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-0f87afd8a996>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mnew_mask_fn_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_mask_from_add_ch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexported_file_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0minput_def\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"root\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"sub_f\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrun_def\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"run_ID\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrun_def\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"para_mp\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0;31m# make a dic that has the segmentation output mask name as key, the name of the threshold mask as value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0mpair_dic\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmake_pair_second_mask_simple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmask_files\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnew_mask_fn_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m     \u001b[0;31m# create new seqmentation masks per class and return a list of file_names\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mclass1_to_img_dic\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mclass2_to_img_dic\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msplit_by_mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_def\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"root\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrun_def\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"run_ID\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"sub_f\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpair_dic\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmask_to_8bitimg_dic\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmask_to_img_dic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/github/OpSeF-IV/Utils_and_Configs/Segmentation_Func_06.py\u001b[0m in \u001b[0;36mmake_pair_second_mask_simple\u001b[0;34m(masks_input_files, new_mask_fn_list)\u001b[0m\n\u001b[1;32m    432\u001b[0m         \u001b[0msplit_fn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmask_fn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"_\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    433\u001b[0m         \u001b[0mroot_fn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"_\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msplit_fn\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 434\u001b[0;31m         \u001b[0mmatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnew_mask_fn_list\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mroot_fn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    435\u001b[0m         \u001b[0mmask_to_img\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmask_fn\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmatch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    436\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mmask_to_img\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "# optional in case segmentation results shall be filtered by a mask:\n",
    "if pc[\"create_filter_mask_from_channel\"]:\n",
    "    # create new masks (by thresolding the additional input) and extract their names\n",
    "    new_mask_fn_list = create_mask_from_add_ch(exported_file_list,input_def[\"root\"],pc[\"sub_f\"],run_def[\"run_ID\"],run_def[\"para_mp\"])\n",
    "    # make a dic that has the segmentation output mask name as key, the name of the threshold mask as value\n",
    "    pair_dic = make_pair_second_mask_simple(mask_files,new_mask_fn_list)\n",
    "    # create new seqmentation masks per class and return a list of file_names\n",
    "    class1_to_img_dic,class2_to_img_dic = split_by_mask(input_def[\"root\"],run_def[\"run_ID\"],pc[\"sub_f\"],pair_dic,mask_to_8bitimg_dic,mask_to_img_dic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# quantify original mask\n",
    "if pc[\"Export_to_CSV\"]:\n",
    "    all_combined.append(results_to_csv(mask_to_img_dic,pc[\"get_property\"],input_def[\"root\"],pc[\"sub_f\"],run_def[\"run_ID\"],4,\"All_Main\",input_def[\"subset\"])) # 4 is the main result folder\n",
    "    if pc[\"plot_head_main\"]:\n",
    "        all_combined[0].head()\n",
    "            \n",
    "if pc[\"create_filter_mask_from_channel\"]:\n",
    "    # quantify class1 masks\n",
    "    results_to_csv(class1_to_img_dic,pc[\"get_property\"],input_def[\"root\"],pc[\"sub_f\"],run_def[\"run_ID\"],9,\"Class00\",input_def[\"post_subset\"]) # 9 is the classified result folder\n",
    "    # quantify class2 masks\n",
    "    results_to_csv(class2_to_img_dic,pc[\"get_property\"],input_def[\"root\"],pc[\"sub_f\"],run_def[\"run_ID\"],9,\"Class01\",input_def[\"post_subset\"]) # 9 is the classified result folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if pc[\"Quantify_2ndCh\"]:\n",
    "    mask_to_img_dic, mask_to_8bitimg_dic = make_mask_to_img_dic(mask_files,pc,input_def,run_def,5,pc[\"Intensity_2ndCh\"])\n",
    "    all_combined.append(results_to_csv(mask_to_img_dic,pc[\"get_property\"],input_def[\"root\"],pc[\"sub_f\"],run_def[\"run_ID\"],4,\"All_2nd\",input_def[\"subset\"]))\n",
    "    if pc[\"merge_results\"]:\n",
    "        result_summary = merge_intensity_results(all_combined,input_def,pc[\"sub_f\"],run_def,4)\n",
    "        if pc[\"plot_merged\"]:\n",
    "            result_summary.head()\n",
    "else:\n",
    "    if pc[\"Export_to_CSV\"]:\n",
    "        result_summary = all_combined[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AddOn 1: Basic plotting of results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if pc[\"Plot_Results\"]:\n",
    "    fig, axs = plt.subplots(len(pc[\"Plot_xy\"]), 1, figsize=(5, 5*len(pc[\"Plot_xy\"])), constrained_layout=True)\n",
    "    for i in range(0,len(pc[\"Plot_xy\"])):\n",
    "        axs[i].scatter(result_summary[pc[\"Plot_xy\"][i][0]],result_summary[pc[\"Plot_xy\"][i][1]], c=\"red\")\n",
    "        axs[i].set_title('{} vs {}'.format(*pc[\"Plot_xy\"][i]))\n",
    "        axs[i].set_xlabel(pc[\"Plot_xy\"][i][0],fontsize=15)\n",
    "        axs[i].set_ylabel(pc[\"Plot_xy\"][i][1],fontsize=15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AddOn 2: Do PCA and TSNE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example pipeline auto-clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if pc[\"Cluster_How\"] == \"Auto\":\n",
    "# get data for PCA / TSNE\n",
    "    df_for_tsne_list = extract_values_for_TSNE_PCA(input_def[\"root\"],run_def[\"run_ID\"],pc[\"sub_f\"],4,pc[\"include_in_tsne\"])\n",
    "# get cluster\n",
    "    data = df_for_tsne_list[0].values\n",
    "    auto_clustering = AgglomerativeClustering(linkage=pc[\"link_method\"], n_clusters=pc[\"cluster_expected\"]).fit(data)\n",
    "# do analysis\n",
    "    result_tsne = TSNE(learning_rate=pc[\"tSNE_learning_rate\"]).fit_transform(data)\n",
    "    result_pca = PCA().fit_transform(data)\n",
    "# display results\n",
    "    fig, axs = plt.subplots(2, 1, figsize=(10, 20), constrained_layout=True)\n",
    "    axs[0].scatter(result_tsne[:, 0], result_tsne[:, 1], c=auto_clustering.labels_)\n",
    "    axs[0].set_title('tSNE')\n",
    "    axs[1].scatter(result_pca[:, 0], result_pca[:, 1], c=auto_clustering.labels_)\n",
    "    axs[1].set_title('PCA')    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example pipeline mask-clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get data for PCA / TSNE\n",
    "if pc[\"Cluster_How\"] == \"Mask\":\n",
    "    df_for_tsne_list_by_class = extract_values_for_TSNE_PCA(input_def[\"root\"],run_def[\"run_ID\"],pc[\"sub_f\"],9,pc[\"include_in_tsne\"])\n",
    "    fused_df = pd.concat(df_for_tsne_list_by_class,axis = 0,join=\"outer\")\n",
    "    data_by_class = fused_df.values\n",
    "    class_def_by_mask = [0 for x in range (0,df_for_tsne_list_by_class[0].shape[0])] + [1 for x in range (0,df_for_tsne_list_by_class[1].shape[0])]\n",
    "# do analysis\n",
    "    result_tsne_by_class = TSNE(learning_rate=pc[\"tSNE_learning_rate\"]).fit_transform(data_by_class)\n",
    "    result_pca_by_class = PCA().fit_transform(data_by_class)\n",
    "# display results\n",
    "    fig, axs = plt.subplots(2, 1, figsize=(10, 20), constrained_layout=True)\n",
    "    axs[0].scatter(result_tsne_by_class[:, 0], result_tsne_by_class[:, 1], c=class_def_by_mask)\n",
    "    axs[0].set_title('tSNE')\n",
    "    axs[1].scatter(result_pca_by_class[:, 0], result_pca_by_class[:, 1], c=class_def_by_mask)\n",
    "    axs[1].set_title('PCA')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Processing completed sucessfully !\\n\")\n",
    "print(\"All results have been saved in this folder: \\n\")\n",
    "print(os.path.join(input_def[\"root\"],\"Processed_{}\".format(run_def[\"run_ID\"])))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}