notebooks/03-Local_processing_cluster_activation_test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8f690a00-e4a8-4139-b3d4-ac6f1737709f",
   "metadata": {},
   "source": [
    "# Local processing cluster activation test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8760dd4d-5be3-45d2-a77e-e112ded35087",
   "metadata": {},
   "source": [
    "The aim of this notebook is to determine if you can initialize the Pipeline and activate the processing cluster.  \n",
    "- You need a processing_folder containing the required folder and files (see README.md)\n",
    "- The mock_experiment_to_test_local_setup has example files that are required for the initialization of the pipeline."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65379268-53db-4fa8-856e-9d12c7ffe332",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8c73cc10-26cf-48b1-bd69-6b7028155316",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "25f34ecd-a8bd-456f-9805-46713c96bbc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "28167f59-9268-466c-ba0e-14c9fc435a2d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/simone.codeluppi/opt/miniconda3/envs/test_d/lib/python3.8/site-packages/dask_jobqueue/core.py:19: FutureWarning: format_bytes is deprecated and will be removed in a future release. Please use dask.utils.format_bytes instead.\n",
      "  from distributed.utils import format_bytes, parse_bytes, tmpfile\n",
      "/Users/simone.codeluppi/opt/miniconda3/envs/test_d/lib/python3.8/site-packages/dask_jobqueue/core.py:19: FutureWarning: parse_bytes is deprecated and will be removed in a future release. Please use dask.utils.parse_bytes instead.\n",
      "  from distributed.utils import format_bytes, parse_bytes, tmpfile\n",
      "/Users/simone.codeluppi/opt/miniconda3/envs/test_d/lib/python3.8/site-packages/dask_jobqueue/htcondor.py:6: FutureWarning: parse_bytes is deprecated and will be removed in a future release. Please use dask.utils.parse_bytes instead.\n",
      "  from distributed.utils import parse_bytes\n"
     ]
    }
   ],
   "source": [
    "from pysmFISH.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0a3e5cd-9da1-4bd3-86fe-3a1b9551a00d",
   "metadata": {},
   "source": [
    "## Initialize the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a74482c0-fbac-4fd0-8df1-02b9e17287df",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Enter the required parameters\n",
    "experiment_fpath = Path('/fish/work_std/LBEXP20210718_EEL_Mouse_448_2')\n",
    "date_tag = time.strftime(\"%y%m%d_%H_%M_%S\")\n",
    "pipeline_run_name = date_tag + '_' + experiment_fpath.stem\n",
    "run_type = 're-run'\n",
    "parsing_type = 'no_parsing'\n",
    "processing_engine = 'local'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6c509b06-2771-4ca9-b524-5f44087acd63",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 79 µs, sys: 2 µs, total: 81 µs\n",
      "Wall time: 82.7 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Because you are running the pipeline locally you should define the number of cores and memory/core\n",
    "running_pipeline = Pipeline(\n",
    "        pipeline_run_name= pipeline_run_name,\n",
    "        experiment_fpath= experiment_fpath,\n",
    "        run_type= run_type,\n",
    "        parsing_type= parsing_type,\n",
    "        processing_engine= processing_engine,\n",
    "        cores=2,\n",
    "        memory='5GB')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7206ec8f-0028-421e-b5d0-228ec265a6f5",
   "metadata": {},
   "source": [
    "## Start the cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f0908e91-15f5-4af4-b519-e04ca0d5b2e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 266 ms, sys: 108 ms, total: 374 ms\n",
      "Wall time: 1.97 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "running_pipeline.processing_cluster_init_step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "890d2b73-3c64-4465-b156-4818c174fa78",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Visualize dashboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "372e6120-b904-4aa5-a28a-b80814d851b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "    <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\"> </div>\n",
       "    <div style=\"margin-left: 48px;\">\n",
       "        <h3 style=\"margin-bottom: 0px;\">Client</h3>\n",
       "        <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Client-59912aec-52ae-11ec-bdf0-82302a80a805</p>\n",
       "        <table style=\"width: 100%; text-align: left;\">\n",
       "\n",
       "        <tr>\n",
       "        \n",
       "            <td style=\"text-align: left;\"><strong>Connection method:</strong> Cluster object</td>\n",
       "            <td style=\"text-align: left;\"><strong>Cluster type:</strong> distributed.LocalCluster</td>\n",
       "        \n",
       "        </tr>\n",
       "\n",
       "        \n",
       "            <tr>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:8787/status\" target=\"_blank\">http://127.0.0.1:8787/status</a>\n",
       "                </td>\n",
       "                <td style=\"text-align: left;\"></td>\n",
       "            </tr>\n",
       "        \n",
       "\n",
       "        </table>\n",
       "\n",
       "        \n",
       "            <details>\n",
       "            <summary style=\"margin-bottom: 20px;\"><h3 style=\"display: inline;\">Cluster Info</h3></summary>\n",
       "            <div class=\"jp-RenderedHTMLCommon jp-RenderedHTML jp-mod-trusted jp-OutputArea-output\">\n",
       "    <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\">\n",
       "    </div>\n",
       "    <div style=\"margin-left: 48px;\">\n",
       "        <h3 style=\"margin-bottom: 0px; margin-top: 0px;\">LocalCluster</h3>\n",
       "        <p style=\"color: #9D9D9D; margin-bottom: 0px;\">b6be5f42</p>\n",
       "        <table style=\"width: 100%; text-align: left;\">\n",
       "            <tr>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Dashboard:</strong> <a href=\"http://127.0.0.1:8787/status\" target=\"_blank\">http://127.0.0.1:8787/status</a>\n",
       "                </td>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Workers:</strong> 2\n",
       "                </td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Total threads:</strong> 2\n",
       "                </td>\n",
       "                <td style=\"text-align: left;\">\n",
       "                    <strong>Total memory:</strong> 9.31 GiB\n",
       "                </td>\n",
       "            </tr>\n",
       "            \n",
       "            <tr>\n",
       "    <td style=\"text-align: left;\"><strong>Status:</strong> running</td>\n",
       "    <td style=\"text-align: left;\"><strong>Using processes:</strong> True</td>\n",
       "</tr>\n",
       "\n",
       "            \n",
       "        </table>\n",
       "\n",
       "        <details>\n",
       "            <summary style=\"margin-bottom: 20px;\">\n",
       "                <h3 style=\"display: inline;\">Scheduler Info</h3>\n",
       "            </summary>\n",
       "\n",
       "            <div style=\"\">\n",
       "    <div>\n",
       "        <div style=\"width: 24px; height: 24px; background-color: #FFF7E5; border: 3px solid #FF6132; border-radius: 5px; position: absolute;\"> </div>\n",
       "        <div style=\"margin-left: 48px;\">\n",
       "            <h3 style=\"margin-bottom: 0px;\">Scheduler</h3>\n",
       "            <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Scheduler-4e8cf62a-075b-4be2-b3a5-a2842eef4cd3</p>\n",
       "            <table style=\"width: 100%; text-align: left;\">\n",
       "                <tr>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Comm:</strong> tcp://127.0.0.1:58034\n",
       "                    </td>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Workers:</strong> 2\n",
       "                    </td>\n",
       "                </tr>\n",
       "                <tr>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Dashboard:</strong> <a href=\"http://127.0.0.1:8787/status\" target=\"_blank\">http://127.0.0.1:8787/status</a>\n",
       "                    </td>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Total threads:</strong> 2\n",
       "                    </td>\n",
       "                </tr>\n",
       "                <tr>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Started:</strong> Just now\n",
       "                    </td>\n",
       "                    <td style=\"text-align: left;\">\n",
       "                        <strong>Total memory:</strong> 9.31 GiB\n",
       "                    </td>\n",
       "                </tr>\n",
       "            </table>\n",
       "        </div>\n",
       "    </div>\n",
       "\n",
       "    <details style=\"margin-left: 48px;\">\n",
       "        <summary style=\"margin-bottom: 20px;\">\n",
       "            <h3 style=\"display: inline;\">Workers</h3>\n",
       "        </summary>\n",
       "\n",
       "        \n",
       "        <div style=\"margin-bottom: 20px;\">\n",
       "            <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
       "            <div style=\"margin-left: 48px;\">\n",
       "            <details>\n",
       "                <summary>\n",
       "                    <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 0</h4>\n",
       "                </summary>\n",
       "                <table style=\"width: 100%; text-align: left;\">\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Comm: </strong> tcp://127.0.0.1:58042\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Total threads: </strong> 1\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:58045/status\" target=\"_blank\">http://127.0.0.1:58045/status</a>\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Memory: </strong> 4.66 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Nanny: </strong> tcp://127.0.0.1:58037\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\"></td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td colspan=\"2\" style=\"text-align: left;\">\n",
       "                            <strong>Local directory: </strong> /Users/simone.codeluppi/Dropbox (Linnarsson Group)/My Mac (KI-C02C77ZEMD6R)/Documents/data_analysis_jlabs_sc/notebooks_pysmFISH/dask-worker-space/worker-fwsxznwb\n",
       "                        </td>\n",
       "                    </tr>\n",
       "\n",
       "                    \n",
       "\n",
       "                    \n",
       "\n",
       "                </table>\n",
       "            </details>\n",
       "            </div>\n",
       "        </div>\n",
       "        \n",
       "        <div style=\"margin-bottom: 20px;\">\n",
       "            <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
       "            <div style=\"margin-left: 48px;\">\n",
       "            <details>\n",
       "                <summary>\n",
       "                    <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 1</h4>\n",
       "                </summary>\n",
       "                <table style=\"width: 100%; text-align: left;\">\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Comm: </strong> tcp://127.0.0.1:58043\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Total threads: </strong> 1\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:58044/status\" target=\"_blank\">http://127.0.0.1:58044/status</a>\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Memory: </strong> 4.66 GiB\n",
       "                        </td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td style=\"text-align: left;\">\n",
       "                            <strong>Nanny: </strong> tcp://127.0.0.1:58036\n",
       "                        </td>\n",
       "                        <td style=\"text-align: left;\"></td>\n",
       "                    </tr>\n",
       "                    <tr>\n",
       "                        <td colspan=\"2\" style=\"text-align: left;\">\n",
       "                            <strong>Local directory: </strong> /Users/simone.codeluppi/Dropbox (Linnarsson Group)/My Mac (KI-C02C77ZEMD6R)/Documents/data_analysis_jlabs_sc/notebooks_pysmFISH/dask-worker-space/worker-src0z_fq\n",
       "                        </td>\n",
       "                    </tr>\n",
       "\n",
       "                    \n",
       "\n",
       "                    \n",
       "\n",
       "                </table>\n",
       "            </details>\n",
       "            </div>\n",
       "        </div>\n",
       "        \n",
       "\n",
       "    </details>\n",
       "</div>\n",
       "\n",
       "        </details>\n",
       "    </div>\n",
       "</div>\n",
       "            </details>\n",
       "        \n",
       "\n",
       "    </div>\n",
       "</div>"
      ],
      "text/plain": [
       "<Client: 'tcp://127.0.0.1:58034' processes=2 threads=2, memory=9.31 GiB>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "running_pipeline.client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6bc91f02-e270-4612-ac96-96d99362ebc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "running_pipeline.client.close()\n",
    "running_pipeline.cluster.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1480ec06-f702-4d48-b57e-6227c7568ca1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "test_d",
   "language": "python",
   "name": "test_d"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}