From e5a5d634ba90ef2a9be157d02642c27ee7d4fc6b Mon Sep 17 00:00:00 2001 From: Milton Pividori Date: Thu, 7 Sep 2023 23:01:22 -0600 Subject: [PATCH] ccc profiling: add notebook to test many genes with multiple CPUs --- .../16_00-2_CPUs-default-many_genes.ipynb | 590 ++++++++++++++++ ...6_01-2_CPUs-disable_numba-many_genes.ipynb | 627 ++++++++++++++++++ ...6_02-2_CPUs-use_ari_numba-many_genes.ipynb | 595 +++++++++++++++++ .../py/16_00-2_CPUs-default-many_genes.py | 89 +++ .../16_01-2_CPUs-disable_numba-many_genes.py | 95 +++ .../16_02-2_CPUs-use_ari_numba-many_genes.py | 89 +++ 6 files changed, 2085 insertions(+) create mode 100644 nbs/others/05_ccc_profiling/12_cm_optimized/16_00-2_CPUs-default-many_genes.ipynb create mode 100644 nbs/others/05_ccc_profiling/12_cm_optimized/16_01-2_CPUs-disable_numba-many_genes.ipynb create mode 100644 nbs/others/05_ccc_profiling/12_cm_optimized/16_02-2_CPUs-use_ari_numba-many_genes.ipynb create mode 100644 nbs/others/05_ccc_profiling/12_cm_optimized/py/16_00-2_CPUs-default-many_genes.py create mode 100644 nbs/others/05_ccc_profiling/12_cm_optimized/py/16_01-2_CPUs-disable_numba-many_genes.py create mode 100644 nbs/others/05_ccc_profiling/12_cm_optimized/py/16_02-2_CPUs-use_ari_numba-many_genes.py diff --git a/nbs/others/05_ccc_profiling/12_cm_optimized/16_00-2_CPUs-default-many_genes.ipynb b/nbs/others/05_ccc_profiling/12_cm_optimized/16_00-2_CPUs-default-many_genes.ipynb new file mode 100644 index 00000000..ad1d721e --- /dev/null +++ b/nbs/others/05_ccc_profiling/12_cm_optimized/16_00-2_CPUs-default-many_genes.ipynb @@ -0,0 +1,590 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "507e9319-381b-4934-987c-2958e7f6ace7", + "metadata": { + "papermill": { + "duration": 0.00595, + "end_time": "2023-09-07T21:45:43.464040", + "exception": false, + "start_time": "2023-09-07T21:45:43.458090", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Description" + ] + }, + { + "cell_type": "markdown", + "id": "9a88f2a7-3773-459c-8fdc-c69c1b0bb338", + "metadata": { + "papermill": { + "duration": 0.004405, + "end_time": "2023-09-07T21:45:43.478248", + "exception": false, + "start_time": "2023-09-07T21:45:43.473843", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "Like `06_00` but using more CPU cores." + ] + }, + { + "cell_type": "markdown", + "id": "72de9f6f-dc00-435e-9dba-871db859a78a", + "metadata": { + "papermill": { + "duration": 0.002879, + "end_time": "2023-09-07T21:45:43.485629", + "exception": false, + "start_time": "2023-09-07T21:45:43.482750", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Use multiple CPU core" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "44fd0e26-4162-44b8-b2a9-9f989e892719", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:45:43.491609Z", + "iopub.status.busy": "2023-09-07T21:45:43.491456Z", + "iopub.status.idle": "2023-09-07T21:45:43.499615Z", + "shell.execute_reply": "2023-09-07T21:45:43.499115Z" + }, + "papermill": { + "duration": 0.012108, + "end_time": "2023-09-07T21:45:43.501110", + "exception": false, + "start_time": "2023-09-07T21:45:43.489002", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env: CM_N_JOBS=2\n", + "env: NUMBA_NUM_THREADS=2\n", + "env: MKL_NUM_THREADS=2\n", + "env: OPEN_BLAS_NUM_THREADS=2\n", + "env: NUMEXPR_NUM_THREADS=2\n", + "env: OMP_NUM_THREADS=2\n" + ] + } + ], + "source": [ + "%env CM_N_JOBS=2\n", + "%env NUMBA_NUM_THREADS=2\n", + "%env MKL_NUM_THREADS=2\n", + "%env OPEN_BLAS_NUM_THREADS=2\n", + "%env NUMEXPR_NUM_THREADS=2\n", + "%env OMP_NUM_THREADS=2" + ] + }, + { + "cell_type": "markdown", + "id": "4d885a9d-482d-47e7-bfaf-11c8efae3079", + "metadata": { + "papermill": { + "duration": 0.001797, + "end_time": "2023-09-07T21:45:43.505171", + "exception": false, + "start_time": "2023-09-07T21:45:43.503374", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Remove pycache dir" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "88d32361-f1b5-4cf0-9a2c-7ab927d14b42", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:45:43.508253Z", + "iopub.status.busy": "2023-09-07T21:45:43.508181Z", + "iopub.status.idle": "2023-09-07T21:45:43.699643Z", + "shell.execute_reply": "2023-09-07T21:45:43.698365Z" + }, + "papermill": { + "duration": 0.195914, + "end_time": "2023-09-07T21:45:43.702627", + "exception": false, + "start_time": "2023-09-07T21:45:43.506713", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/opt/code\r\n" + ] + } + ], + "source": [ + "!echo ${CODE_DIR}" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "8a27da6e-cf59-4276-888b-57e98fd23ccf", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:45:43.720405Z", + "iopub.status.busy": "2023-09-07T21:45:43.720111Z", + "iopub.status.idle": "2023-09-07T21:45:43.927188Z", + "shell.execute_reply": "2023-09-07T21:45:43.926299Z" + }, + "papermill": { + "duration": 0.221712, + "end_time": "2023-09-07T21:45:43.930066", + "exception": false, + "start_time": "2023-09-07T21:45:43.708354", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/opt/code/libs/ccc/coef/__pycache__\r\n", + "/opt/code/libs/ccc/pytorch/__pycache__\r\n", + "/opt/code/libs/ccc/scipy/__pycache__\r\n", + "/opt/code/libs/ccc/utils/__pycache__\r\n", + "/opt/code/libs/ccc/__pycache__\r\n", + "/opt/code/libs/ccc/sklearn/__pycache__\r\n" + ] + } + ], + "source": [ + "!find ${CODE_DIR} -regex '^.*\\(__pycache__\\)$' -print" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "561480f7-c610-4f60-b40f-8100974ab4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:45:43.947586Z", + "iopub.status.busy": "2023-09-07T21:45:43.947272Z", + "iopub.status.idle": "2023-09-07T21:45:44.161416Z", + "shell.execute_reply": "2023-09-07T21:45:44.160188Z" + }, + "papermill": { + "duration": 0.228922, + "end_time": "2023-09-07T21:45:44.164466", + "exception": false, + "start_time": "2023-09-07T21:45:43.935544", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "find: ‘/opt/code/libs/ccc/coef/__pycache__’: No such file or directory\r\n", + "find: ‘/opt/code/libs/ccc/pytorch/__pycache__’: No such file or directory\r\n", + "find: ‘/opt/code/libs/ccc/scipy/__pycache__’: No such file or directory\r\n", + "find: ‘/opt/code/libs/ccc/utils/__pycache__’: No such file or directory\r\n", + "find: ‘/opt/code/libs/ccc/__pycache__’: No such file or directory\r\n", + "find: ‘/opt/code/libs/ccc/sklearn/__pycache__’: No such file or directory\r\n" + ] + } + ], + "source": [ + "!find ${CODE_DIR} -regex '^.*\\(__pycache__\\)$' -exec rm -rf {} \\;" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "a5c90a17-c0ac-49f0-968d-ea9ea8710b00", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:45:44.182151Z", + "iopub.status.busy": "2023-09-07T21:45:44.181828Z", + "iopub.status.idle": "2023-09-07T21:45:44.394488Z", + "shell.execute_reply": "2023-09-07T21:45:44.393489Z" + }, + "papermill": { + "duration": 0.22684, + "end_time": "2023-09-07T21:45:44.397116", + "exception": false, + "start_time": "2023-09-07T21:45:44.170276", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "!find ${CODE_DIR} -regex '^.*\\(__pycache__\\)$' -print" + ] + }, + { + "cell_type": "markdown", + "id": "145563a2-3e46-4f62-8191-7444a0b315bb", + "metadata": { + "papermill": { + "duration": 0.002713, + "end_time": "2023-09-07T21:45:44.403919", + "exception": false, + "start_time": "2023-09-07T21:45:44.401206", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Modules" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "bea3d48e-8823-403f-90f5-aea8a17b357c", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:45:44.408613Z", + "iopub.status.busy": "2023-09-07T21:45:44.408519Z", + "iopub.status.idle": "2023-09-07T21:45:48.093285Z", + "shell.execute_reply": "2023-09-07T21:45:48.093030Z" + }, + "papermill": { + "duration": 3.688091, + "end_time": "2023-09-07T21:45:48.094225", + "exception": false, + "start_time": "2023-09-07T21:45:44.406134", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "from ccc.coef import ccc" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "df6b3793-930b-4c54-9f29-ecc47fc586fb", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:45:48.105640Z", + "iopub.status.busy": "2023-09-07T21:45:48.105558Z", + "iopub.status.idle": "2023-09-07T21:45:48.109631Z", + "shell.execute_reply": "2023-09-07T21:45:48.109498Z" + }, + "papermill": { + "duration": 0.01467, + "end_time": "2023-09-07T21:45:48.110298", + "exception": false, + "start_time": "2023-09-07T21:45:48.095628", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# let numba compile all the code before profiling\n", + "ccc(np.random.rand(10), np.random.rand(10))" + ] + }, + { + "cell_type": "markdown", + "id": "f8399ca8-265e-4e96-b582-54045cb2f9eb", + "metadata": { + "papermill": { + "duration": 0.001104, + "end_time": "2023-09-07T21:45:48.112666", + "exception": false, + "start_time": "2023-09-07T21:45:48.111562", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Data" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "2316ffcd-a6e4-453f-bb52-779685c5c5bf", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:45:48.115641Z", + "iopub.status.busy": "2023-09-07T21:45:48.115521Z", + "iopub.status.idle": "2023-09-07T21:45:48.116818Z", + "shell.execute_reply": "2023-09-07T21:45:48.116678Z" + }, + "papermill": { + "duration": 0.003452, + "end_time": "2023-09-07T21:45:48.117323", + "exception": false, + "start_time": "2023-09-07T21:45:48.113871", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "n_genes, n_samples = 100, 1000" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "b2f92fb1-113d-479b-8bbf-2be229e26e8f", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:45:48.120768Z", + "iopub.status.busy": "2023-09-07T21:45:48.120368Z", + "iopub.status.idle": "2023-09-07T21:45:48.121990Z", + "shell.execute_reply": "2023-09-07T21:45:48.121852Z" + }, + "papermill": { + "duration": 0.003765, + "end_time": "2023-09-07T21:45:48.122435", + "exception": false, + "start_time": "2023-09-07T21:45:48.118670", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "np.random.seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "63638c0b-b436-48a9-93e0-db2adb939a61", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:45:48.125277Z", + "iopub.status.busy": "2023-09-07T21:45:48.125175Z", + "iopub.status.idle": "2023-09-07T21:45:48.126765Z", + "shell.execute_reply": "2023-09-07T21:45:48.126637Z" + }, + "papermill": { + "duration": 0.003447, + "end_time": "2023-09-07T21:45:48.127169", + "exception": false, + "start_time": "2023-09-07T21:45:48.123722", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "data = np.random.rand(n_genes, n_samples)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "808017ed-9a8a-4bf7-a3dd-42317a39ce8f", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:45:48.129736Z", + "iopub.status.busy": "2023-09-07T21:45:48.129640Z", + "iopub.status.idle": "2023-09-07T21:45:48.131152Z", + "shell.execute_reply": "2023-09-07T21:45:48.131026Z" + }, + "papermill": { + "duration": 0.00322, + "end_time": "2023-09-07T21:45:48.131565", + "exception": false, + "start_time": "2023-09-07T21:45:48.128345", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(100, 1000)" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.shape" + ] + }, + { + "cell_type": "markdown", + "id": "716e4219-cad5-453b-8331-47d310689e03", + "metadata": { + "papermill": { + "duration": 0.001149, + "end_time": "2023-09-07T21:45:48.133997", + "exception": false, + "start_time": "2023-09-07T21:45:48.132848", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Profile" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "67807856-f337-4c6e-ae31-cd306577a314", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:45:48.136863Z", + "iopub.status.busy": "2023-09-07T21:45:48.136766Z", + "iopub.status.idle": "2023-09-07T21:45:48.138426Z", + "shell.execute_reply": "2023-09-07T21:45:48.138212Z" + }, + "papermill": { + "duration": 0.003623, + "end_time": "2023-09-07T21:45:48.138843", + "exception": false, + "start_time": "2023-09-07T21:45:48.135220", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def func():\n", + " n_clust = list(range(2, 10 + 1))\n", + " return ccc(data, internal_n_clusters=n_clust, n_jobs=2)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "2965a695-5c0c-4e9e-8435-dcbfa610eb81", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:45:48.141486Z", + "iopub.status.busy": "2023-09-07T21:45:48.141414Z", + "iopub.status.idle": "2023-09-07T21:48:49.864783Z", + "shell.execute_reply": "2023-09-07T21:48:49.864453Z" + }, + "papermill": { + "duration": 181.726169, + "end_time": "2023-09-07T21:48:49.866236", + "exception": false, + "start_time": "2023-09-07T21:45:48.140067", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "11.4 s ± 55.3 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" + ] + } + ], + "source": [ + "%%timeit func()\n", + "func()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2556204-1c10-4e01-8c6c-ea63ddb37530", + "metadata": { + "papermill": { + "duration": 0.004461, + "end_time": "2023-09-07T21:48:49.880060", + "exception": false, + "start_time": "2023-09-07T21:48:49.875599", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "all,-execution,-papermill,-trusted", + "notebook_metadata_filter": "-jupytext.text_representation.jupytext_version" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + }, + "papermill": { + "default_parameters": {}, + "duration": 187.453637, + "end_time": "2023-09-07T21:48:50.100455", + "environment_variables": {}, + "exception": null, + "input_path": "nbs/others/05_ccc_profiling/12_cm_optimized/16_00-2_CPUs-default-many_genes.ipynb", + "output_path": "nbs/others/05_ccc_profiling/12_cm_optimized/16_00-2_CPUs-default-many_genes.run.ipynb", + "parameters": {}, + "start_time": "2023-09-07T21:45:42.646818", + "version": "2.3.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/nbs/others/05_ccc_profiling/12_cm_optimized/16_01-2_CPUs-disable_numba-many_genes.ipynb b/nbs/others/05_ccc_profiling/12_cm_optimized/16_01-2_CPUs-disable_numba-many_genes.ipynb new file mode 100644 index 00000000..11a68ada --- /dev/null +++ b/nbs/others/05_ccc_profiling/12_cm_optimized/16_01-2_CPUs-disable_numba-many_genes.ipynb @@ -0,0 +1,627 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "507e9319-381b-4934-987c-2958e7f6ace7", + "metadata": { + "papermill": { + "duration": 0.007021, + "end_time": "2023-09-07T21:48:51.584091", + "exception": false, + "start_time": "2023-09-07T21:48:51.577070", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Description" + ] + }, + { + "cell_type": "markdown", + "id": "e7ec2eb7-bc8f-4dea-90fb-05a3bf5f89e7", + "metadata": { + "papermill": { + "duration": 0.00475, + "end_time": "2023-09-07T21:48:51.598675", + "exception": false, + "start_time": "2023-09-07T21:48:51.593925", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "Like `06_01` but using more CPU cores." + ] + }, + { + "cell_type": "markdown", + "id": "fc48320a-d320-4334-a64d-9777d2b0e48b", + "metadata": { + "papermill": { + "duration": 0.002137, + "end_time": "2023-09-07T21:48:51.605860", + "exception": false, + "start_time": "2023-09-07T21:48:51.603723", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Use multiple CPU core" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "5d804148-5abf-4081-ab3d-1fe077111a67", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:48:51.610346Z", + "iopub.status.busy": "2023-09-07T21:48:51.610227Z", + "iopub.status.idle": "2023-09-07T21:48:51.616439Z", + "shell.execute_reply": "2023-09-07T21:48:51.616073Z" + }, + "papermill": { + "duration": 0.009973, + "end_time": "2023-09-07T21:48:51.617556", + "exception": false, + "start_time": "2023-09-07T21:48:51.607583", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env: CM_N_JOBS=2\n", + "env: NUMBA_NUM_THREADS=2\n", + "env: MKL_NUM_THREADS=2\n", + "env: OPEN_BLAS_NUM_THREADS=2\n", + "env: NUMEXPR_NUM_THREADS=2\n", + "env: OMP_NUM_THREADS=2\n" + ] + } + ], + "source": [ + "%env CM_N_JOBS=2\n", + "%env NUMBA_NUM_THREADS=2\n", + "%env MKL_NUM_THREADS=2\n", + "%env OPEN_BLAS_NUM_THREADS=2\n", + "%env NUMEXPR_NUM_THREADS=2\n", + "%env OMP_NUM_THREADS=2" + ] + }, + { + "cell_type": "markdown", + "id": "61443ae0-70fe-42c4-adf2-76b8a7b22217", + "metadata": { + "papermill": { + "duration": 0.001473, + "end_time": "2023-09-07T21:48:51.621052", + "exception": false, + "start_time": "2023-09-07T21:48:51.619579", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Disable numba" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "9495a29b-3ba6-4a51-99b0-780fa5c1d9c7", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:48:51.624039Z", + "iopub.status.busy": "2023-09-07T21:48:51.623964Z", + "iopub.status.idle": "2023-09-07T21:48:51.625471Z", + "shell.execute_reply": "2023-09-07T21:48:51.625337Z" + }, + "papermill": { + "duration": 0.003609, + "end_time": "2023-09-07T21:48:51.625897", + "exception": false, + "start_time": "2023-09-07T21:48:51.622288", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env: NUMBA_DISABLE_JIT=1\n" + ] + } + ], + "source": [ + "%env NUMBA_DISABLE_JIT=1" + ] + }, + { + "cell_type": "markdown", + "id": "72de9f6f-dc00-435e-9dba-871db859a78a", + "metadata": { + "papermill": { + "duration": 0.001017, + "end_time": "2023-09-07T21:48:51.628020", + "exception": false, + "start_time": "2023-09-07T21:48:51.627003", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Remove pycache dir" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "88d32361-f1b5-4cf0-9a2c-7ab927d14b42", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:48:51.630698Z", + "iopub.status.busy": "2023-09-07T21:48:51.630582Z", + "iopub.status.idle": "2023-09-07T21:48:51.816790Z", + "shell.execute_reply": "2023-09-07T21:48:51.815649Z" + }, + "papermill": { + "duration": 0.190549, + "end_time": "2023-09-07T21:48:51.819619", + "exception": false, + "start_time": "2023-09-07T21:48:51.629070", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/opt/code\r\n" + ] + } + ], + "source": [ + "!echo ${CODE_DIR}" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "8a27da6e-cf59-4276-888b-57e98fd23ccf", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:48:51.833421Z", + "iopub.status.busy": "2023-09-07T21:48:51.833086Z", + "iopub.status.idle": "2023-09-07T21:48:52.035351Z", + "shell.execute_reply": "2023-09-07T21:48:52.034311Z" + }, + "papermill": { + "duration": 0.211109, + "end_time": "2023-09-07T21:48:52.036587", + "exception": false, + "start_time": "2023-09-07T21:48:51.825478", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/opt/code/libs/ccc/coef/__pycache__\r\n", + "/opt/code/libs/ccc/pytorch/__pycache__\r\n", + "/opt/code/libs/ccc/scipy/__pycache__\r\n", + "/opt/code/libs/ccc/utils/__pycache__\r\n", + "/opt/code/libs/ccc/__pycache__\r\n", + "/opt/code/libs/ccc/sklearn/__pycache__\r\n" + ] + } + ], + "source": [ + "!find ${CODE_DIR} -regex '^.*\\(__pycache__\\)$' -print" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "561480f7-c610-4f60-b40f-8100974ab4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:48:52.043164Z", + "iopub.status.busy": "2023-09-07T21:48:52.042895Z", + "iopub.status.idle": "2023-09-07T21:48:52.243459Z", + "shell.execute_reply": "2023-09-07T21:48:52.241690Z" + }, + "papermill": { + "duration": 0.207557, + "end_time": "2023-09-07T21:48:52.247066", + "exception": false, + "start_time": "2023-09-07T21:48:52.039509", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "!find ${CODE_DIR} -regex '^.*\\(__pycache__\\)$' -prune -exec rm -rf {} \\;" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "a5c90a17-c0ac-49f0-968d-ea9ea8710b00", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:48:52.266142Z", + "iopub.status.busy": "2023-09-07T21:48:52.265278Z", + "iopub.status.idle": "2023-09-07T21:48:52.471509Z", + "shell.execute_reply": "2023-09-07T21:48:52.470309Z" + }, + "papermill": { + "duration": 0.220435, + "end_time": "2023-09-07T21:48:52.473797", + "exception": false, + "start_time": "2023-09-07T21:48:52.253362", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "!find ${CODE_DIR} -regex '^.*\\(__pycache__\\)$' -print" + ] + }, + { + "cell_type": "markdown", + "id": "145563a2-3e46-4f62-8191-7444a0b315bb", + "metadata": { + "papermill": { + "duration": 0.009946, + "end_time": "2023-09-07T21:48:52.487212", + "exception": false, + "start_time": "2023-09-07T21:48:52.477266", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Modules" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "bea3d48e-8823-403f-90f5-aea8a17b357c", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:48:52.497192Z", + "iopub.status.busy": "2023-09-07T21:48:52.496573Z", + "iopub.status.idle": "2023-09-07T21:48:52.611830Z", + "shell.execute_reply": "2023-09-07T21:48:52.611582Z" + }, + "papermill": { + "duration": 0.120804, + "end_time": "2023-09-07T21:48:52.612600", + "exception": false, + "start_time": "2023-09-07T21:48:52.491796", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "from ccc.coef import ccc" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "df6b3793-930b-4c54-9f29-ecc47fc586fb", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:48:52.615656Z", + "iopub.status.busy": "2023-09-07T21:48:52.615585Z", + "iopub.status.idle": "2023-09-07T21:48:52.620299Z", + "shell.execute_reply": "2023-09-07T21:48:52.620173Z" + }, + "papermill": { + "duration": 0.006803, + "end_time": "2023-09-07T21:48:52.620801", + "exception": false, + "start_time": "2023-09-07T21:48:52.613998", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "0.15625" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# let numba compile all the code before profiling\n", + "ccc(np.random.rand(10), np.random.rand(10))" + ] + }, + { + "cell_type": "markdown", + "id": "f8399ca8-265e-4e96-b582-54045cb2f9eb", + "metadata": { + "papermill": { + "duration": 0.001153, + "end_time": "2023-09-07T21:48:52.623380", + "exception": false, + "start_time": "2023-09-07T21:48:52.622227", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Data" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "2316ffcd-a6e4-453f-bb52-779685c5c5bf", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:48:52.627197Z", + "iopub.status.busy": "2023-09-07T21:48:52.627146Z", + "iopub.status.idle": "2023-09-07T21:48:52.628620Z", + "shell.execute_reply": "2023-09-07T21:48:52.628492Z" + }, + "papermill": { + "duration": 0.00372, + "end_time": "2023-09-07T21:48:52.629081", + "exception": false, + "start_time": "2023-09-07T21:48:52.625361", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "n_genes, n_samples = 100, 1000" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "b2f92fb1-113d-479b-8bbf-2be229e26e8f", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:48:52.631885Z", + "iopub.status.busy": "2023-09-07T21:48:52.631770Z", + "iopub.status.idle": "2023-09-07T21:48:52.632986Z", + "shell.execute_reply": "2023-09-07T21:48:52.632849Z" + }, + "papermill": { + "duration": 0.003073, + "end_time": "2023-09-07T21:48:52.633405", + "exception": false, + "start_time": "2023-09-07T21:48:52.630332", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "np.random.seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "63638c0b-b436-48a9-93e0-db2adb939a61", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:48:52.636169Z", + "iopub.status.busy": "2023-09-07T21:48:52.636083Z", + "iopub.status.idle": "2023-09-07T21:48:52.637636Z", + "shell.execute_reply": "2023-09-07T21:48:52.637504Z" + }, + "papermill": { + "duration": 0.003397, + "end_time": "2023-09-07T21:48:52.638059", + "exception": false, + "start_time": "2023-09-07T21:48:52.634662", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "data = np.random.rand(n_genes, n_samples)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "808017ed-9a8a-4bf7-a3dd-42317a39ce8f", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:48:52.640829Z", + "iopub.status.busy": "2023-09-07T21:48:52.640741Z", + "iopub.status.idle": "2023-09-07T21:48:52.642236Z", + "shell.execute_reply": "2023-09-07T21:48:52.642117Z" + }, + "papermill": { + "duration": 0.003343, + "end_time": "2023-09-07T21:48:52.642661", + "exception": false, + "start_time": "2023-09-07T21:48:52.639318", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(100, 1000)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.shape" + ] + }, + { + "cell_type": "markdown", + "id": "716e4219-cad5-453b-8331-47d310689e03", + "metadata": { + "papermill": { + "duration": 0.001196, + "end_time": "2023-09-07T21:48:52.645115", + "exception": false, + "start_time": "2023-09-07T21:48:52.643919", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Profile" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "67807856-f337-4c6e-ae31-cd306577a314", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:48:52.648010Z", + "iopub.status.busy": "2023-09-07T21:48:52.647911Z", + "iopub.status.idle": "2023-09-07T21:48:52.649253Z", + "shell.execute_reply": "2023-09-07T21:48:52.649128Z" + }, + "papermill": { + "duration": 0.00325, + "end_time": "2023-09-07T21:48:52.649634", + "exception": false, + "start_time": "2023-09-07T21:48:52.646384", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def func():\n", + " n_clust = list(range(2, 10 + 1))\n", + " return ccc(data, internal_n_clusters=n_clust, n_jobs=2)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "2965a695-5c0c-4e9e-8435-dcbfa610eb81", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T21:48:52.652445Z", + "iopub.status.busy": "2023-09-07T21:48:52.652357Z", + "iopub.status.idle": "2023-09-07T22:11:39.917119Z", + "shell.execute_reply": "2023-09-07T22:11:39.916871Z" + }, + "papermill": { + "duration": 1367.268702, + "end_time": "2023-09-07T22:11:39.919642", + "exception": false, + "start_time": "2023-09-07T21:48:52.650940", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1min 25s ± 358 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" + ] + } + ], + "source": [ + "%%timeit func()\n", + "func()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c171afa1-7734-4ba5-b140-060d232c8f9c", + "metadata": { + "papermill": { + "duration": 0.004474, + "end_time": "2023-09-07T22:11:39.933349", + "exception": false, + "start_time": "2023-09-07T22:11:39.928875", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "all,-execution,-papermill,-trusted", + "notebook_metadata_filter": "-jupytext.text_representation.jupytext_version" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + }, + "papermill": { + "default_parameters": {}, + "duration": 1369.14247, + "end_time": "2023-09-07T22:11:40.055239", + "environment_variables": {}, + "exception": null, + "input_path": "nbs/others/05_ccc_profiling/12_cm_optimized/16_01-2_CPUs-disable_numba-many_genes.ipynb", + "output_path": "nbs/others/05_ccc_profiling/12_cm_optimized/16_01-2_CPUs-disable_numba-many_genes.run.ipynb", + "parameters": {}, + "start_time": "2023-09-07T21:48:50.912769", + "version": "2.3.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/nbs/others/05_ccc_profiling/12_cm_optimized/16_02-2_CPUs-use_ari_numba-many_genes.ipynb b/nbs/others/05_ccc_profiling/12_cm_optimized/16_02-2_CPUs-use_ari_numba-many_genes.ipynb new file mode 100644 index 00000000..6f5eb474 --- /dev/null +++ b/nbs/others/05_ccc_profiling/12_cm_optimized/16_02-2_CPUs-use_ari_numba-many_genes.ipynb @@ -0,0 +1,595 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "507e9319-381b-4934-987c-2958e7f6ace7", + "metadata": { + "papermill": { + "duration": 0.006575, + "end_time": "2023-09-07T22:11:41.690504", + "exception": false, + "start_time": "2023-09-07T22:11:41.683929", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Description" + ] + }, + { + "cell_type": "markdown", + "id": "7cc1fe32-136c-405d-a48f-763105012eb8", + "metadata": { + "papermill": { + "duration": 0.004164, + "end_time": "2023-09-07T22:11:41.699971", + "exception": false, + "start_time": "2023-09-07T22:11:41.695807", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "Like `06_02` but using more CPU cores." + ] + }, + { + "cell_type": "markdown", + "id": "dca4cada-cdd9-4989-ba16-7a63c53e7b1f", + "metadata": { + "papermill": { + "duration": 0.002127, + "end_time": "2023-09-07T22:11:41.705245", + "exception": false, + "start_time": "2023-09-07T22:11:41.703118", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Use multiple CPU core" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "5d804148-5abf-4081-ab3d-1fe077111a67", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T22:11:41.710102Z", + "iopub.status.busy": "2023-09-07T22:11:41.709849Z", + "iopub.status.idle": "2023-09-07T22:11:41.715249Z", + "shell.execute_reply": "2023-09-07T22:11:41.715038Z" + }, + "papermill": { + "duration": 0.008065, + "end_time": "2023-09-07T22:11:41.715989", + "exception": false, + "start_time": "2023-09-07T22:11:41.707924", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env: CM_N_JOBS=2\n", + "env: NUMBA_NUM_THREADS=2\n", + "env: MKL_NUM_THREADS=2\n", + "env: OPEN_BLAS_NUM_THREADS=2\n", + "env: NUMEXPR_NUM_THREADS=2\n", + "env: OMP_NUM_THREADS=2\n" + ] + } + ], + "source": [ + "%env CM_N_JOBS=2\n", + "%env NUMBA_NUM_THREADS=2\n", + "%env MKL_NUM_THREADS=2\n", + "%env OPEN_BLAS_NUM_THREADS=2\n", + "%env NUMEXPR_NUM_THREADS=2\n", + "%env OMP_NUM_THREADS=2" + ] + }, + { + "cell_type": "markdown", + "id": "4d885a9d-482d-47e7-bfaf-11c8efae3079", + "metadata": { + "papermill": { + "duration": 0.001986, + "end_time": "2023-09-07T22:11:41.720009", + "exception": false, + "start_time": "2023-09-07T22:11:41.718023", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Remove pycache dir" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "88d32361-f1b5-4cf0-9a2c-7ab927d14b42", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T22:11:41.726713Z", + "iopub.status.busy": "2023-09-07T22:11:41.726388Z", + "iopub.status.idle": "2023-09-07T22:11:41.919443Z", + "shell.execute_reply": "2023-09-07T22:11:41.917918Z" + }, + "papermill": { + "duration": 0.199308, + "end_time": "2023-09-07T22:11:41.922214", + "exception": false, + "start_time": "2023-09-07T22:11:41.722906", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/opt/code\n" + ] + } + ], + "source": [ + "!echo ${CODE_DIR}" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "8a27da6e-cf59-4276-888b-57e98fd23ccf", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T22:11:41.940423Z", + "iopub.status.busy": "2023-09-07T22:11:41.939528Z", + "iopub.status.idle": "2023-09-07T22:11:42.144958Z", + "shell.execute_reply": "2023-09-07T22:11:42.143827Z" + }, + "papermill": { + "duration": 0.219934, + "end_time": "2023-09-07T22:11:42.147681", + "exception": false, + "start_time": "2023-09-07T22:11:41.927747", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/opt/code/libs/ccc/coef/__pycache__\n", + "/opt/code/libs/ccc/pytorch/__pycache__\n", + "/opt/code/libs/ccc/scipy/__pycache__\n", + "/opt/code/libs/ccc/utils/__pycache__\n", + "/opt/code/libs/ccc/__pycache__\n", + "/opt/code/libs/ccc/sklearn/__pycache__\n" + ] + } + ], + "source": [ + "!find ${CODE_DIR} -regex '^.*\\(__pycache__\\)$' -print" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "561480f7-c610-4f60-b40f-8100974ab4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T22:11:42.160792Z", + "iopub.status.busy": "2023-09-07T22:11:42.160375Z", + "iopub.status.idle": "2023-09-07T22:11:42.370255Z", + "shell.execute_reply": "2023-09-07T22:11:42.369154Z" + }, + "papermill": { + "duration": 0.219443, + "end_time": "2023-09-07T22:11:42.372986", + "exception": false, + "start_time": "2023-09-07T22:11:42.153543", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "find: ‘/opt/code/libs/ccc/coef/__pycache__’: No such file or directory\n", + "find: ‘/opt/code/libs/ccc/pytorch/__pycache__’: No such file or directory\n", + "find: ‘/opt/code/libs/ccc/scipy/__pycache__’: No such file or directory\n", + "find: ‘/opt/code/libs/ccc/utils/__pycache__’: No such file or directory\n", + "find: ‘/opt/code/libs/ccc/__pycache__’: No such file or directory\n", + "find: ‘/opt/code/libs/ccc/sklearn/__pycache__’: No such file or directory\n" + ] + } + ], + "source": [ + "!find ${CODE_DIR} -regex '^.*\\(__pycache__\\)$' -exec rm -rf {} \\;" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "a5c90a17-c0ac-49f0-968d-ea9ea8710b00", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T22:11:42.381184Z", + "iopub.status.busy": "2023-09-07T22:11:42.380919Z", + "iopub.status.idle": "2023-09-07T22:11:42.577334Z", + "shell.execute_reply": "2023-09-07T22:11:42.575996Z" + }, + "papermill": { + "duration": 0.201605, + "end_time": "2023-09-07T22:11:42.580563", + "exception": false, + "start_time": "2023-09-07T22:11:42.378958", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "!find ${CODE_DIR} -regex '^.*\\(__pycache__\\)$' -print" + ] + }, + { + "cell_type": "markdown", + "id": "145563a2-3e46-4f62-8191-7444a0b315bb", + "metadata": { + "papermill": { + "duration": 0.009875, + "end_time": "2023-09-07T22:11:42.596372", + "exception": false, + "start_time": "2023-09-07T22:11:42.586497", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Modules" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "bea3d48e-8823-403f-90f5-aea8a17b357c", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T22:11:42.608227Z", + "iopub.status.busy": "2023-09-07T22:11:42.608039Z", + "iopub.status.idle": "2023-09-07T22:11:46.297457Z", + "shell.execute_reply": "2023-09-07T22:11:46.297216Z" + }, + "papermill": { + "duration": 3.696182, + "end_time": "2023-09-07T22:11:46.298336", + "exception": false, + "start_time": "2023-09-07T22:11:42.602154", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "from ccc.coef import ccc" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "df6b3793-930b-4c54-9f29-ecc47fc586fb", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T22:11:46.307570Z", + "iopub.status.busy": "2023-09-07T22:11:46.307515Z", + "iopub.status.idle": "2023-09-07T22:11:46.311903Z", + "shell.execute_reply": "2023-09-07T22:11:46.311774Z" + }, + "papermill": { + "duration": 0.012697, + "end_time": "2023-09-07T22:11:46.312399", + "exception": false, + "start_time": "2023-09-07T22:11:46.299702", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "0.20454545454545456" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# let numba compile all the code before profiling\n", + "ccc(np.random.rand(10), np.random.rand(10))" + ] + }, + { + "cell_type": "markdown", + "id": "f8399ca8-265e-4e96-b582-54045cb2f9eb", + "metadata": { + "papermill": { + "duration": 0.009559, + "end_time": "2023-09-07T22:11:46.323333", + "exception": false, + "start_time": "2023-09-07T22:11:46.313774", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Data" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "2316ffcd-a6e4-453f-bb52-779685c5c5bf", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T22:11:46.329827Z", + "iopub.status.busy": "2023-09-07T22:11:46.329631Z", + "iopub.status.idle": "2023-09-07T22:11:46.331427Z", + "shell.execute_reply": "2023-09-07T22:11:46.331208Z" + }, + "papermill": { + "duration": 0.005926, + "end_time": "2023-09-07T22:11:46.332215", + "exception": false, + "start_time": "2023-09-07T22:11:46.326289", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "n_genes, n_samples = 100, 1000" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "b2f92fb1-113d-479b-8bbf-2be229e26e8f", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T22:11:46.336896Z", + "iopub.status.busy": "2023-09-07T22:11:46.336700Z", + "iopub.status.idle": "2023-09-07T22:11:46.338376Z", + "shell.execute_reply": "2023-09-07T22:11:46.338164Z" + }, + "papermill": { + "duration": 0.004776, + "end_time": "2023-09-07T22:11:46.339061", + "exception": false, + "start_time": "2023-09-07T22:11:46.334285", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "np.random.seed(0)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "63638c0b-b436-48a9-93e0-db2adb939a61", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T22:11:46.343186Z", + "iopub.status.busy": "2023-09-07T22:11:46.343066Z", + "iopub.status.idle": "2023-09-07T22:11:46.344943Z", + "shell.execute_reply": "2023-09-07T22:11:46.344788Z" + }, + "papermill": { + "duration": 0.004357, + "end_time": "2023-09-07T22:11:46.345415", + "exception": false, + "start_time": "2023-09-07T22:11:46.341058", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "data = np.random.rand(n_genes, n_samples)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "808017ed-9a8a-4bf7-a3dd-42317a39ce8f", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T22:11:46.348539Z", + "iopub.status.busy": "2023-09-07T22:11:46.348432Z", + "iopub.status.idle": "2023-09-07T22:11:46.349888Z", + "shell.execute_reply": "2023-09-07T22:11:46.349771Z" + }, + "papermill": { + "duration": 0.003441, + "end_time": "2023-09-07T22:11:46.350329", + "exception": false, + "start_time": "2023-09-07T22:11:46.346888", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(100, 1000)" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.shape" + ] + }, + { + "cell_type": "markdown", + "id": "716e4219-cad5-453b-8331-47d310689e03", + "metadata": { + "papermill": { + "duration": 0.00114, + "end_time": "2023-09-07T22:11:46.352730", + "exception": false, + "start_time": "2023-09-07T22:11:46.351590", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Profile" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "67807856-f337-4c6e-ae31-cd306577a314", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T22:11:46.355380Z", + "iopub.status.busy": "2023-09-07T22:11:46.355330Z", + "iopub.status.idle": "2023-09-07T22:11:46.356661Z", + "shell.execute_reply": "2023-09-07T22:11:46.356536Z" + }, + "papermill": { + "duration": 0.003145, + "end_time": "2023-09-07T22:11:46.357085", + "exception": false, + "start_time": "2023-09-07T22:11:46.353940", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def func():\n", + " n_clust = list(range(2, 10 + 1))\n", + " return ccc(data, internal_n_clusters=n_clust, use_ari_numba=True, n_jobs=2)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "2965a695-5c0c-4e9e-8435-dcbfa610eb81", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-07T22:11:46.359885Z", + "iopub.status.busy": "2023-09-07T22:11:46.359787Z", + "iopub.status.idle": "2023-09-07T22:14:38.499215Z", + "shell.execute_reply": "2023-09-07T22:14:38.499037Z" + }, + "papermill": { + "duration": 172.142755, + "end_time": "2023-09-07T22:14:38.501086", + "exception": false, + "start_time": "2023-09-07T22:11:46.358331", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "10.7 s ± 189 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" + ] + } + ], + "source": [ + "%%timeit func()\n", + "func()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e2556204-1c10-4e01-8c6c-ea63ddb37530", + "metadata": { + "papermill": { + "duration": 0.0097, + "end_time": "2023-09-07T22:14:49.199847", + "exception": false, + "start_time": "2023-09-07T22:14:49.190147", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "all,-execution,-papermill,-trusted", + "notebook_metadata_filter": "-jupytext.text_representation.jupytext_version", + "text_representation": { + "extension": ".py", + "format_name": "percent", + "format_version": "1.3" + } + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + }, + "papermill": { + "default_parameters": {}, + "duration": 188.544552, + "end_time": "2023-09-07T22:14:49.418496", + "environment_variables": {}, + "exception": null, + "input_path": "nbs/others/05_ccc_profiling/12_cm_optimized/16_02-2_CPUs-use_ari_numba-many_genes.ipynb", + "output_path": "nbs/others/05_ccc_profiling/12_cm_optimized/16_02-2_CPUs-use_ari_numba-many_genes.run.ipynb", + "parameters": {}, + "start_time": "2023-09-07T22:11:40.873944", + "version": "2.3.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/nbs/others/05_ccc_profiling/12_cm_optimized/py/16_00-2_CPUs-default-many_genes.py b/nbs/others/05_ccc_profiling/12_cm_optimized/py/16_00-2_CPUs-default-many_genes.py new file mode 100644 index 00000000..f9dd9205 --- /dev/null +++ b/nbs/others/05_ccc_profiling/12_cm_optimized/py/16_00-2_CPUs-default-many_genes.py @@ -0,0 +1,89 @@ +# --- +# jupyter: +# jupytext: +# cell_metadata_filter: all,-execution,-papermill,-trusted +# notebook_metadata_filter: -jupytext.text_representation.jupytext_version +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# kernelspec: +# display_name: Python 3 (ipykernel) +# language: python +# name: python3 +# --- + +# %% [markdown] tags=[] +# # Description + +# %% [markdown] tags=[] +# Like `06_00` but using more CPU cores. + +# %% [markdown] tags=[] +# # Use multiple CPU core + +# %% tags=[] +# %env CM_N_JOBS=2 +# %env NUMBA_NUM_THREADS=2 +# %env MKL_NUM_THREADS=2 +# %env OPEN_BLAS_NUM_THREADS=2 +# %env NUMEXPR_NUM_THREADS=2 +# %env OMP_NUM_THREADS=2 + +# %% [markdown] tags=[] +# # Remove pycache dir + +# %% tags=[] +# !echo ${CODE_DIR} + +# %% tags=[] +# !find ${CODE_DIR} -regex '^.*\(__pycache__\)$' -print + +# %% tags=[] +# !find ${CODE_DIR} -regex '^.*\(__pycache__\)$' -exec rm -rf {} \; + +# %% tags=[] +# !find ${CODE_DIR} -regex '^.*\(__pycache__\)$' -print + +# %% [markdown] tags=[] +# # Modules + +# %% tags=[] +import numpy as np + +from ccc.coef import ccc + +# %% tags=[] +# let numba compile all the code before profiling +ccc(np.random.rand(10), np.random.rand(10)) + +# %% [markdown] tags=[] +# # Data + +# %% tags=[] +n_genes, n_samples = 100, 1000 + +# %% tags=[] +np.random.seed(0) + +# %% tags=[] +data = np.random.rand(n_genes, n_samples) + +# %% tags=[] +data.shape + + +# %% [markdown] tags=[] +# # Profile + +# %% tags=[] +def func(): + n_clust = list(range(2, 10 + 1)) + return ccc(data, internal_n_clusters=n_clust, n_jobs=2) + + +# %% tags=[] +# %%timeit func() +func() + +# %% tags=[] diff --git a/nbs/others/05_ccc_profiling/12_cm_optimized/py/16_01-2_CPUs-disable_numba-many_genes.py b/nbs/others/05_ccc_profiling/12_cm_optimized/py/16_01-2_CPUs-disable_numba-many_genes.py new file mode 100644 index 00000000..6fbf6eb9 --- /dev/null +++ b/nbs/others/05_ccc_profiling/12_cm_optimized/py/16_01-2_CPUs-disable_numba-many_genes.py @@ -0,0 +1,95 @@ +# --- +# jupyter: +# jupytext: +# cell_metadata_filter: all,-execution,-papermill,-trusted +# notebook_metadata_filter: -jupytext.text_representation.jupytext_version +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# kernelspec: +# display_name: Python 3 (ipykernel) +# language: python +# name: python3 +# --- + +# %% [markdown] tags=[] +# # Description + +# %% [markdown] tags=[] +# Like `06_01` but using more CPU cores. + +# %% [markdown] tags=[] +# # Use multiple CPU core + +# %% tags=[] +# %env CM_N_JOBS=2 +# %env NUMBA_NUM_THREADS=2 +# %env MKL_NUM_THREADS=2 +# %env OPEN_BLAS_NUM_THREADS=2 +# %env NUMEXPR_NUM_THREADS=2 +# %env OMP_NUM_THREADS=2 + +# %% [markdown] tags=[] +# # Disable numba + +# %% tags=[] +# %env NUMBA_DISABLE_JIT=1 + +# %% [markdown] tags=[] +# # Remove pycache dir + +# %% tags=[] +# !echo ${CODE_DIR} + +# %% tags=[] +# !find ${CODE_DIR} -regex '^.*\(__pycache__\)$' -print + +# %% tags=[] +# !find ${CODE_DIR} -regex '^.*\(__pycache__\)$' -prune -exec rm -rf {} \; + +# %% tags=[] +# !find ${CODE_DIR} -regex '^.*\(__pycache__\)$' -print + +# %% [markdown] tags=[] +# # Modules + +# %% tags=[] +import numpy as np + +from ccc.coef import ccc + +# %% tags=[] +# let numba compile all the code before profiling +ccc(np.random.rand(10), np.random.rand(10)) + +# %% [markdown] tags=[] +# # Data + +# %% tags=[] +n_genes, n_samples = 100, 1000 + +# %% tags=[] +np.random.seed(0) + +# %% tags=[] +data = np.random.rand(n_genes, n_samples) + +# %% tags=[] +data.shape + + +# %% [markdown] tags=[] +# # Profile + +# %% tags=[] +def func(): + n_clust = list(range(2, 10 + 1)) + return ccc(data, internal_n_clusters=n_clust, n_jobs=2) + + +# %% tags=[] +# %%timeit func() +func() + +# %% tags=[] diff --git a/nbs/others/05_ccc_profiling/12_cm_optimized/py/16_02-2_CPUs-use_ari_numba-many_genes.py b/nbs/others/05_ccc_profiling/12_cm_optimized/py/16_02-2_CPUs-use_ari_numba-many_genes.py new file mode 100644 index 00000000..4f21b7e5 --- /dev/null +++ b/nbs/others/05_ccc_profiling/12_cm_optimized/py/16_02-2_CPUs-use_ari_numba-many_genes.py @@ -0,0 +1,89 @@ +# --- +# jupyter: +# jupytext: +# cell_metadata_filter: all,-execution,-papermill,-trusted +# notebook_metadata_filter: -jupytext.text_representation.jupytext_version +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# kernelspec: +# display_name: Python 3 (ipykernel) +# language: python +# name: python3 +# --- + +# %% [markdown] tags=[] +# # Description + +# %% [markdown] tags=[] +# Like `06_02` but using more CPU cores. + +# %% [markdown] tags=[] +# # Use multiple CPU core + +# %% tags=[] +# %env CM_N_JOBS=2 +# %env NUMBA_NUM_THREADS=2 +# %env MKL_NUM_THREADS=2 +# %env OPEN_BLAS_NUM_THREADS=2 +# %env NUMEXPR_NUM_THREADS=2 +# %env OMP_NUM_THREADS=2 + +# %% [markdown] tags=[] +# # Remove pycache dir + +# %% tags=[] +# !echo ${CODE_DIR} + +# %% tags=[] +# !find ${CODE_DIR} -regex '^.*\(__pycache__\)$' -print + +# %% tags=[] +# !find ${CODE_DIR} -regex '^.*\(__pycache__\)$' -exec rm -rf {} \; + +# %% tags=[] +# !find ${CODE_DIR} -regex '^.*\(__pycache__\)$' -print + +# %% [markdown] tags=[] +# # Modules + +# %% tags=[] +import numpy as np + +from ccc.coef import ccc + +# %% tags=[] +# let numba compile all the code before profiling +ccc(np.random.rand(10), np.random.rand(10)) + +# %% [markdown] tags=[] +# # Data + +# %% tags=[] +n_genes, n_samples = 100, 1000 + +# %% tags=[] +np.random.seed(0) + +# %% tags=[] +data = np.random.rand(n_genes, n_samples) + +# %% tags=[] +data.shape + + +# %% [markdown] tags=[] +# # Profile + +# %% tags=[] +def func(): + n_clust = list(range(2, 10 + 1)) + return ccc(data, internal_n_clusters=n_clust, use_ari_numba=True, n_jobs=2) + + +# %% tags=[] +# %%timeit func() +func() + +# %% tags=[]