From fb197bad85e004910ffd7821e119128814ace566 Mon Sep 17 00:00:00 2001 From: Milton Pividori Date: Mon, 11 Sep 2023 11:33:54 -0600 Subject: [PATCH] 20_comparison_others: add time test with 6 CPU cores and run --- .../60-time_test-1_cpu_core.ipynb | 264 +++--- .../61-time_test-3_cpu_cores.ipynb | 290 +++--- .../62-time_test-6_cpu_cores.ipynb | 850 ++++++++++++++++++ .../py/62-time_test-6_cpu_cores.py | 155 ++++ 4 files changed, 1282 insertions(+), 277 deletions(-) create mode 100644 nbs/20_comparison_others/62-time_test-6_cpu_cores.ipynb create mode 100644 nbs/20_comparison_others/py/62-time_test-6_cpu_cores.py diff --git a/nbs/20_comparison_others/60-time_test-1_cpu_core.ipynb b/nbs/20_comparison_others/60-time_test-1_cpu_core.ipynb index 1e865f99..00fb2589 100644 --- a/nbs/20_comparison_others/60-time_test-1_cpu_core.ipynb +++ b/nbs/20_comparison_others/60-time_test-1_cpu_core.ipynb @@ -5,10 +5,10 @@ "id": "c1948eb4-fb63-4fad-8243-bcd57b98def3", "metadata": { "papermill": { - "duration": 0.006927, - "end_time": "2023-09-11T12:28:33.507484", + "duration": 0.00652, + "end_time": "2023-09-11T17:10:08.881304", "exception": false, - "start_time": "2023-09-11T12:28:33.500557", + "start_time": "2023-09-11T17:10:08.874784", "status": "completed" }, "tags": [] @@ -22,10 +22,10 @@ "id": "0abb1d73-c786-4c6d-9dc2-b9243e5183e2", "metadata": { "papermill": { - "duration": 0.002372, - "end_time": "2023-09-11T12:28:33.511069", + "duration": 0.00543, + "end_time": "2023-09-11T17:10:08.896773", "exception": false, - "start_time": "2023-09-11T12:28:33.508697", + "start_time": "2023-09-11T17:10:08.891343", "status": "completed" }, "tags": [] @@ -41,10 +41,10 @@ "id": "d6252b4c-6d56-4cf0-aecd-5d8c769a3609", "metadata": { "papermill": { - "duration": 0.002433, - "end_time": "2023-09-11T12:28:33.516763", + "duration": 0.002716, + "end_time": "2023-09-11T17:10:08.905896", "exception": false, - "start_time": "2023-09-11T12:28:33.514330", + "start_time": "2023-09-11T17:10:08.903180", "status": "completed" }, "tags": [] @@ -58,10 +58,10 @@ "id": "d31be5d5-739c-4bec-8409-69f5b6045326", "metadata": { "papermill": { - "duration": 0.001564, - "end_time": "2023-09-11T12:28:33.520152", + "duration": 0.001463, + "end_time": "2023-09-11T17:10:08.909565", "exception": false, - "start_time": "2023-09-11T12:28:33.518588", + "start_time": "2023-09-11T17:10:08.908102", "status": "completed" }, "tags": [] @@ -76,16 +76,16 @@ "id": "ea7d30d5-da78-454a-a4ff-e6307bfdbb37", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:28:33.523599Z", - "iopub.status.busy": "2023-09-11T12:28:33.523485Z", - "iopub.status.idle": "2023-09-11T12:28:33.528234Z", - "shell.execute_reply": "2023-09-11T12:28:33.528075Z" + "iopub.execute_input": "2023-09-11T17:10:08.915627Z", + "iopub.status.busy": "2023-09-11T17:10:08.915518Z", + "iopub.status.idle": "2023-09-11T17:10:08.921657Z", + "shell.execute_reply": "2023-09-11T17:10:08.921399Z" }, "papermill": { - "duration": 0.007052, - "end_time": "2023-09-11T12:28:33.528689", + "duration": 0.010774, + "end_time": "2023-09-11T17:10:08.922797", "exception": false, - "start_time": "2023-09-11T12:28:33.521637", + "start_time": "2023-09-11T17:10:08.912023", "status": "completed" }, "tags": [] @@ -119,16 +119,16 @@ "id": "77e9d29d-5307-4b4a-b103-7d1fbd6a7e56", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:28:33.531286Z", - "iopub.status.busy": "2023-09-11T12:28:33.531232Z", - "iopub.status.idle": "2023-09-11T12:28:33.832672Z", - "shell.execute_reply": "2023-09-11T12:28:33.832423Z" + "iopub.execute_input": "2023-09-11T17:10:08.926569Z", + "iopub.status.busy": "2023-09-11T17:10:08.926506Z", + "iopub.status.idle": "2023-09-11T17:10:09.234441Z", + "shell.execute_reply": "2023-09-11T17:10:09.234266Z" }, "papermill": { - "duration": 0.3036, - "end_time": "2023-09-11T12:28:33.833478", + "duration": 0.310593, + "end_time": "2023-09-11T17:10:09.235229", "exception": false, - "start_time": "2023-09-11T12:28:33.529878", + "start_time": "2023-09-11T17:10:08.924636", "status": "completed" }, "tags": [] @@ -151,10 +151,10 @@ "id": "262eb535-3e5d-43d7-9efd-bd6cfdac9190", "metadata": { "papermill": { - "duration": 0.009637, - "end_time": "2023-09-11T12:28:33.844495", + "duration": 0.00966, + "end_time": "2023-09-11T17:10:09.246267", "exception": false, - "start_time": "2023-09-11T12:28:33.834858", + "start_time": "2023-09-11T17:10:09.236607", "status": "completed" }, "tags": [] @@ -169,16 +169,16 @@ "id": "311cc161-c947-447e-b622-6df22ad0e03a", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:28:33.852513Z", - "iopub.status.busy": "2023-09-11T12:28:33.852437Z", - "iopub.status.idle": "2023-09-11T12:28:33.854147Z", - "shell.execute_reply": "2023-09-11T12:28:33.853938Z" + "iopub.execute_input": "2023-09-11T17:10:09.253889Z", + "iopub.status.busy": "2023-09-11T17:10:09.253819Z", + "iopub.status.idle": "2023-09-11T17:10:09.255472Z", + "shell.execute_reply": "2023-09-11T17:10:09.255296Z" }, "papermill": { - "duration": 0.005822, - "end_time": "2023-09-11T12:28:33.854904", + "duration": 0.005161, + "end_time": "2023-09-11T17:10:09.256091", "exception": false, - "start_time": "2023-09-11T12:28:33.849082", + "start_time": "2023-09-11T17:10:09.250930", "status": "completed" }, "tags": [] @@ -194,16 +194,16 @@ "id": "2d2bd14a-a40d-4710-98c2-587148d0f532", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:28:33.858884Z", - "iopub.status.busy": "2023-09-11T12:28:33.858757Z", - "iopub.status.idle": "2023-09-11T12:28:33.860737Z", - "shell.execute_reply": "2023-09-11T12:28:33.860489Z" + "iopub.execute_input": "2023-09-11T17:10:09.259746Z", + "iopub.status.busy": "2023-09-11T17:10:09.259606Z", + "iopub.status.idle": "2023-09-11T17:10:09.261398Z", + "shell.execute_reply": "2023-09-11T17:10:09.261196Z" }, "papermill": { - "duration": 0.004895, - "end_time": "2023-09-11T12:28:33.861521", + "duration": 0.004393, + "end_time": "2023-09-11T17:10:09.262084", "exception": false, - "start_time": "2023-09-11T12:28:33.856626", + "start_time": "2023-09-11T17:10:09.257691", "status": "completed" }, "tags": [] @@ -231,16 +231,16 @@ "id": "d24607ca-2774-4e51-b392-538c85c86052", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:28:33.866148Z", - "iopub.status.busy": "2023-09-11T12:28:33.866064Z", - "iopub.status.idle": "2023-09-11T12:28:33.867690Z", - "shell.execute_reply": "2023-09-11T12:28:33.867506Z" + "iopub.execute_input": "2023-09-11T17:10:09.266814Z", + "iopub.status.busy": "2023-09-11T17:10:09.266672Z", + "iopub.status.idle": "2023-09-11T17:10:09.268301Z", + "shell.execute_reply": "2023-09-11T17:10:09.268121Z" }, "papermill": { - "duration": 0.004786, - "end_time": "2023-09-11T12:28:33.868436", + "duration": 0.004666, + "end_time": "2023-09-11T17:10:09.268919", "exception": false, - "start_time": "2023-09-11T12:28:33.863650", + "start_time": "2023-09-11T17:10:09.264253", "status": "completed" }, "tags": [] @@ -255,10 +255,10 @@ "id": "eec9bb6d-4ba2-4817-9b36-bc8e68a2beae", "metadata": { "papermill": { - "duration": 0.0014, - "end_time": "2023-09-11T12:28:33.871550", + "duration": 0.001508, + "end_time": "2023-09-11T17:10:09.272230", "exception": false, - "start_time": "2023-09-11T12:28:33.870150", + "start_time": "2023-09-11T17:10:09.270722", "status": "completed" }, "tags": [] @@ -273,16 +273,16 @@ "id": "ee9e6b70-9abd-4714-9ce5-49c270a1b0be", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:28:33.874271Z", - "iopub.status.busy": "2023-09-11T12:28:33.874219Z", - "iopub.status.idle": "2023-09-11T12:28:33.877439Z", - "shell.execute_reply": "2023-09-11T12:28:33.877317Z" + "iopub.execute_input": "2023-09-11T17:10:09.275281Z", + "iopub.status.busy": "2023-09-11T17:10:09.275189Z", + "iopub.status.idle": "2023-09-11T17:10:09.278390Z", + "shell.execute_reply": "2023-09-11T17:10:09.278269Z" }, "papermill": { - "duration": 0.005066, - "end_time": "2023-09-11T12:28:33.877887", + "duration": 0.005245, + "end_time": "2023-09-11T17:10:09.278917", "exception": false, - "start_time": "2023-09-11T12:28:33.872821", + "start_time": "2023-09-11T17:10:09.273672", "status": "completed" }, "tags": [] @@ -309,10 +309,10 @@ "id": "9cb68076-f855-45c8-9843-ce2d93bdc4e7", "metadata": { "papermill": { - "duration": 0.001097, - "end_time": "2023-09-11T12:28:33.880143", + "duration": 0.001071, + "end_time": "2023-09-11T17:10:09.281153", "exception": false, - "start_time": "2023-09-11T12:28:33.879046", + "start_time": "2023-09-11T17:10:09.280082", "status": "completed" }, "tags": [] @@ -327,16 +327,16 @@ "id": "e2c25ecf-0d87-4447-a9ff-a3b511629a61", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:28:33.882668Z", - "iopub.status.busy": "2023-09-11T12:28:33.882619Z", - "iopub.status.idle": "2023-09-11T12:28:33.884766Z", - "shell.execute_reply": "2023-09-11T12:28:33.884635Z" + "iopub.execute_input": "2023-09-11T17:10:09.283774Z", + "iopub.status.busy": "2023-09-11T17:10:09.283658Z", + "iopub.status.idle": "2023-09-11T17:10:09.286033Z", + "shell.execute_reply": "2023-09-11T17:10:09.285710Z" }, "papermill": { - "duration": 0.003858, - "end_time": "2023-09-11T12:28:33.885138", + "duration": 0.004278, + "end_time": "2023-09-11T17:10:09.286584", "exception": false, - "start_time": "2023-09-11T12:28:33.881280", + "start_time": "2023-09-11T17:10:09.282306", "status": "completed" }, "tags": [] @@ -352,16 +352,16 @@ "id": "ffc0d8de-79a6-4ab8-8cc2-5b720392a0e6", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:28:33.887670Z", - "iopub.status.busy": "2023-09-11T12:28:33.887619Z", - "iopub.status.idle": "2023-09-11T12:28:33.889403Z", - "shell.execute_reply": "2023-09-11T12:28:33.889266Z" + "iopub.execute_input": "2023-09-11T17:10:09.289281Z", + "iopub.status.busy": "2023-09-11T17:10:09.289176Z", + "iopub.status.idle": "2023-09-11T17:10:09.290943Z", + "shell.execute_reply": "2023-09-11T17:10:09.290814Z" }, "papermill": { - "duration": 0.003443, - "end_time": "2023-09-11T12:28:33.889781", + "duration": 0.003551, + "end_time": "2023-09-11T17:10:09.291373", "exception": false, - "start_time": "2023-09-11T12:28:33.886338", + "start_time": "2023-09-11T17:10:09.287822", "status": "completed" }, "tags": [] @@ -391,10 +391,10 @@ "id": "7656ffa5-c9c3-41f5-aeb5-21bde8836e81", "metadata": { "papermill": { - "duration": 0.001121, - "end_time": "2023-09-11T12:28:33.892060", + "duration": 0.001076, + "end_time": "2023-09-11T17:10:09.293614", "exception": false, - "start_time": "2023-09-11T12:28:33.890939", + "start_time": "2023-09-11T17:10:09.292538", "status": "completed" }, "tags": [] @@ -409,16 +409,16 @@ "id": "3e4fac59-34a7-4b63-8aa0-e845c0a156af", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:28:33.894563Z", - "iopub.status.busy": "2023-09-11T12:28:33.894509Z", - "iopub.status.idle": "2023-09-11T12:28:33.897050Z", - "shell.execute_reply": "2023-09-11T12:28:33.896922Z" + "iopub.execute_input": "2023-09-11T17:10:09.296181Z", + "iopub.status.busy": "2023-09-11T17:10:09.296081Z", + "iopub.status.idle": "2023-09-11T17:10:09.298806Z", + "shell.execute_reply": "2023-09-11T17:10:09.298671Z" }, "papermill": { - "duration": 0.004231, - "end_time": "2023-09-11T12:28:33.897451", + "duration": 0.004464, + "end_time": "2023-09-11T17:10:09.299234", "exception": false, - "start_time": "2023-09-11T12:28:33.893220", + "start_time": "2023-09-11T17:10:09.294770", "status": "completed" }, "tags": [] @@ -446,16 +446,16 @@ "id": "e3f26ab4-ec59-4e8f-b887-14abc2cc7cae", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:28:33.900078Z", - "iopub.status.busy": "2023-09-11T12:28:33.900026Z", - "iopub.status.idle": "2023-09-11T12:37:39.864620Z", - "shell.execute_reply": "2023-09-11T12:37:39.864450Z" + "iopub.execute_input": "2023-09-11T17:10:09.301885Z", + "iopub.status.busy": "2023-09-11T17:10:09.301805Z", + "iopub.status.idle": "2023-09-11T17:19:38.305534Z", + "shell.execute_reply": "2023-09-11T17:19:38.305236Z" }, "papermill": { - "duration": 545.968749, - "end_time": "2023-09-11T12:37:39.867447", + "duration": 569.007037, + "end_time": "2023-09-11T17:19:38.307528", "exception": false, - "start_time": "2023-09-11T12:28:33.898698", + "start_time": "2023-09-11T17:10:09.300491", "status": "completed" }, "tags": [] @@ -574,10 +574,10 @@ "id": "2354576e-8a81-48a9-84e4-08e8a990740e", "metadata": { "papermill": { - "duration": 0.00306, - "end_time": "2023-09-11T12:37:39.876320", + "duration": 0.002692, + "end_time": "2023-09-11T17:19:38.316765", "exception": false, - "start_time": "2023-09-11T12:37:39.873260", + "start_time": "2023-09-11T17:19:38.314073", "status": "completed" }, "tags": [] @@ -592,16 +592,16 @@ "id": "e2876c4e-cc31-45ca-aa8c-61ab38ddd9b9", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:37:39.880005Z", - "iopub.status.busy": "2023-09-11T12:37:39.879926Z", - "iopub.status.idle": "2023-09-11T12:37:39.881962Z", - "shell.execute_reply": "2023-09-11T12:37:39.881833Z" + "iopub.execute_input": "2023-09-11T17:19:38.320318Z", + "iopub.status.busy": "2023-09-11T17:19:38.320231Z", + "iopub.status.idle": "2023-09-11T17:19:38.322694Z", + "shell.execute_reply": "2023-09-11T17:19:38.322486Z" }, "papermill": { - "duration": 0.004934, - "end_time": "2023-09-11T12:37:39.882752", + "duration": 0.004978, + "end_time": "2023-09-11T17:19:38.323332", "exception": false, - "start_time": "2023-09-11T12:37:39.877818", + "start_time": "2023-09-11T17:19:38.318354", "status": "completed" }, "tags": [] @@ -628,16 +628,16 @@ "id": "8e863af1-aa4e-4302-9093-1729797af641", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:37:39.888550Z", - "iopub.status.busy": "2023-09-11T12:37:39.888502Z", - "iopub.status.idle": "2023-09-11T12:37:39.892186Z", - "shell.execute_reply": "2023-09-11T12:37:39.892060Z" + "iopub.execute_input": "2023-09-11T17:19:38.327062Z", + "iopub.status.busy": "2023-09-11T17:19:38.326945Z", + "iopub.status.idle": "2023-09-11T17:19:38.331168Z", + "shell.execute_reply": "2023-09-11T17:19:38.331034Z" }, "papermill": { - "duration": 0.007402, - "end_time": "2023-09-11T12:37:39.892923", + "duration": 0.006805, + "end_time": "2023-09-11T17:19:38.331662", "exception": false, - "start_time": "2023-09-11T12:37:39.885521", + "start_time": "2023-09-11T17:19:38.324857", "status": "completed" }, "tags": [] @@ -675,36 +675,36 @@ " 0\n", " 100\n", " p-1\n", - " 0.000090\n", - " -0.002969\n", + " 0.000096\n", + " -0.014058\n", " \n", " \n", " 1\n", " 100\n", " p-1\n", - " 0.000064\n", - " 0.043811\n", + " 0.000038\n", + " -0.019761\n", " \n", " \n", " 2\n", " 100\n", " p-1\n", - " 0.000031\n", - " 0.001142\n", + " 0.000041\n", + " -0.274154\n", " \n", " \n", " 3\n", " 100\n", " p-1\n", - " 0.000085\n", - " 0.258282\n", + " 0.000034\n", + " -0.071098\n", " \n", " \n", " 4\n", " 100\n", " p-1\n", - " 0.000031\n", - " 0.041389\n", + " 0.000032\n", + " 0.016989\n", " \n", " \n", "\n", @@ -712,11 +712,11 @@ ], "text/plain": [ " data_size method time sim\n", - "0 100 p-1 0.000090 -0.002969\n", - "1 100 p-1 0.000064 0.043811\n", - "2 100 p-1 0.000031 0.001142\n", - "3 100 p-1 0.000085 0.258282\n", - "4 100 p-1 0.000031 0.041389" + "0 100 p-1 0.000096 -0.014058\n", + "1 100 p-1 0.000038 -0.019761\n", + "2 100 p-1 0.000041 -0.274154\n", + "3 100 p-1 0.000034 -0.071098\n", + "4 100 p-1 0.000032 0.016989" ] }, "execution_count": 12, @@ -734,10 +734,10 @@ "id": "2d4ad769-2693-4e0b-829b-b9dfadf8d4da", "metadata": { "papermill": { - "duration": 0.002511, - "end_time": "2023-09-11T12:37:39.898318", + "duration": 0.001477, + "end_time": "2023-09-11T17:19:38.334703", "exception": false, - "start_time": "2023-09-11T12:37:39.895807", + "start_time": "2023-09-11T17:19:38.333226", "status": "completed" }, "tags": [] @@ -770,8 +770,8 @@ }, "papermill": { "default_parameters": {}, - "duration": 547.279037, - "end_time": "2023-09-11T12:37:40.114643", + "duration": 570.345768, + "end_time": "2023-09-11T17:19:38.553460", "environment_variables": {}, "exception": null, "input_path": "nbs/20_comparison_others/60-time_test-1_cpu_core.ipynb", @@ -779,7 +779,7 @@ "parameters": { "GTEX_TISSUE": "skin_sun_exposed_lower_leg" }, - "start_time": "2023-09-11T12:28:32.835606", + "start_time": "2023-09-11T17:10:08.207692", "version": "2.3.3" }, "toc-autonumbering": true diff --git a/nbs/20_comparison_others/61-time_test-3_cpu_cores.ipynb b/nbs/20_comparison_others/61-time_test-3_cpu_cores.ipynb index 33811ae3..9c10b55e 100644 --- a/nbs/20_comparison_others/61-time_test-3_cpu_cores.ipynb +++ b/nbs/20_comparison_others/61-time_test-3_cpu_cores.ipynb @@ -5,10 +5,10 @@ "id": "c1948eb4-fb63-4fad-8243-bcd57b98def3", "metadata": { "papermill": { - "duration": 0.007283, - "end_time": "2023-09-11T12:42:47.087604", + "duration": 0.005132, + "end_time": "2023-09-11T17:19:40.195247", "exception": false, - "start_time": "2023-09-11T12:42:47.080321", + "start_time": "2023-09-11T17:19:40.190115", "status": "completed" }, "tags": [] @@ -22,10 +22,10 @@ "id": "d5775ff8-d222-4fbc-b1a9-8366662c2896", "metadata": { "papermill": { - "duration": 0.005516, - "end_time": "2023-09-11T12:42:47.102979", + "duration": 0.004162, + "end_time": "2023-09-11T17:19:40.205116", "exception": false, - "start_time": "2023-09-11T12:42:47.097463", + "start_time": "2023-09-11T17:19:40.200954", "status": "completed" }, "tags": [] @@ -41,10 +41,10 @@ "id": "d6252b4c-6d56-4cf0-aecd-5d8c769a3609", "metadata": { "papermill": { - "duration": 0.002663, - "end_time": "2023-09-11T12:42:47.111751", + "duration": 0.001873, + "end_time": "2023-09-11T17:19:40.211060", "exception": false, - "start_time": "2023-09-11T12:42:47.109088", + "start_time": "2023-09-11T17:19:40.209187", "status": "completed" }, "tags": [] @@ -59,16 +59,16 @@ "id": "ea7d30d5-da78-454a-a4ff-e6307bfdbb37", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:42:47.116278Z", - "iopub.status.busy": "2023-09-11T12:42:47.116101Z", - "iopub.status.idle": "2023-09-11T12:42:47.121262Z", - "shell.execute_reply": "2023-09-11T12:42:47.121055Z" + "iopub.execute_input": "2023-09-11T17:19:40.215175Z", + "iopub.status.busy": "2023-09-11T17:19:40.215023Z", + "iopub.status.idle": "2023-09-11T17:19:40.220124Z", + "shell.execute_reply": "2023-09-11T17:19:40.219849Z" }, "papermill": { - "duration": 0.007987, - "end_time": "2023-09-11T12:42:47.121897", + "duration": 0.008059, + "end_time": "2023-09-11T17:19:40.220813", "exception": false, - "start_time": "2023-09-11T12:42:47.113910", + "start_time": "2023-09-11T17:19:40.212754", "status": "completed" }, "tags": [] @@ -102,16 +102,16 @@ "id": "77e9d29d-5307-4b4a-b103-7d1fbd6a7e56", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:42:47.125237Z", - "iopub.status.busy": "2023-09-11T12:42:47.125151Z", - "iopub.status.idle": "2023-09-11T12:42:47.436016Z", - "shell.execute_reply": "2023-09-11T12:42:47.435737Z" + "iopub.execute_input": "2023-09-11T17:19:40.224008Z", + "iopub.status.busy": "2023-09-11T17:19:40.223874Z", + "iopub.status.idle": "2023-09-11T17:19:40.527762Z", + "shell.execute_reply": "2023-09-11T17:19:40.527504Z" }, "papermill": { - "duration": 0.313333, - "end_time": "2023-09-11T12:42:47.436883", + "duration": 0.30608, + "end_time": "2023-09-11T17:19:40.528491", "exception": false, - "start_time": "2023-09-11T12:42:47.123550", + "start_time": "2023-09-11T17:19:40.222411", "status": "completed" }, "tags": [] @@ -135,10 +135,10 @@ "id": "262eb535-3e5d-43d7-9efd-bd6cfdac9190", "metadata": { "papermill": { - "duration": 0.009685, - "end_time": "2023-09-11T12:42:47.448060", + "duration": 0.00123, + "end_time": "2023-09-11T17:19:40.531231", "exception": false, - "start_time": "2023-09-11T12:42:47.438375", + "start_time": "2023-09-11T17:19:40.530001", "status": "completed" }, "tags": [] @@ -153,16 +153,16 @@ "id": "c1e5a523-e22f-42e1-b27f-929bebaea711", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:42:47.454474Z", - "iopub.status.busy": "2023-09-11T12:42:47.454405Z", - "iopub.status.idle": "2023-09-11T12:42:47.458187Z", - "shell.execute_reply": "2023-09-11T12:42:47.458035Z" + "iopub.execute_input": "2023-09-11T17:19:40.536693Z", + "iopub.status.busy": "2023-09-11T17:19:40.536580Z", + "iopub.status.idle": "2023-09-11T17:19:40.540258Z", + "shell.execute_reply": "2023-09-11T17:19:40.540125Z" }, "papermill": { - "duration": 0.006239, - "end_time": "2023-09-11T12:42:47.458734", + "duration": 0.007326, + "end_time": "2023-09-11T17:19:40.540700", "exception": false, - "start_time": "2023-09-11T12:42:47.452495", + "start_time": "2023-09-11T17:19:40.533374", "status": "completed" }, "tags": [] @@ -189,16 +189,16 @@ "id": "311cc161-c947-447e-b622-6df22ad0e03a", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:42:47.461588Z", - "iopub.status.busy": "2023-09-11T12:42:47.461493Z", - "iopub.status.idle": "2023-09-11T12:42:47.462765Z", - "shell.execute_reply": "2023-09-11T12:42:47.462626Z" + "iopub.execute_input": "2023-09-11T17:19:40.543522Z", + "iopub.status.busy": "2023-09-11T17:19:40.543413Z", + "iopub.status.idle": "2023-09-11T17:19:40.544690Z", + "shell.execute_reply": "2023-09-11T17:19:40.544557Z" }, "papermill": { - "duration": 0.0032, - "end_time": "2023-09-11T12:42:47.463206", + "duration": 0.003165, + "end_time": "2023-09-11T17:19:40.545124", "exception": false, - "start_time": "2023-09-11T12:42:47.460006", + "start_time": "2023-09-11T17:19:40.541959", "status": "completed" }, "tags": [] @@ -214,16 +214,16 @@ "id": "2d2bd14a-a40d-4710-98c2-587148d0f532", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:42:47.465927Z", - "iopub.status.busy": "2023-09-11T12:42:47.465842Z", - "iopub.status.idle": "2023-09-11T12:42:47.467161Z", - "shell.execute_reply": "2023-09-11T12:42:47.467027Z" + "iopub.execute_input": "2023-09-11T17:19:40.547868Z", + "iopub.status.busy": "2023-09-11T17:19:40.547782Z", + "iopub.status.idle": "2023-09-11T17:19:40.549084Z", + "shell.execute_reply": "2023-09-11T17:19:40.548956Z" }, "papermill": { - "duration": 0.003156, - "end_time": "2023-09-11T12:42:47.467612", + "duration": 0.003145, + "end_time": "2023-09-11T17:19:40.549513", "exception": false, - "start_time": "2023-09-11T12:42:47.464456", + "start_time": "2023-09-11T17:19:40.546368", "status": "completed" }, "tags": [] @@ -251,16 +251,16 @@ "id": "d24607ca-2774-4e51-b392-538c85c86052", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:42:47.470402Z", - "iopub.status.busy": "2023-09-11T12:42:47.470318Z", - "iopub.status.idle": "2023-09-11T12:42:47.471538Z", - "shell.execute_reply": "2023-09-11T12:42:47.471410Z" + "iopub.execute_input": "2023-09-11T17:19:40.552261Z", + "iopub.status.busy": "2023-09-11T17:19:40.552148Z", + "iopub.status.idle": "2023-09-11T17:19:40.553377Z", + "shell.execute_reply": "2023-09-11T17:19:40.553248Z" }, "papermill": { - "duration": 0.003057, - "end_time": "2023-09-11T12:42:47.471928", + "duration": 0.003082, + "end_time": "2023-09-11T17:19:40.553807", "exception": false, - "start_time": "2023-09-11T12:42:47.468871", + "start_time": "2023-09-11T17:19:40.550725", "status": "completed" }, "tags": [] @@ -275,10 +275,10 @@ "id": "eec9bb6d-4ba2-4817-9b36-bc8e68a2beae", "metadata": { "papermill": { - "duration": 0.001154, - "end_time": "2023-09-11T12:42:47.474318", + "duration": 0.001253, + "end_time": "2023-09-11T17:19:40.556278", "exception": false, - "start_time": "2023-09-11T12:42:47.473164", + "start_time": "2023-09-11T17:19:40.555025", "status": "completed" }, "tags": [] @@ -293,16 +293,16 @@ "id": "ee9e6b70-9abd-4714-9ce5-49c270a1b0be", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:42:47.477201Z", - "iopub.status.busy": "2023-09-11T12:42:47.477106Z", - "iopub.status.idle": "2023-09-11T12:42:47.478760Z", - "shell.execute_reply": "2023-09-11T12:42:47.478637Z" + "iopub.execute_input": "2023-09-11T17:19:40.559026Z", + "iopub.status.busy": "2023-09-11T17:19:40.558936Z", + "iopub.status.idle": "2023-09-11T17:19:40.560525Z", + "shell.execute_reply": "2023-09-11T17:19:40.560406Z" }, "papermill": { - "duration": 0.003614, - "end_time": "2023-09-11T12:42:47.479235", + "duration": 0.003413, + "end_time": "2023-09-11T17:19:40.560946", "exception": false, - "start_time": "2023-09-11T12:42:47.475621", + "start_time": "2023-09-11T17:19:40.557533", "status": "completed" }, "tags": [] @@ -329,10 +329,10 @@ "id": "9cb68076-f855-45c8-9843-ce2d93bdc4e7", "metadata": { "papermill": { - "duration": 0.001143, - "end_time": "2023-09-11T12:42:47.481614", + "duration": 0.001182, + "end_time": "2023-09-11T17:19:40.563389", "exception": false, - "start_time": "2023-09-11T12:42:47.480471", + "start_time": "2023-09-11T17:19:40.562207", "status": "completed" }, "tags": [] @@ -347,16 +347,16 @@ "id": "e2c25ecf-0d87-4447-a9ff-a3b511629a61", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:42:47.484374Z", - "iopub.status.busy": "2023-09-11T12:42:47.484269Z", - "iopub.status.idle": "2023-09-11T12:42:47.486135Z", - "shell.execute_reply": "2023-09-11T12:42:47.485996Z" + "iopub.execute_input": "2023-09-11T17:19:40.566243Z", + "iopub.status.busy": "2023-09-11T17:19:40.566129Z", + "iopub.status.idle": "2023-09-11T17:19:40.567956Z", + "shell.execute_reply": "2023-09-11T17:19:40.567815Z" }, "papermill": { - "duration": 0.003696, - "end_time": "2023-09-11T12:42:47.486536", + "duration": 0.00373, + "end_time": "2023-09-11T17:19:40.568357", "exception": false, - "start_time": "2023-09-11T12:42:47.482840", + "start_time": "2023-09-11T17:19:40.564627", "status": "completed" }, "tags": [] @@ -373,16 +373,16 @@ "id": "3ab0c0b8-f340-4cb4-a236-6523cf0aa995", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:42:47.489320Z", - "iopub.status.busy": "2023-09-11T12:42:47.489232Z", - "iopub.status.idle": "2023-09-11T12:42:47.491185Z", - "shell.execute_reply": "2023-09-11T12:42:47.490897Z" + "iopub.execute_input": "2023-09-11T17:19:40.571175Z", + "iopub.status.busy": "2023-09-11T17:19:40.571059Z", + "iopub.status.idle": "2023-09-11T17:19:40.572615Z", + "shell.execute_reply": "2023-09-11T17:19:40.572491Z" }, "papermill": { - "duration": 0.003764, - "end_time": "2023-09-11T12:42:47.491613", + "duration": 0.003383, + "end_time": "2023-09-11T17:19:40.573042", "exception": false, - "start_time": "2023-09-11T12:42:47.487849", + "start_time": "2023-09-11T17:19:40.569659", "status": "completed" }, "tags": [] @@ -391,7 +391,7 @@ { "data": { "text/plain": [ - "(10630, 4)" + "(5330, 4)" ] }, "execution_count": 9, @@ -409,16 +409,16 @@ "id": "ffc0d8de-79a6-4ab8-8cc2-5b720392a0e6", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:42:47.494496Z", - "iopub.status.busy": "2023-09-11T12:42:47.494315Z", - "iopub.status.idle": "2023-09-11T12:42:47.496241Z", - "shell.execute_reply": "2023-09-11T12:42:47.496114Z" + "iopub.execute_input": "2023-09-11T17:19:40.575866Z", + "iopub.status.busy": "2023-09-11T17:19:40.575760Z", + "iopub.status.idle": "2023-09-11T17:19:40.577587Z", + "shell.execute_reply": "2023-09-11T17:19:40.577457Z" }, "papermill": { - "duration": 0.003729, - "end_time": "2023-09-11T12:42:47.496614", + "duration": 0.003636, + "end_time": "2023-09-11T17:19:40.577969", "exception": false, - "start_time": "2023-09-11T12:42:47.492885", + "start_time": "2023-09-11T17:19:40.574333", "status": "completed" }, "tags": [] @@ -448,10 +448,10 @@ "id": "7656ffa5-c9c3-41f5-aeb5-21bde8836e81", "metadata": { "papermill": { - "duration": 0.001222, - "end_time": "2023-09-11T12:42:47.499097", + "duration": 0.001204, + "end_time": "2023-09-11T17:19:40.580487", "exception": false, - "start_time": "2023-09-11T12:42:47.497875", + "start_time": "2023-09-11T17:19:40.579283", "status": "completed" }, "tags": [] @@ -466,16 +466,16 @@ "id": "3e4fac59-34a7-4b63-8aa0-e845c0a156af", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:42:47.501825Z", - "iopub.status.busy": "2023-09-11T12:42:47.501719Z", - "iopub.status.idle": "2023-09-11T12:42:47.504178Z", - "shell.execute_reply": "2023-09-11T12:42:47.504049Z" + "iopub.execute_input": "2023-09-11T17:19:40.583250Z", + "iopub.status.busy": "2023-09-11T17:19:40.583137Z", + "iopub.status.idle": "2023-09-11T17:19:40.585602Z", + "shell.execute_reply": "2023-09-11T17:19:40.585477Z" }, "papermill": { - "duration": 0.00423, - "end_time": "2023-09-11T12:42:47.504560", + "duration": 0.004268, + "end_time": "2023-09-11T17:19:40.586007", "exception": false, - "start_time": "2023-09-11T12:42:47.500330", + "start_time": "2023-09-11T17:19:40.581739", "status": "completed" }, "tags": [] @@ -503,16 +503,16 @@ "id": "e3f26ab4-ec59-4e8f-b887-14abc2cc7cae", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:42:47.507537Z", - "iopub.status.busy": "2023-09-11T12:42:47.507468Z", - "iopub.status.idle": "2023-09-11T12:48:40.584287Z", - "shell.execute_reply": "2023-09-11T12:48:40.584048Z" + "iopub.execute_input": "2023-09-11T17:19:40.589158Z", + "iopub.status.busy": "2023-09-11T17:19:40.589057Z", + "iopub.status.idle": "2023-09-11T17:24:30.286608Z", + "shell.execute_reply": "2023-09-11T17:24:30.286217Z" }, "papermill": { - "duration": 353.080512, - "end_time": "2023-09-11T12:48:40.586483", + "duration": 289.701093, + "end_time": "2023-09-11T17:24:30.288465", "exception": false, - "start_time": "2023-09-11T12:42:47.505971", + "start_time": "2023-09-11T17:19:40.587372", "status": "completed" }, "tags": [] @@ -631,10 +631,10 @@ "id": "2597ad7d", "metadata": { "papermill": { - "duration": 0.002766, - "end_time": "2023-09-11T12:48:40.595863", + "duration": 0.00156, + "end_time": "2023-09-11T17:24:30.296478", "exception": false, - "start_time": "2023-09-11T12:48:40.593097", + "start_time": "2023-09-11T17:24:30.294918", "status": "completed" }, "tags": [] @@ -649,16 +649,16 @@ "id": "3846d7bb-67fe-4084-afe8-d7226fbef0f1", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:48:40.599650Z", - "iopub.status.busy": "2023-09-11T12:48:40.599583Z", - "iopub.status.idle": "2023-09-11T12:48:40.601826Z", - "shell.execute_reply": "2023-09-11T12:48:40.601609Z" + "iopub.execute_input": "2023-09-11T17:24:30.300328Z", + "iopub.status.busy": "2023-09-11T17:24:30.300150Z", + "iopub.status.idle": "2023-09-11T17:24:30.302196Z", + "shell.execute_reply": "2023-09-11T17:24:30.302067Z" }, "papermill": { - "duration": 0.004895, - "end_time": "2023-09-11T12:48:40.602366", + "duration": 0.004568, + "end_time": "2023-09-11T17:24:30.302661", "exception": false, - "start_time": "2023-09-11T12:48:40.597471", + "start_time": "2023-09-11T17:24:30.298093", "status": "completed" }, "tags": [] @@ -667,7 +667,7 @@ { "data": { "text/plain": [ - "(15960, 4)" + "(10660, 4)" ] }, "execution_count": 13, @@ -685,16 +685,16 @@ "id": "1e699725", "metadata": { "execution": { - "iopub.execute_input": "2023-09-11T12:48:40.605896Z", - "iopub.status.busy": "2023-09-11T12:48:40.605849Z", - "iopub.status.idle": "2023-09-11T12:48:40.609591Z", - "shell.execute_reply": "2023-09-11T12:48:40.609449Z" + "iopub.execute_input": "2023-09-11T17:24:30.306196Z", + "iopub.status.busy": "2023-09-11T17:24:30.306085Z", + "iopub.status.idle": "2023-09-11T17:24:30.309573Z", + "shell.execute_reply": "2023-09-11T17:24:30.309446Z" }, "papermill": { - "duration": 0.006123, - "end_time": "2023-09-11T12:48:40.610066", + "duration": 0.005669, + "end_time": "2023-09-11T17:24:30.309968", "exception": false, - "start_time": "2023-09-11T12:48:40.603943", + "start_time": "2023-09-11T17:24:30.304299", "status": "completed" }, "tags": [] @@ -732,36 +732,36 @@ " 0\n", " 100\n", " p-1\n", - " 0.000090\n", - " -0.002969\n", + " 0.000096\n", + " -0.014058\n", " \n", " \n", " 1\n", " 100\n", " p-1\n", - " 0.000064\n", - " 0.043811\n", + " 0.000038\n", + " -0.019761\n", " \n", " \n", " 2\n", " 100\n", " p-1\n", - " 0.000031\n", - " 0.001142\n", + " 0.000041\n", + " -0.274154\n", " \n", " \n", " 3\n", " 100\n", " p-1\n", - " 0.000085\n", - " 0.258282\n", + " 0.000034\n", + " -0.071098\n", " \n", " \n", " 4\n", " 100\n", " p-1\n", - " 0.000031\n", - " 0.041389\n", + " 0.000032\n", + " 0.016989\n", " \n", " \n", "\n", @@ -769,11 +769,11 @@ ], "text/plain": [ " data_size method time sim\n", - "0 100 p-1 0.000090 -0.002969\n", - "1 100 p-1 0.000064 0.043811\n", - "2 100 p-1 0.000031 0.001142\n", - "3 100 p-1 0.000085 0.258282\n", - "4 100 p-1 0.000031 0.041389" + "0 100 p-1 0.000096 -0.014058\n", + "1 100 p-1 0.000038 -0.019761\n", + "2 100 p-1 0.000041 -0.274154\n", + "3 100 p-1 0.000034 -0.071098\n", + "4 100 p-1 0.000032 0.016989" ] }, "execution_count": 14, @@ -791,10 +791,10 @@ "id": "9c772e87-9117-4f2e-b217-29ea33c64d12", "metadata": { "papermill": { - "duration": 0.00157, - "end_time": "2023-09-11T12:48:40.613333", + "duration": 0.00156, + "end_time": "2023-09-11T17:24:30.313179", "exception": false, - "start_time": "2023-09-11T12:48:40.611763", + "start_time": "2023-09-11T17:24:30.311619", "status": "completed" }, "tags": [] @@ -827,8 +827,8 @@ }, "papermill": { "default_parameters": {}, - "duration": 354.416187, - "end_time": "2023-09-11T12:48:40.832374", + "duration": 291.152266, + "end_time": "2023-09-11T17:24:30.532162", "environment_variables": {}, "exception": null, "input_path": "nbs/20_comparison_others/61-time_test-3_cpu_cores.ipynb", @@ -836,7 +836,7 @@ "parameters": { "GTEX_TISSUE": "skin_sun_exposed_lower_leg" }, - "start_time": "2023-09-11T12:42:46.416187", + "start_time": "2023-09-11T17:19:39.379896", "version": "2.3.3" } }, diff --git a/nbs/20_comparison_others/62-time_test-6_cpu_cores.ipynb b/nbs/20_comparison_others/62-time_test-6_cpu_cores.ipynb new file mode 100644 index 00000000..cab7bb8e --- /dev/null +++ b/nbs/20_comparison_others/62-time_test-6_cpu_cores.ipynb @@ -0,0 +1,850 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "c1948eb4-fb63-4fad-8243-bcd57b98def3", + "metadata": { + "papermill": { + "duration": 0.006809, + "end_time": "2023-09-11T17:24:32.012248", + "exception": false, + "start_time": "2023-09-11T17:24:32.005439", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Description" + ] + }, + { + "cell_type": "markdown", + "id": "d5775ff8-d222-4fbc-b1a9-8366662c2896", + "metadata": { + "papermill": { + "duration": 0.005703, + "end_time": "2023-09-11T17:24:32.028308", + "exception": false, + "start_time": "2023-09-11T17:24:32.022605", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "It generates random variables of varying sizes to compare the time taken by CCC and MIC.\n", + "\n", + "This notebook uses 6 CPU core." + ] + }, + { + "cell_type": "markdown", + "id": "d6252b4c-6d56-4cf0-aecd-5d8c769a3609", + "metadata": { + "papermill": { + "duration": 0.001332, + "end_time": "2023-09-11T17:24:32.032045", + "exception": false, + "start_time": "2023-09-11T17:24:32.030713", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Modules loading" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "ea7d30d5-da78-454a-a4ff-e6307bfdbb37", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-11T17:24:32.037911Z", + "iopub.status.busy": "2023-09-11T17:24:32.037787Z", + "iopub.status.idle": "2023-09-11T17:24:32.044887Z", + "shell.execute_reply": "2023-09-11T17:24:32.044454Z" + }, + "papermill": { + "duration": 0.011505, + "end_time": "2023-09-11T17:24:32.045870", + "exception": false, + "start_time": "2023-09-11T17:24:32.034365", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env: CM_N_JOBS=6\n", + "env: NUMBA_NUM_THREADS=6\n", + "env: MKL_NUM_THREADS=6\n", + "env: OPEN_BLAS_NUM_THREADS=6\n", + "env: NUMEXPR_NUM_THREADS=6\n", + "env: OMP_NUM_THREADS=6\n" + ] + } + ], + "source": [ + "%env CM_N_JOBS=6\n", + "%env NUMBA_NUM_THREADS=6\n", + "%env MKL_NUM_THREADS=6\n", + "%env OPEN_BLAS_NUM_THREADS=6\n", + "%env NUMEXPR_NUM_THREADS=6\n", + "%env OMP_NUM_THREADS=6" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "77e9d29d-5307-4b4a-b103-7d1fbd6a7e56", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-11T17:24:32.050269Z", + "iopub.status.busy": "2023-09-11T17:24:32.050191Z", + "iopub.status.idle": "2023-09-11T17:24:32.357100Z", + "shell.execute_reply": "2023-09-11T17:24:32.356917Z" + }, + "papermill": { + "duration": 0.309912, + "end_time": "2023-09-11T17:24:32.358023", + "exception": false, + "start_time": "2023-09-11T17:24:32.048111", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "from time import time\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "from scipy.stats import pearsonr, spearmanr\n", + "\n", + "from ccc import conf\n", + "from ccc.coef import ccc\n", + "from ccc.methods import mic" + ] + }, + { + "cell_type": "markdown", + "id": "262eb535-3e5d-43d7-9efd-bd6cfdac9190", + "metadata": { + "papermill": { + "duration": 0.009731, + "end_time": "2023-09-11T17:24:32.369260", + "exception": false, + "start_time": "2023-09-11T17:24:32.359529", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Settings" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "c1e5a523-e22f-42e1-b27f-929bebaea711", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-11T17:24:32.377212Z", + "iopub.status.busy": "2023-09-11T17:24:32.377134Z", + "iopub.status.idle": "2023-09-11T17:24:32.381696Z", + "shell.execute_reply": "2023-09-11T17:24:32.381531Z" + }, + "papermill": { + "duration": 0.008273, + "end_time": "2023-09-11T17:24:32.382278", + "exception": false, + "start_time": "2023-09-11T17:24:32.374005", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "6" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "N_JOBS = int(os.environ[\"CM_N_JOBS\"])\n", + "display(N_JOBS)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "311cc161-c947-447e-b622-6df22ad0e03a", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-11T17:24:32.385464Z", + "iopub.status.busy": "2023-09-11T17:24:32.385353Z", + "iopub.status.idle": "2023-09-11T17:24:32.386610Z", + "shell.execute_reply": "2023-09-11T17:24:32.386480Z" + }, + "papermill": { + "duration": 0.003324, + "end_time": "2023-09-11T17:24:32.387117", + "exception": false, + "start_time": "2023-09-11T17:24:32.383793", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "OUTPUT_FILENAME = \"time_test.pkl\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "2d2bd14a-a40d-4710-98c2-587148d0f532", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-11T17:24:32.389868Z", + "iopub.status.busy": "2023-09-11T17:24:32.389753Z", + "iopub.status.idle": "2023-09-11T17:24:32.391064Z", + "shell.execute_reply": "2023-09-11T17:24:32.390936Z" + }, + "papermill": { + "duration": 0.003186, + "end_time": "2023-09-11T17:24:32.391566", + "exception": false, + "start_time": "2023-09-11T17:24:32.388380", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "DATA_SIZES = [\n", + " 100,\n", + " 500,\n", + " 1000,\n", + " 5000,\n", + " 10000,\n", + " 50000,\n", + " 100000,\n", + " 1000000,\n", + " 10000000,\n", + "]\n", + "\n", + "N_REPS = 10" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "d24607ca-2774-4e51-b392-538c85c86052", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-11T17:24:32.394450Z", + "iopub.status.busy": "2023-09-11T17:24:32.394355Z", + "iopub.status.idle": "2023-09-11T17:24:32.396007Z", + "shell.execute_reply": "2023-09-11T17:24:32.395863Z" + }, + "papermill": { + "duration": 0.00377, + "end_time": "2023-09-11T17:24:32.396573", + "exception": false, + "start_time": "2023-09-11T17:24:32.392803", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "np.random.seed(0)" + ] + }, + { + "cell_type": "markdown", + "id": "eec9bb6d-4ba2-4817-9b36-bc8e68a2beae", + "metadata": { + "papermill": { + "duration": 0.001317, + "end_time": "2023-09-11T17:24:32.399156", + "exception": false, + "start_time": "2023-09-11T17:24:32.397839", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Paths" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "ee9e6b70-9abd-4714-9ce5-49c270a1b0be", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-11T17:24:32.402047Z", + "iopub.status.busy": "2023-09-11T17:24:32.401952Z", + "iopub.status.idle": "2023-09-11T17:24:32.403654Z", + "shell.execute_reply": "2023-09-11T17:24:32.403506Z" + }, + "papermill": { + "duration": 0.00372, + "end_time": "2023-09-11T17:24:32.404189", + "exception": false, + "start_time": "2023-09-11T17:24:32.400469", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "PosixPath('/opt/data/results/time_test')" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "OUTPUT_DIR = conf.RESULTS_DIR / \"time_test\"\n", + "OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n", + "display(OUTPUT_DIR)" + ] + }, + { + "cell_type": "markdown", + "id": "9cb68076-f855-45c8-9843-ce2d93bdc4e7", + "metadata": { + "papermill": { + "duration": 0.001482, + "end_time": "2023-09-11T17:24:32.407109", + "exception": false, + "start_time": "2023-09-11T17:24:32.405627", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Functions" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "e2c25ecf-0d87-4447-a9ff-a3b511629a61", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-11T17:24:32.410432Z", + "iopub.status.busy": "2023-09-11T17:24:32.410327Z", + "iopub.status.idle": "2023-09-11T17:24:32.412244Z", + "shell.execute_reply": "2023-09-11T17:24:32.412105Z" + }, + "papermill": { + "duration": 0.003992, + "end_time": "2023-09-11T17:24:32.412675", + "exception": false, + "start_time": "2023-09-11T17:24:32.408683", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# append to previous run\n", + "time_results = pd.read_pickle(OUTPUT_DIR / OUTPUT_FILENAME)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "3ab0c0b8-f340-4cb4-a236-6523cf0aa995", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-11T17:24:32.415524Z", + "iopub.status.busy": "2023-09-11T17:24:32.415437Z", + "iopub.status.idle": "2023-09-11T17:24:32.416932Z", + "shell.execute_reply": "2023-09-11T17:24:32.416810Z" + }, + "papermill": { + "duration": 0.003391, + "end_time": "2023-09-11T17:24:32.417366", + "exception": false, + "start_time": "2023-09-11T17:24:32.413975", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(10660, 4)" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "time_results.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "ffc0d8de-79a6-4ab8-8cc2-5b720392a0e6", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-11T17:24:32.420188Z", + "iopub.status.busy": "2023-09-11T17:24:32.420104Z", + "iopub.status.idle": "2023-09-11T17:24:32.421911Z", + "shell.execute_reply": "2023-09-11T17:24:32.421774Z" + }, + "papermill": { + "duration": 0.003688, + "end_time": "2023-09-11T17:24:32.422337", + "exception": false, + "start_time": "2023-09-11T17:24:32.418649", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def run_method(func, method_name, size):\n", + " n_reps = N_REPS\n", + " if size < 500:\n", + " n_reps = 1000\n", + "\n", + " for r in range(n_reps):\n", + " d1 = np.random.rand(size)\n", + " d2 = np.random.rand(size)\n", + "\n", + " start_time = time()\n", + " sim = func(d1, d2)\n", + " end_time = time()\n", + " met_time = end_time - start_time\n", + "\n", + " idx = time_results.shape[0]\n", + " time_results.loc[idx] = [d1.shape[0], method_name, met_time, sim]" + ] + }, + { + "cell_type": "markdown", + "id": "7656ffa5-c9c3-41f5-aeb5-21bde8836e81", + "metadata": { + "papermill": { + "duration": 0.00124, + "end_time": "2023-09-11T17:24:32.424856", + "exception": false, + "start_time": "2023-09-11T17:24:32.423616", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Run" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "3e4fac59-34a7-4b63-8aa0-e845c0a156af", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-11T17:24:32.427658Z", + "iopub.status.busy": "2023-09-11T17:24:32.427556Z", + "iopub.status.idle": "2023-09-11T17:24:32.429896Z", + "shell.execute_reply": "2023-09-11T17:24:32.429773Z" + }, + "papermill": { + "duration": 0.004216, + "end_time": "2023-09-11T17:24:32.430329", + "exception": false, + "start_time": "2023-09-11T17:24:32.426113", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "0.02047038824894002" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# initialize methods\n", + "ccc(np.random.rand(100), np.random.rand(100))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "e3f26ab4-ec59-4e8f-b887-14abc2cc7cae", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-11T17:24:32.433310Z", + "iopub.status.busy": "2023-09-11T17:24:32.433227Z", + "iopub.status.idle": "2023-09-11T17:28:12.186209Z", + "shell.execute_reply": "2023-09-11T17:28:12.185906Z" + }, + "papermill": { + "duration": 219.757231, + "end_time": "2023-09-11T17:28:12.188930", + "exception": false, + "start_time": "2023-09-11T17:24:32.431699", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Size: 100\n", + " p\n", + " s\n", + " cm\n", + " mic_e\n", + " mic\n", + "Saving to pickle\n", + "\n", + "\n", + "Size: 500\n", + " p\n", + " s\n", + " cm\n", + " mic_e\n", + " mic\n", + "Saving to pickle\n", + "\n", + "\n", + "Size: 1000\n", + " p\n", + " s\n", + " cm\n", + " mic_e\n", + " mic\n", + "Saving to pickle\n", + "\n", + "\n", + "Size: 5000\n", + " p\n", + " s\n", + " cm\n", + " mic_e\n", + " mic\n", + "Saving to pickle\n", + "\n", + "\n", + "Size: 10000\n", + " p\n", + " s\n", + " cm\n", + " mic_e\n", + " mic\n", + "Saving to pickle\n", + "\n", + "\n", + "Size: 50000\n", + " p\n", + " s\n", + " cm\n", + " mic_e\n", + "Saving to pickle\n", + "\n", + "\n", + "Size: 100000\n", + " p\n", + " s\n", + " cm\n", + "Saving to pickle\n", + "\n", + "\n", + "Size: 1000000\n", + " p\n", + " s\n", + " cm\n", + "Saving to pickle\n", + "\n", + "\n", + "Size: 10000000\n", + " p\n", + " s\n", + " cm\n", + "Saving to pickle\n", + "\n", + "\n" + ] + } + ], + "source": [ + "for s in DATA_SIZES:\n", + " print(f\"Size: {s}\")\n", + "\n", + " print(\" p\")\n", + " run_method(lambda x, y: pearsonr(x, y)[0], \"p-6\", s)\n", + "\n", + " print(\" s\")\n", + " run_method(lambda x, y: spearmanr(x, y)[0], \"s-6\", s)\n", + "\n", + " print(\" cm\")\n", + " run_method(lambda x, y: ccc(x, y, n_jobs=N_JOBS), \"cm-6\", s)\n", + "\n", + " if s <= 50000:\n", + " print(\" mic_e\")\n", + " run_method(lambda x, y: mic(x, y, estimator=\"mic_e\"), \"mic_e-6\", s)\n", + "\n", + " if s <= 10000:\n", + " print(\" mic\")\n", + " run_method(lambda x, y: mic(x, y), \"mic-6\", s)\n", + "\n", + " print(\"Saving to pickle\")\n", + " time_results.to_pickle(OUTPUT_DIR / OUTPUT_FILENAME)\n", + "\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "markdown", + "id": "2597ad7d", + "metadata": { + "papermill": { + "duration": 0.003281, + "end_time": "2023-09-11T17:28:12.194657", + "exception": false, + "start_time": "2023-09-11T17:28:12.191376", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Summary of results" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "3846d7bb-67fe-4084-afe8-d7226fbef0f1", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-11T17:28:12.199062Z", + "iopub.status.busy": "2023-09-11T17:28:12.198985Z", + "iopub.status.idle": "2023-09-11T17:28:12.201229Z", + "shell.execute_reply": "2023-09-11T17:28:12.201085Z" + }, + "papermill": { + "duration": 0.005563, + "end_time": "2023-09-11T17:28:12.201822", + "exception": false, + "start_time": "2023-09-11T17:28:12.196259", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(15990, 4)" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "time_results.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "1e699725", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-11T17:28:12.206039Z", + "iopub.status.busy": "2023-09-11T17:28:12.205929Z", + "iopub.status.idle": "2023-09-11T17:28:12.209831Z", + "shell.execute_reply": "2023-09-11T17:28:12.209695Z" + }, + "papermill": { + "duration": 0.006257, + "end_time": "2023-09-11T17:28:12.210283", + "exception": false, + "start_time": "2023-09-11T17:28:12.204026", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
data_sizemethodtimesim
0100p-10.000096-0.014058
1100p-10.000038-0.019761
2100p-10.000041-0.274154
3100p-10.000034-0.071098
4100p-10.0000320.016989
\n", + "
" + ], + "text/plain": [ + " data_size method time sim\n", + "0 100 p-1 0.000096 -0.014058\n", + "1 100 p-1 0.000038 -0.019761\n", + "2 100 p-1 0.000041 -0.274154\n", + "3 100 p-1 0.000034 -0.071098\n", + "4 100 p-1 0.000032 0.016989" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "time_results.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c772e87-9117-4f2e-b217-29ea33c64d12", + "metadata": { + "papermill": { + "duration": 0.001618, + "end_time": "2023-09-11T17:28:12.213625", + "exception": false, + "start_time": "2023-09-11T17:28:12.212007", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "all,-execution,-papermill,-trusted", + "notebook_metadata_filter": "-jupytext.text_representation.jupytext_version", + "text_representation": { + "extension": ".py", + "format_name": "percent", + "format_version": "1.3" + } + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + }, + "papermill": { + "default_parameters": {}, + "duration": 221.092162, + "end_time": "2023-09-11T17:28:12.434543", + "environment_variables": {}, + "exception": null, + "input_path": "nbs/20_comparison_others/62-time_test-6_cpu_cores.ipynb", + "output_path": "nbs/20_comparison_others/62-time_test-6_cpu_cores.run.ipynb", + "parameters": { + "GTEX_TISSUE": "skin_sun_exposed_lower_leg" + }, + "start_time": "2023-09-11T17:24:31.342381", + "version": "2.3.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/nbs/20_comparison_others/py/62-time_test-6_cpu_cores.py b/nbs/20_comparison_others/py/62-time_test-6_cpu_cores.py new file mode 100644 index 00000000..f1442657 --- /dev/null +++ b/nbs/20_comparison_others/py/62-time_test-6_cpu_cores.py @@ -0,0 +1,155 @@ +# --- +# jupyter: +# jupytext: +# cell_metadata_filter: all,-execution,-papermill,-trusted +# notebook_metadata_filter: -jupytext.text_representation.jupytext_version +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# kernelspec: +# display_name: Python 3 (ipykernel) +# language: python +# name: python3 +# --- + +# %% [markdown] tags=[] +# # Description + +# %% [markdown] tags=[] +# It generates random variables of varying sizes to compare the time taken by CCC and MIC. +# +# This notebook uses 6 CPU core. + +# %% [markdown] tags=[] +# # Modules loading + +# %% tags=[] +# %env CM_N_JOBS=6 +# %env NUMBA_NUM_THREADS=6 +# %env MKL_NUM_THREADS=6 +# %env OPEN_BLAS_NUM_THREADS=6 +# %env NUMEXPR_NUM_THREADS=6 +# %env OMP_NUM_THREADS=6 + +# %% tags=[] +import os +from time import time + +import numpy as np +import pandas as pd +from scipy.stats import pearsonr, spearmanr + +from ccc import conf +from ccc.coef import ccc +from ccc.methods import mic + +# %% [markdown] tags=[] +# # Settings + +# %% tags=[] +N_JOBS = int(os.environ["CM_N_JOBS"]) +display(N_JOBS) + +# %% tags=[] +OUTPUT_FILENAME = "time_test.pkl" + +# %% tags=[] +DATA_SIZES = [ + 100, + 500, + 1000, + 5000, + 10000, + 50000, + 100000, + 1000000, + 10000000, +] + +N_REPS = 10 + +# %% tags=[] +np.random.seed(0) + +# %% [markdown] tags=[] +# # Paths + +# %% tags=[] +OUTPUT_DIR = conf.RESULTS_DIR / "time_test" +OUTPUT_DIR.mkdir(parents=True, exist_ok=True) +display(OUTPUT_DIR) + +# %% [markdown] tags=[] +# # Functions + +# %% tags=[] +# append to previous run +time_results = pd.read_pickle(OUTPUT_DIR / OUTPUT_FILENAME) + +# %% tags=[] +time_results.shape + + +# %% tags=[] +def run_method(func, method_name, size): + n_reps = N_REPS + if size < 500: + n_reps = 1000 + + for r in range(n_reps): + d1 = np.random.rand(size) + d2 = np.random.rand(size) + + start_time = time() + sim = func(d1, d2) + end_time = time() + met_time = end_time - start_time + + idx = time_results.shape[0] + time_results.loc[idx] = [d1.shape[0], method_name, met_time, sim] + + +# %% [markdown] tags=[] +# # Run + +# %% tags=[] +# initialize methods +ccc(np.random.rand(100), np.random.rand(100)) + +# %% tags=[] +for s in DATA_SIZES: + print(f"Size: {s}") + + print(" p") + run_method(lambda x, y: pearsonr(x, y)[0], "p-6", s) + + print(" s") + run_method(lambda x, y: spearmanr(x, y)[0], "s-6", s) + + print(" cm") + run_method(lambda x, y: ccc(x, y, n_jobs=N_JOBS), "cm-6", s) + + if s <= 50000: + print(" mic_e") + run_method(lambda x, y: mic(x, y, estimator="mic_e"), "mic_e-6", s) + + if s <= 10000: + print(" mic") + run_method(lambda x, y: mic(x, y), "mic-6", s) + + print("Saving to pickle") + time_results.to_pickle(OUTPUT_DIR / OUTPUT_FILENAME) + + print("\n") + +# %% [markdown] tags=[] +# # Summary of results + +# %% tags=[] +time_results.shape + +# %% tags=[] +time_results.head() + +# %% tags=[]