UnsupervisedDeep.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Deep_Unsupervised.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "QZtdOQm4oAu1"
      ],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "interpreter": {
      "hash": "39944124f9b5d60d862a3cf40ea9b11aa8e952b3995fe09a01f412de3b98a252"
    },
    "kernelspec": {
      "display_name": "Python 3.9.4 64-bit",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.4"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "72836c8033994ae08c6542d426b2746d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_54ca608f2b134bc99cbe8552617aae11",
              "IPY_MODEL_9a69bcb494be4448ac833bf86e647937",
              "IPY_MODEL_363ac6728f3c421cbfc738c8695c2d13"
            ],
            "layout": "IPY_MODEL_59a07da32df4407caef87ed495e52506"
          }
        },
        "59a07da32df4407caef87ed495e52506": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "54ca608f2b134bc99cbe8552617aae11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_92de25edbf6a46ef8118dd42b4902551",
            "placeholder": "​",
            "style": "IPY_MODEL_8fa60c3185ed4196b91d8f78d758da21",
            "value": "Running accuracy: 50.00%, Current val accuracy: 0.00%, : "
          }
        },
        "9a69bcb494be4448ac833bf86e647937": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b9cb8a95bb8d4034a362b1995b2e3607",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7bec0305213245cbb64121e1cd6957b7",
            "value": 1
          }
        },
        "363ac6728f3c421cbfc738c8695c2d13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_17f30272a75e4f948a94926e052e6317",
            "placeholder": "​",
            "style": "IPY_MODEL_cbaa198b9b184cbf9212f1726a295d08",
            "value": " 2/? [00:01&lt;00:00,  1.66it/s]"
          }
        },
        "8fa60c3185ed4196b91d8f78d758da21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "92de25edbf6a46ef8118dd42b4902551": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7bec0305213245cbb64121e1cd6957b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b9cb8a95bb8d4034a362b1995b2e3607": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cbaa198b9b184cbf9212f1726a295d08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "17f30272a75e4f948a94926e052e6317": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "42f941495e98470aabd084db3e9a3eb9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c7ca306e81134bb0913b099265b23f8b",
              "IPY_MODEL_5117bc33a6514d2cb652c2202275bb77",
              "IPY_MODEL_f089e106cc564148be9738fff31a506a"
            ],
            "layout": "IPY_MODEL_a0dd44abe23543cd96a3cd55dccee40b"
          }
        },
        "a0dd44abe23543cd96a3cd55dccee40b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c7ca306e81134bb0913b099265b23f8b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_45627b0c9c8c4c11bd13d7237ea1f606",
            "placeholder": "​",
            "style": "IPY_MODEL_5edd981b809e4dd18120b74d41ee02bc",
            "value": ""
          }
        },
        "5117bc33a6514d2cb652c2202275bb77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fb77a66894e0404c9252853fb8d413aa",
            "max": 9999,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7c481db05ae94f3097a55194076efa08",
            "value": 9999
          }
        },
        "f089e106cc564148be9738fff31a506a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c65f8dd3a44648999c32829d724e86d2",
            "placeholder": "​",
            "style": "IPY_MODEL_46dd37f63e0f4c22910ed49c4a113caf",
            "value": " 10000/? [1:20:33&lt;00:00,  2.07it/s]"
          }
        },
        "5edd981b809e4dd18120b74d41ee02bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "45627b0c9c8c4c11bd13d7237ea1f606": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7c481db05ae94f3097a55194076efa08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fb77a66894e0404c9252853fb8d413aa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "46dd37f63e0f4c22910ed49c4a113caf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c65f8dd3a44648999c32829d724e86d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Singular-Brain/DeepBioLCNet/blob/main/UnsupervisedDeep.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7fTSvrK3T_GA"
      },
      "source": [
        "#Notebook setups"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lXtgP_iEPE0G",
        "outputId": "50aa2e17-45d0-43fa-e852-2147780e3f61"
      },
      "source": [
        "!pip install -q git+https://github.com/Singular-Brain/DeepBioLCNet"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[?25l\r\u001b[K     |██▊                             | 10 kB 16.5 MB/s eta 0:00:01\r\u001b[K     |█████▍                          | 20 kB 22.8 MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 30 kB 12.5 MB/s eta 0:00:01\r\u001b[K     |██████████▉                     | 40 kB 9.4 MB/s eta 0:00:01\r\u001b[K     |█████████████▋                  | 51 kB 5.4 MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 61 kB 5.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 71 kB 5.7 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▊          | 81 kB 6.4 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▌       | 92 kB 6.4 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▏    | 102 kB 5.3 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 112 kB 5.3 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 120 kB 5.3 MB/s \n",
            "\u001b[K     |████████████████████████████████| 73 kB 1.7 MB/s \n",
            "\u001b[K     |████████████████████████████████| 280 kB 43.8 MB/s \n",
            "\u001b[K     |████████████████████████████████| 28.5 MB 53 kB/s \n",
            "\u001b[?25h  Building wheel for BindsNET (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\n",
            "albumentations 0.1.12 requires imgaug<0.2.7,>=0.2.5, but you have imgaug 0.2.9 which is incompatible.\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OW7m3ugEHZP_",
        "outputId": "6daeb918-ef23-46b6-b334-f487a716f5cf"
      },
      "source": [
        "!wget https://data.deepai.org/mnist.zip\n",
        "!mkdir -p ../data/MNIST/TorchvisionDatasetWrapper/raw\n",
        "!unzip mnist.zip -d ../data/MNIST/TorchvisionDatasetWrapper/raw/"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2021-10-22 18:03:41--  https://data.deepai.org/mnist.zip\n",
            "Resolving data.deepai.org (data.deepai.org)... 138.201.36.183\n",
            "Connecting to data.deepai.org (data.deepai.org)|138.201.36.183|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 11597176 (11M) [application/x-zip-compressed]\n",
            "Saving to: ‘mnist.zip’\n",
            "\n",
            "mnist.zip           100%[===================>]  11.06M  11.0MB/s    in 1.0s    \n",
            "\n",
            "2021-10-22 18:03:42 (11.0 MB/s) - ‘mnist.zip’ saved [11597176/11597176]\n",
            "\n",
            "Archive:  mnist.zip\n",
            "  inflating: ../data/MNIST/TorchvisionDatasetWrapper/raw/train-labels-idx1-ubyte.gz  \n",
            "  inflating: ../data/MNIST/TorchvisionDatasetWrapper/raw/train-images-idx3-ubyte.gz  \n",
            "  inflating: ../data/MNIST/TorchvisionDatasetWrapper/raw/t10k-images-idx3-ubyte.gz  \n",
            "  inflating: ../data/MNIST/TorchvisionDatasetWrapper/raw/t10k-labels-idx1-ubyte.gz  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KXcXvvsXcOlv",
        "outputId": "6e485baf-4d24-479c-f662-b132c5aff80c"
      },
      "source": [
        "!git clone https://github.com/Singular-Brain/DeepBioLCNet/"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'DeepBioLCNet'...\n",
            "remote: Enumerating objects: 347, done.\u001b[K\n",
            "remote: Counting objects: 100% (347/347), done.\u001b[K\n",
            "remote: Compressing objects: 100% (273/273), done.\u001b[K\n",
            "remote: Total 347 (delta 124), reused 249 (delta 57), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (347/347), 31.43 MiB | 26.15 MiB/s, done.\n",
            "Resolving deltas: 100% (124/124), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BFGNAecpT-Lj"
      },
      "source": [
        "from bindsnet.network.nodes import Nodes\n",
        "import os\n",
        "import torch\n",
        "import random\n",
        "import numpy as np\n",
        "import copy\n",
        "import math\n",
        "import matplotlib.pyplot as plt\n",
        "import collections\n",
        "from torchvision import transforms\n",
        "from tqdm.notebook import tqdm\n",
        "from sklearn.metrics import confusion_matrix\n",
        "import seaborn as sn\n",
        "import torch.nn.functional as fn\n",
        "\n",
        "from abc import ABC, abstractmethod\n",
        "from matplotlib.axes import Axes\n",
        "from matplotlib.image import AxesImage\n",
        "from mpl_toolkits.axes_grid1 import make_axes_locatable\n",
        "from typing import Union, Tuple, Optional, Sequence\n",
        "from torch.nn.modules.utils import _pair\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn import svm\n",
        "from sklearn import metrics\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "from bindsnet.datasets import MNIST\n",
        "from bindsnet.encoding import PoissonEncoder\n",
        "from bindsnet.network import Network\n",
        "from bindsnet.network.nodes import Input, LIFNodes, AdaptiveLIFNodes, IFNodes\n",
        "from bindsnet.network.topology import LocalConnection, Connection, LocalConnectionOrig, MaxPool2dLocalConnection\n",
        "from bindsnet.network.monitors import Monitor, AbstractMonitor, TensorBoardMonitor\n",
        "from bindsnet.learning import PostPre, MSTDP, MSTDPET, WeightDependentPostPre, Hebbian\n",
        "from bindsnet.learning.reward import DynamicDopamineInjection, DopaminergicRPE\n",
        "from bindsnet.analysis.plotting import plot_locally_connected_weights,plot_locally_connected_weights_meh,plot_locally_connected_weights_one_chan,plot_spikes,plot_locally_connected_weights_meh2,plot_convergence_and_histogram,plot_locally_connected_weights_meh3\n",
        "from bindsnet.analysis.visualization import plot_weights_movie, plot_spike_trains_for_example,summary, plot_voltage\n",
        "from bindsnet.utils import reshape_locally_connected_weights, reshape_locally_connected_weights_meh, reshape_conv2d_weights"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ULGGHW43UksI"
      },
      "source": [
        "## Sets up Gpu use and manual seed\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LiUmFrpcUfmR",
        "outputId": "6c73ff0a-9930-4834-a4a4-e53727a2bfd8"
      },
      "source": [
        "if torch.cuda.is_available():\n",
        "    device =  torch.device(\"cuda\")\n",
        "    gpu = True\n",
        "else:\n",
        "    device =  torch.device(\"cpu\")\n",
        "    gpu = False\n",
        "\n",
        "def manual_seed(seed):\n",
        "    random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    np.random.seed(seed)\n",
        "    if torch.cuda.is_available():\n",
        "        torch.cuda.manual_seed_all(seed)\n",
        "\n",
        "\n",
        "SEED = 2045 # The Singularity is Near!\n",
        "manual_seed(SEED)\n",
        "\n",
        "torch.set_num_threads(os.cpu_count() - 1)\n",
        "print(\"Running on Device = \", device)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Running on Device =  cuda\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MBKedMpIleMr"
      },
      "source": [
        "# Custom Monitors"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6tfqpsr2a1WV"
      },
      "source": [
        "## Reward Monitor"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M44GJ65GleMs"
      },
      "source": [
        "class RewardMonitor(AbstractMonitor):\n",
        "    # language=rst\n",
        "    \"\"\"\n",
        "    Records state variables of interest.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        time: None,\n",
        "        batch_size: int = 1,\n",
        "        device: str = \"cpu\",\n",
        "    ):\n",
        "        # language=rst\n",
        "        \"\"\"\n",
        "        Constructs a ``Monitor`` object.\n",
        "\n",
        "        :param obj: An object to record state variables from during network simulation.\n",
        "        :param state_vars: Iterable of strings indicating names of state variables to record.\n",
        "        :param time: If not ``None``, pre-allocate memory for state variable recording.\n",
        "        :param device: Allow the monitor to be on different device separate from Network device\n",
        "        \"\"\"\n",
        "        super().__init__()\n",
        "\n",
        "        self.time = time\n",
        "        self.batch_size = batch_size\n",
        "        self.device = device\n",
        "\n",
        "        # if time is not specified the monitor variable accumulate the logs\n",
        "        if self.time is None:\n",
        "            self.device = \"cpu\"\n",
        "\n",
        "        self.recording = []\n",
        "        self.reset_state_variables()\n",
        "\n",
        "    def get(self,) -> torch.Tensor:\n",
        "        # language=rst\n",
        "        \"\"\"\n",
        "        Return recording to user.\n",
        "\n",
        "        :return: Tensor of shape ``[time, n_1, ..., n_k]``, where ``[n_1, ..., n_k]`` is the shape of the recorded state\n",
        "        variable.\n",
        "        Note, if time == `None`, get return the logs and empty the monitor variable\n",
        "\n",
        "        \"\"\"\n",
        "        # return_logs = torch.as_tensor(self.recording)\n",
        "        # if self.time is None:\n",
        "        #     self.recording = []\n",
        "        return self.recording\n",
        "\n",
        "    def record(self, **kwargs) -> None:\n",
        "        # language=rst\n",
        "        \"\"\"\n",
        "        Appends the current value of the recorded state variables to the recording.\n",
        "        \"\"\"\n",
        "        if \"reward\" in kwargs:\n",
        "            self.recording.append(kwargs[\"reward\"])\n",
        "        # remove the oldest element (first in the list)\n",
        "        # if self.time is not None:\n",
        "        #     self.recording.pop(0)\n",
        "\n",
        "    def reset_state_variables(self) -> None:\n",
        "        # language=rst\n",
        "        \"\"\"\n",
        "        Resets recordings to empty ``List``s.\n",
        "        \"\"\"\n",
        "        self.recording = []\n"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8clxN_npa1WY"
      },
      "source": [
        "## Plot Eligibility trace"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SshGlRwpa1WZ"
      },
      "source": [
        "class PlotET(AbstractMonitor):\n",
        "    # language=rst\n",
        "    \"\"\"\n",
        "    Records state variables of interest.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        i,\n",
        "        j,\n",
        "        source,\n",
        "        target,\n",
        "        connection,\n",
        "    ):\n",
        "        # language=rst\n",
        "        \"\"\"\n",
        "        Constructs a ``Monitor`` object.\n",
        "\n",
        "        :param obj: An object to record state variables from during network simulation.\n",
        "        :param state_vars: Iterable of strings indicating names of state variables to record.\n",
        "        :param time: If not ``None``, pre-allocate memory for state variable recording.\n",
        "        :param device: Allow the monitor to be on different device separate from Network device\n",
        "        \"\"\"\n",
        "        super().__init__()\n",
        "        self.i = i\n",
        "        self.j = j\n",
        "        self.source = source\n",
        "        self.target = target\n",
        "        self.connection = connection\n",
        "\n",
        "        self.reset_state_variables()\n",
        "\n",
        "    def get(self,) -> torch.Tensor:\n",
        "        # language=rst\n",
        "        \"\"\"\n",
        "        Return recording to user.\n",
        "\n",
        "        :return: Tensor of shape ``[time, n_1, ..., n_k]``, where ``[n_1, ..., n_k]`` is the shape of the recorded state\n",
        "        variable.\n",
        "        Note, if time == `None`, get return the logs and empty the monitor variable\n",
        "\n",
        "        \"\"\"\n",
        "        # return_logs = torch.as_tensor(self.recording)\n",
        "        # if self.time is None:\n",
        "        #     self.recording = []\n",
        "        return self.recording\n",
        "\n",
        "    def record(self, **kwargs) -> None:\n",
        "        # language=rst\n",
        "        \"\"\"\n",
        "        Appends the current value of the recorded state variables to the recording.\n",
        "        \"\"\"\n",
        "        if hasattr(self.connection.update_rule, 'p_plus'):\n",
        "            self.recording['spikes_i'].append(self.source.s.ravel()[self.i].item())\n",
        "            self.recording['spikes_j'].append(self.target.s.ravel()[self.j].item())\n",
        "            self.recording['p_plus'].append(self.connection.update_rule.p_plus[self.i].item())\n",
        "            self.recording['p_minus'].append(self.connection.update_rule.p_minus[self.j].item())\n",
        "            self.recording['eligibility'].append(self.connection.update_rule.eligibility[self.i,self.j].item())\n",
        "            self.recording['eligibility_trace'].append(self.connection.update_rule.eligibility_trace[self.i,self.j].item())\n",
        "            self.recording['w'].append(self.connection.w[self.i,self.j].item())\n",
        "\n",
        "    def plot(self):\n",
        "\n",
        "        fig, axs  = plt.subplots(7)\n",
        "        fig.set_size_inches(10, 20)\n",
        "        for i, (name, p) in enumerate(self.recording.items()):\n",
        "            axs[i].plot(p[-250:])\n",
        "            axs[i].set_title(name)\n",
        "    \n",
        "        fig.show()\n",
        "\n",
        "    def reset_state_variables(self) -> None:\n",
        "        # language=rst\n",
        "        \"\"\"\n",
        "        Resets recordings to empty ``List``s.\n",
        "        \"\"\"\n",
        "        self.recording = {\n",
        "        'spikes_i': [],\n",
        "        'spikes_j': [],\n",
        "        'p_plus':[],\n",
        "        'p_minus':[],\n",
        "        'eligibility':[],\n",
        "        'eligibility_trace':[],\n",
        "        'w': [],\n",
        "        }\n"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R_YGE1XjvIkZ"
      },
      "source": [
        "## Kernel "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-4hp2V46vOUv"
      },
      "source": [
        "class AbstractKernel(ABC):\n",
        "    def __init__(self, kernel_size):\n",
        "        \"\"\"\n",
        "        Base class for generating image filter kernels such as Gabor, DoG, etc. Each subclass should override :attr:`__call__` function.\n",
        "        Instantiates a ``Filter Kernel`` object.\n",
        "        :param window_size : The size of the kernel (int)\n",
        "        \"\"\"\n",
        "        self.window_size = kernel_size\n",
        "\n",
        "    def __call__(self):\n",
        "        pass\n"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PL2L6_ABwBH4"
      },
      "source": [
        "class DoGKernel(AbstractKernel):\n",
        "    def __init__(self, kernel_size: Union[int, Tuple[int, int]], sigma1 : float, sigma2 : float):\n",
        "        \"\"\"\n",
        "        Generates DoG filter kernels.\n",
        "        :param kernel_size: Horizontal and vertical size of DOG kernels.(If pass int, we consider it as a square filter) \n",
        "        :param sigma1 : The sigma parameter for the first Gaussian function.\n",
        "        :param sigma2 : The sigma parameter for the second Gaussian function.\n",
        "        \"\"\"\n",
        "        super(DoGKernel, self).__init__(kernel_size)\n",
        "        self.sigma1 = sigma1\n",
        "        self.sigma2 = sigma2\n",
        "        \n",
        "    def __call__(self):\n",
        "        k = self.window_size//2\n",
        "        x, y = np.mgrid[-k:k+1:1, -k:k+1:1]\n",
        "        a = 1.0 / (2 * math.pi)\n",
        "        prod = x*x + y*y\n",
        "        f1 = (1/(self.sigma1*self.sigma1)) * np.exp(-0.5 * (1/(self.sigma1*self.sigma1)) * (prod))\n",
        "        f2 = (1/(self.sigma2*self.sigma2)) * np.exp(-0.5 * (1/(self.sigma2*self.sigma2)) * (prod))\n",
        "        dog = a * (f1-f2)\n",
        "        dog_mean = np.mean(dog)\n",
        "        dog = dog - dog_mean\n",
        "        dog_max = np.max(dog)\n",
        "        dog = dog / dog_max\n",
        "        dog_tensor = torch.from_numpy(dog)\n",
        "        # returns a 2d tensor corresponding to the requested DoG filter\n",
        "        return dog_tensor.float()"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zBUT0IUZDXxW"
      },
      "source": [
        "class Filter():\n",
        "    \"\"\"\n",
        "    Applies a filter transform. Each filter contains a sequence of :attr:`FilterKernel` objects.\n",
        "    The result of each filter kernel will be passed through a given threshold (if not :attr:`None`).\n",
        "    Args:\n",
        "        filter_kernels (sequence of FilterKernels): The sequence of filter kernels.\n",
        "        padding (int, optional): The size of the padding for the convolution of filter kernels. Default: 0\n",
        "        thresholds (sequence of floats, optional): The threshold for each filter kernel. Default: None\n",
        "        use_abs (boolean, optional): To compute the absolute value of the outputs or not. Default: False\n",
        "    .. note::\n",
        "        The size of the compund filter kernel tensor (stack of individual filter kernels) will be equal to the \n",
        "        greatest window size among kernels. All other smaller kernels will be zero-padded with an appropriate \n",
        "        amount.\n",
        "    \"\"\"\n",
        "    # filter_kernels must be a list of filter kernels\n",
        "    # thresholds must be a list of thresholds for each kernel\n",
        "    def __init__(self, filter_kernels, padding=0, thresholds=None, use_abs=False):\n",
        "        tensor_list = []\n",
        "        self.max_window_size = 0\n",
        "        for kernel in filter_kernels:\n",
        "            if isinstance(kernel, torch.Tensor):\n",
        "                tensor_list.append(kernel)\n",
        "                self.max_window_size = max(self.max_window_size, kernel.size(-1))\n",
        "            else:\n",
        "                tensor_list.append(kernel().unsqueeze(0))\n",
        "                self.max_window_size = max(self.max_window_size, kernel.window_size)\n",
        "        for i in range(len(tensor_list)):\n",
        "            p = (self.max_window_size - filter_kernels[i].window_size)//2\n",
        "            tensor_list[i] = fn.pad(tensor_list[i], (p,p,p,p))\n",
        "\n",
        "        self.kernels = torch.stack(tensor_list)\n",
        "        self.number_of_kernels = len(filter_kernels)\n",
        "        self.padding = padding\n",
        "        if isinstance(thresholds, list):\n",
        "            self.thresholds = thresholds.clone().detach()\n",
        "            self.thresholds.unsqueeze_(0).unsqueeze_(2).unsqueeze_(3)\n",
        "        else:\n",
        "            self.thresholds = thresholds\n",
        "        self.use_abs = use_abs\n",
        "\n",
        "    # returns a 4d tensor containing the flitered versions of the input image\n",
        "    # input is a 4d tensor. dim: (minibatch=1, filter_kernels, height, width)\n",
        "    def __call__(self, input):\n",
        "\n",
        "        # if input.dim() == 3:\n",
        "        #     input2 = torch.unsqueeze(input, 0)\n",
        "        input.unsqueeze_(0)\n",
        "        output = fn.conv2d(input, self.kernels, padding = self.padding).float()\n",
        "        if not(self.thresholds is None):\n",
        "            output = torch.where(output < self.thresholds, torch.tensor(0.0, device=output.device), output)\n",
        "        if self.use_abs:\n",
        "            torch.abs_(output)\n",
        "        return output.squeeze(0)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ywXyWP0I83Au"
      },
      "source": [
        "# Design network"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lm23tJtADDRT"
      },
      "source": [
        "def convergence(c):\n",
        "    if c.norm is None:\n",
        "        return 1-torch.mean((c.w-c.wmin)*(c.wmax-c.w))/((c.wmax-c.wmin)/2)**2\n",
        "    else:\n",
        "        mean_norm_factor = c.norm / c.w.shape[-1]\n",
        "        return  1-(torch.mean((c.w-c.wmin)*(c.wmax-c.w))/((mean_norm_factor-c.wmin)*(c.wmax-mean_norm_factor)))"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8bZpJmlrJDa9"
      },
      "source": [
        "compute_size = lambda inp_size, k, s: int((inp_size-k)/s) + 1\n",
        "\n",
        "class LCNet(Network):\n",
        "    def __init__(\n",
        "        self,\n",
        "        dt: float,\n",
        "        time: int,\n",
        "        channels: list,\n",
        "        filters: list,\n",
        "        strides: list,\n",
        "        input_channels: int,\n",
        "        nu_LC: Union[float, Tuple[float, float]],\n",
        "        NodesType_LC,\n",
        "        crop_size:int,\n",
        "        inh_LC: bool,\n",
        "        nu_inh_LC: float,\n",
        "        inh_factor_LC: float,\n",
        "        norm_factor_LC: float,\n",
        "        update_rule_LC,\n",
        "        update_rule_inh_LC,\n",
        "        wmin: float,\n",
        "        wmax: float ,\n",
        "        soft_bound: bool,\n",
        "        theta_plus: float,\n",
        "        tc_theta_decay: float,\n",
        "        tc_trace:int,\n",
        "        trace_additive,\n",
        "        load_path,\n",
        "        save_path,\n",
        "        confusion_matrix,\n",
        "        lc_weights_vis,\n",
        "        out_weights_vis,\n",
        "        lc_convergence_vis,\n",
        "        out_convergence_vis,\n",
        "        **kwargs,\n",
        "    ) -> None:\n",
        "        # language=rst\n",
        "        \"\"\"\n",
        "        Constructor for class ``BioLCNet``.\n",
        "\n",
        "        :param n_inpt: Number of input neurons. Matches the 1D size of the input data.\n",
        "        :param n_neurons: Number of excitatory, inhibitory neurons.\n",
        "        :param exc: Strength of synapse weights from excitatory to inhibitory layer.\n",
        "        :param inh: Strength of synapse weights from inhibitory to excitatory layer.\n",
        "        :param dt: Simulation time step.\n",
        "        :param nu: Single or pair of learning rates for pre- and post-synaptic events,\n",
        "            respectively.\n",
        "        :param reduction: Method for reducing parameter updates along the minibatch\n",
        "            dimension.\n",
        "        :param wmin: Minimum allowed weight on input to excitatory synapses.\n",
        "        :param wmax: Maximum allowed weight on input to excitatory synapses.\n",
        "        :param norm: Input to excitatory layer connection weights normalization\n",
        "            constant.\n",
        "        :param theta_plus: On-spike increment of ``(adaptive)LIFNodes`` membrane\n",
        "            threshold potential.\n",
        "        :param tc_theta_decay: Time constant of ``(adaptive)LIFNodes`` threshold\n",
        "            potential decay.\n",
        "        :param inpt_shape: The dimensionality of the input layer.\n",
        "        \"\"\"\n",
        "        manual_seed(SEED)\n",
        "        super().__init__(dt=dt, reward_fn = None, online=True)\n",
        "        \n",
        "        self.dt = dt\n",
        "        self.save_path = save_path\n",
        "        self.load_path = load_path\n",
        "        self.time = time\n",
        "        self.input_channels = input_channels\n",
        "        self.crop_size = crop_size\n",
        "        self.clamp_intensity = kwargs.get('clamp_intensity',None)\n",
        "        self.soft_bound = soft_bound\n",
        "        self.confusion_matrix = confusion_matrix\n",
        "        self.lc_weights_vis = lc_weights_vis\n",
        "        self.out_weights_vis = out_weights_vis\n",
        "        self.lc_convergence_vis = lc_convergence_vis\n",
        "        self.out_convergence_vis = out_convergence_vis\n",
        "        self.convergences = {}\n",
        "        self.wmin = wmin \n",
        "        self.wmax = wmax\n",
        "\n",
        "        ### nodes\n",
        "        inp = Input(shape= [input_channels,crop_size,crop_size], traces=True, tc_trace=tc_trace,traces_additive = trace_additive)\n",
        "        self.add_layer(inp, name=\"layer0\")\n",
        "\n",
        "        last_layer_size = crop_size\n",
        "        last_layer = inp\n",
        "        last_layer_channels = 1\n",
        "        inh_LC_type = kwargs.get('inh_LC_type', 'recurrent')\n",
        "        LC_weights_path = kwargs.get('LC_weights_path', None)\n",
        "\n",
        "        for i, (n_channels,filter_size,stride) in enumerate(zip(channels,filters,strides)):\n",
        "            new_layer_size = compute_size(last_layer_size, filter_size, stride)\n",
        "            new_layer = NodesType_LC(\n",
        "                shape= [n_channels, new_layer_size, new_layer_size],\n",
        "                traces=True, tc_trace=tc_trace,traces_additive = trace_additive,\n",
        "                tc_theta_decay = tc_theta_decay, theta_plus = theta_plus\n",
        "                )\n",
        "            self.add_layer(new_layer, name=f\"layer{i+1}\")\n",
        "            ### connections \n",
        "            LC_connection = LocalConnection(\n",
        "                last_layer, new_layer,\n",
        "                filter_size, stride,\n",
        "                last_layer_channels, n_channels,\n",
        "                input_shape = [last_layer_size]*2,\n",
        "                nu = nu_LC, update_rule= update_rule_LC,\n",
        "                wmin = wmin, wmax= wmax, soft_bound = soft_bound,\n",
        "                norm = norm_factor_LC)\n",
        "            \n",
        "            if LC_weights_path:\n",
        "                LC_connection.w.data = torch.load(LC_weights_path)['state_dict']['input_to_main1.w']\n",
        "                print(\"Weights loaded ...\")\n",
        "\n",
        "            self.add_connection(LC_connection, f\"layer{i}\", f\"layer{i+1}\")\n",
        "            ###\n",
        "            last_layer_size = new_layer_size\n",
        "            last_layer = new_layer\n",
        "            last_layer_channels = n_channels\n",
        "            ### Inhibitory connection\n",
        "            if inh_LC:\n",
        "                #if inh_LC_type != 'lateral':\n",
        "                w_inh_LC = torch.zeros(n_channels,new_layer_size,new_layer_size,n_channels,new_layer_size,new_layer_size)\n",
        "                for c in range(n_channels):\n",
        "                    for w1 in range(new_layer_size):\n",
        "                        for w2 in range(new_layer_size):\n",
        "                            w_inh_LC[c,w1,w2,:,w1,w2] = - inh_factor_LC\n",
        "                            w_inh_LC[c,w1,w2,c,w1,w2] = 0\n",
        "            \n",
        "                w_inh_LC = w_inh_LC.reshape(new_layer.n,new_layer.n)\n",
        "                                                            \n",
        "                LC_recurrent_inhibition = Connection(\n",
        "                    source=new_layer,\n",
        "                    target=new_layer,\n",
        "                    w=w_inh_LC,\n",
        "                    nu= nu_inh_LC,\n",
        "                )\n",
        "                self.add_connection(LC_recurrent_inhibition, f\"layer{i+1}\", f\"layer{i+1}\")\n",
        "\n",
        "                #else:\n",
        "                inh_neurons_layer = LIFNodes(\n",
        "                    shape= [n_channels],\n",
        "                    traces=True, tc_trace=tc_trace,traces_additive = trace_additive,\n",
        "                    tc_theta_decay = tc_theta_decay, theta_plus = theta_plus\n",
        "                    ).to(device)\n",
        "                self.add_layer(inh_neurons_layer, name=f\"layer{i+1}_inh_neurons\")\n",
        "\n",
        "                w_inh_LC = torch.zeros(n_channels, n_channels,new_layer_size,new_layer_size)\n",
        "                w_inh_LC_rev = torch.zeros(n_channels, new_layer_size, new_layer_size, n_channels)\n",
        "\n",
        "                for c in range(n_channels):\n",
        "                    for w1 in range(new_layer_size):\n",
        "                        for w2 in range(new_layer_size): \n",
        "                            w_inh_LC[c,c,w1,w2] = -inh_factor_LC\n",
        "                            w_inh_LC_rev[c,w1,w2,c] = ((13.0/4)*new_layer_size**2)/(new_layer_size**2)\n",
        "\n",
        "                w_inh_LC = w_inh_LC.reshape(n_channels, new_layer.n)\n",
        "                w_inh_LC_rev = w_inh_LC_rev.reshape(new_layer.n, n_channels)\n",
        "\n",
        "                LC_lateral_inhibition = Connection(\n",
        "                    source=inh_neurons_layer,\n",
        "                    target=new_layer,\n",
        "                    w=w_inh_LC,\n",
        "                    nu= nu_inh_LC,\n",
        "                ).to(device)\n",
        "\n",
        "                LC_lateral_inhibition_rev = Connection(\n",
        "                    source=new_layer,\n",
        "                    target=inh_neurons_layer,\n",
        "                    w=w_inh_LC_rev,\n",
        "                    nu= nu_inh_LC,\n",
        "                ).to(device)\n",
        "\n",
        "                self.add_connection(LC_lateral_inhibition, f\"layer{i+1}_inh_neurons\", f\"layer{i+1}\")\n",
        "                self.add_connection(LC_lateral_inhibition_rev, f\"layer{i+1}\",f\"layer{i+1}_inh_neurons\")\n",
        "\n",
        "        self.to(device)\n",
        "\n",
        "\n",
        "    def fit(\n",
        "        self,\n",
        "        dataloader,\n",
        "        n_train = 2000,\n",
        "        verbose = True,\n",
        "    ):\n",
        "        manual_seed(SEED)\n",
        "        self.verbose = verbose\n",
        "        # add Monitors\n",
        "        #Plot_et = PlotET(i = 0, j = 0, source = self.layers[\"main\"], target = self.layers[\"output\"], connection = self.connections[(\"main\",\"output\")])\n",
        "        #tensorboard = TensorBoardMonitor(self, time = self.time)\n",
        "        #self.add_monitor(main_monitor, name=\"main\")\n",
        "        #self.add_monitor(Plot_et, name=\"Plot_et\")\n",
        "        #self.add_monitor(tensorboard, name=\"tensorboard\")\n",
        "        self.spikes = {}\n",
        "        for layer in set(self.layers):\n",
        "            self.spikes[layer] = Monitor(self.layers[layer], state_vars=[\"s\"], time=None)\n",
        "            self.add_monitor(self.spikes[layer], name=\"%s_spikes\" % layer)\n",
        "\n",
        "        if self.load_path:\n",
        "            self.model_params = torch.load(self.load_path, torch.device('cuda') if gpu else torch.device('cpu'))\n",
        "            self.load_state_dict(torch.load(self.load_path, torch.device('cuda') if gpu else torch.device('cpu'))['state_dict'])\n",
        "            iteration =  self.model_params['iteration']\n",
        "            hparams = self.model_params['hparams']\n",
        "            train_accs = self.model_params['train_accs']\n",
        "            val_accs = self.model_params['val_accs']\n",
        "            acc_rewards = self.model_params['acc_rewards']\n",
        "            print(f'Previous model loaded! Resuming training from iteration {iteration}..., last running training accuracy: {train_accs[-1]}, last validation accuracy: {val_accs[-1]}\\n') if self.verbose else None\n",
        "        else:\n",
        "            print(f'Previous model not found! Training from the beginning...\\n') if self.verbose else None\n",
        "\n",
        "\n",
        "        pbar = tqdm(total=n_train)\n",
        "        self.reset_state_variables()\n",
        "\n",
        "        for (i, datum) in enumerate(dataloader):\n",
        "            if self.load_path:\n",
        "                if i <= iteration:\n",
        "                    n_train += 1\n",
        "                    continue\n",
        "            if i > n_train:\n",
        "                break\n",
        "\n",
        "            image = datum[\"encoded_image\"]\n",
        "            # Run the network on the input.\n",
        "            if gpu:\n",
        "                inputs = {\"layer0\": image.cuda().view(self.time, 1, self.input_channels, self.crop_size, self.crop_size)}\n",
        "            else:\n",
        "                inputs = {\"layer0\": image.view(self.time, 1, self.input_channels, self.crop_size, self.crop_size)}\n",
        "            ### Spike clamping (baseline activity)\n",
        "            clamp = {}\n",
        "            if self.clamp_intensity is not None:\n",
        "                encoder = PoissonEncoder(time = self.time, dt = self.dt)\n",
        "                clamp['output'] = encoder.enc(datum = torch.rand(self.layers['output'].n)*self.clamp_intensity,time = self.time, dt = self.dt)\n",
        "\n",
        "            self.run(inputs=inputs, \n",
        "                    time=self.time, \n",
        "                    one_step=True,\n",
        "                    clamp = clamp\n",
        "                     )\n",
        "\n",
        "            # Get voltage recording.\n",
        "            #tensorboard.update(step= i)\n",
        "\n",
        "            ### convergence\n",
        "            # w_lc1 = self.connections[('input', 'main1')].w\n",
        "            # w_last_main_out = self.connections[(self.final_connection_source_name,'output')].w\n",
        "            # convg_lc1=  1 -  torch.mean((w_lc1 - self.wmin)*(self.wmax-w_lc1))\n",
        "            # convg_out= 1 -  torch.mean((w_last_main_out - self.wmin)*(self.wmax-w_last_main_out))\n",
        "            # if self.norm_factor_LC is not None:\n",
        "            #     mean_norm_factor_lc = self.norm_factor_LC / w_lc1.shape[-1]\n",
        "            #     convg_lc1=  1 - ( torch.mean((w_lc1 - self.wmin)*(self.wmax-w_lc1)) / ((mean_norm_factor_lc - self.wmin)*(self.wmax - mean_norm_factor_lc)) )\n",
        "            # if self.norm_factor_out is not None:    \n",
        "            #     mean_norm_factor_out = self.norm_factor_out / w_last_main_out.shape[-2]\n",
        "            #     convg_out= 1 - ( torch.mean((w_last_main_out - self.wmin)*(self.wmax-w_last_main_out)) / ((mean_norm_factor_out - self.wmin)*(self.wmax - mean_norm_factor_out)) )\n",
        "            # self.convergences['lc1'].append((convg_lc1 * 10**4).round() / (10**4))\n",
        "            # self.convergences['last_main_out'].append((convg_out * 10**4).round() / (10**4)) \n",
        "\n",
        "            #print(f\"\\r{i}/{n_train}\",\n",
        "                # sum_spikes, 'pred_label:',\n",
        "                # predicted_label.item(), 'GT:', label.item(),\n",
        "                # ', Acc Rew:', round(sum(reward_monitor.get()).item(),4),\n",
        "                # f\"Pos dps: {self.reward_fn.dps:.5f}, Neg dps: {self.reward_fn.neg_dps:.5f}, Rew base: {self.reward_fn.rew_base:.5f}, Pun base: {self.reward_fn.punish_base:.5f}, RPe: {self.reward_fn.reward_predict_episode:.3f}\",\n",
        "                # f\"input_mean_fire_freq: {torch.mean(image.float())*1000:.1f},main_mean_fire_freq:{torch.mean(lc_spikes1.float())*1000:.1f}\",#\" main2_mean_fire_freq:{torch.mean(lc_spikes2.float())*1000:.1f}\",\n",
        "                # f\"output_mean_fire_freq:{torch.mean(out_spikes.float())*1000:.1f}\",\n",
        "                # f\"mean_lc1_w: {torch.mean(w_lc1):.5f}, mean_fc_w:{torch.mean(w_last_main_out):.5f}\",\n",
        "                # f\"std_lc1_w: {torch.std(w_lc1):.5f}, std_fc_w:{torch.std(w_last_main_out):.5f}\",\n",
        "                # f\"convergence_lc1: {convg_lc1:.5f}, convergence_fc: {convg_out:.5f}\",\n",
        "                # end = '')       \n",
        "            if verbose:\n",
        "                print(f\"\\r \",\n",
        "                    f\"input_mean_fire_freq: {torch.mean(image.float())*1000:.1f}\",\n",
        "                    '- Spikes:', {name: monitor.get('s').sum().item() for name, monitor in self.spikes.items()},\n",
        "                    '- Layers Mean:', list(map(lambda x: {x[0]: round(x[1].w.mean().item(),3)}, self.connections.items())),\n",
        "                    '- convergences:', {name: round(convergence(c).item() , 5) for name, c in self.connections.items() if c.wmin!=-np.inf and c.wmax!=np.inf },\n",
        "                    '- std:', {name: round(c.w.std().item() , 5) for name, c in self.connections.items() if c.wmin!=-np.inf and c.wmax!=np.inf },\n",
        "                    end = ''\n",
        "                    )    \n",
        "\n",
        "            #Plot_et.plot()    \n",
        "            self.reset_state_variables()  # Reset state variables.\n",
        "            \n",
        "            # pbar.set_description_str(\"Running accuracy: \" + \"{:.2f}\".format(acc) + \"%, \" + \"Current val accuracy: \" + \"{:.2f}\".format(val_acc) + \"%, \")\n",
        "            pbar.update()\n",
        "\n",
        "        # tensorboard.writer.add_hparams(\n",
        "        #     {k:(v if type(v) in (int, float, bool, str, torch.Tensor) else str(v)) for k,v in {**train_hparams, **data_hparams, **network_hparams, **reward_hparams}.items() },\n",
        "        #     result_metrics\n",
        "        # )\n",
        "\n",
        "\n",
        "    def predict(\n",
        "        self,\n",
        "        val_loader,\n",
        "        n_pred,\n",
        "    ):\n",
        "        manual_seed(SEED)\n",
        "\n",
        "        monitors = []\n",
        "        for layer in self.layers:\n",
        "            if layer == 'layer0':\n",
        "                continue\n",
        "            monitor = Monitor(self.layers[layer], [\"s\", \"v\"], time=time)\n",
        "            monitors.append(monitor)\n",
        "            self.add_monitor(monitor, name=f\"{layer} monitor\")\n",
        "        self.train(False)\n",
        "        S= []\n",
        "        V= []\n",
        "        y= []\n",
        "        pbar = tqdm(total=n_pred)\n",
        "        for (i, datum) in enumerate(val_loader):\n",
        "            if i > n_pred:\n",
        "                break\n",
        "\n",
        "            image = datum[\"encoded_image\"]\n",
        "            if self.label is None : \n",
        "              label = datum[\"label\"]\n",
        "            else :\n",
        "              label = self.label\n",
        "\n",
        "            # Run the network on the input.\n",
        "            if gpu:\n",
        "                inputs = {\"layer0\": image.cuda().view(self.time, 1, self.input_channels, self.crop_size, self.crop_size)}\n",
        "            else:\n",
        "                inputs = {\"layer0\": image.view(self.time, 1, self.input_channels, self.crop_size, self.crop_size)}\n",
        "\n",
        "            self.run(inputs=inputs, \n",
        "                    time=self.time, \n",
        "                    **reward_hparams,\n",
        "                    one_step = True,\n",
        "                    true_label = label.int().item(),\n",
        "                    dopaminergic_layers= self.dopaminergic_layers,\n",
        "                     )\n",
        "\n",
        "            S.append(list(map(lambda x: x.get('s'), monitors)))\n",
        "            V.append(list(map(lambda x: x.get('v'), monitors)))\n",
        "            y.append(label)\n",
        "\n",
        "            self.reset_state_variables()  # Reset state variables.\n",
        "            pbar.update()\n",
        "\n",
        "        self.train(True)\n",
        "        return S, V ,y"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6zuUBU9pU3vE"
      },
      "source": [
        "# Load Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2DW7dB11jdqi"
      },
      "source": [
        "class ClassSelector(torch.utils.data.sampler.Sampler):\n",
        "    \"\"\"Select target classes from the dataset\"\"\"\n",
        "    def __init__(self, target_classes, data_source, mask = None):\n",
        "        if mask is not None:\n",
        "            self.mask = mask\n",
        "        else:\n",
        "            self.mask = torch.tensor([1 if data_source[i]['label'] in target_classes else 0 for i in range(len(data_source))])\n",
        "        self.data_source = data_source\n",
        "\n",
        "    def __iter__(self):\n",
        "        return iter([i.item() for i in torch.nonzero(self.mask)])\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.data_source)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DlCXBY0DU3Mc"
      },
      "source": [
        "kernels = [DoGKernel(7,1,2),\n",
        "\t\t\tDoGKernel(7,2,1),]\n",
        "filter = Filter(kernels, padding = 3, thresholds = 50/255)\n",
        "# Load MNIST data.\n",
        "\n",
        "def load_datasets(network_hparams, data_hparams, mask=None, test_mask=None):\n",
        "    manual_seed(SEED)\n",
        "    dataset = MNIST(\n",
        "        PoissonEncoder(time=network_hparams['time'], dt=network_hparams['dt']),\n",
        "        None,\n",
        "        root=os.path.join(\"..\", \"..\", \"data\", \"MNIST\"),\n",
        "        download=True,\n",
        "        transform=transforms.Compose(\n",
        "            [transforms.ToTensor(),\n",
        "            # filter,\n",
        "            transforms.Lambda(lambda x: (\n",
        "                x.round() if data_hparams['round_input'] else x\n",
        "            ) * data_hparams['intensity']),\n",
        "            transforms.CenterCrop(data_hparams['crop_size'])]\n",
        "        ),\n",
        "    )\n",
        "\n",
        "    # Create a dataloader to iterate and batch data\n",
        "    dataloader = torch.utils.data.DataLoader(dataset, batch_size=1,\n",
        "                                            sampler = ClassSelector(\n",
        "                                                    target_classes = target_classes,\n",
        "                                                    data_source = dataset,\n",
        "                                                    mask = mask,\n",
        "                                                    ) if target_classes else None\n",
        "                                            )\n",
        "\n",
        "    # Load test dataset\n",
        "    test_dataset = MNIST(   \n",
        "        PoissonEncoder(time=network_hparams['time'], dt=network_hparams['dt']),\n",
        "        None,\n",
        "        root=os.path.join(\"..\", \"..\", \"data\", \"MNIST\"),\n",
        "        download=True,\n",
        "        train=False,\n",
        "        transform=transforms.Compose(\n",
        "            [transforms.ToTensor(),\n",
        "            # filter,\n",
        "            transforms.Lambda(lambda x: (\n",
        "                x.round() if data_hparams['round_input'] else x\n",
        "            ) * data_hparams['intensity']),\n",
        "            transforms.CenterCrop(data_hparams['crop_size'])]\n",
        "        ),\n",
        "    )\n",
        "\n",
        "    val_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1,\n",
        "                                            sampler = ClassSelector(\n",
        "                                                    target_classes = target_classes,\n",
        "                                                    data_source = test_dataset,\n",
        "                                                    mask = mask_test,\n",
        "                                                    ) if target_classes else None\n",
        "                                            )\n",
        "    \n",
        "\n",
        "    return dataloader, val_loader"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kCqAFucAUDb8"
      },
      "source": [
        "# Set up hyper-parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MVOqxcYtFd5T"
      },
      "source": [
        "# Dataset Hyperparameters\n",
        "target_classes = None #(0,1)\n",
        "if target_classes:\n",
        "    npz_file = np.load(f'bindsnet/mask_{\"_\".join([str(i) for i in target_classes])}.npz')\n",
        "    # npz_file = np.load('bindsnet/mask_0_1.npz') ##### KESAFAT KARI !!!\n",
        "    mask, mask_test = torch.from_numpy(npz_file['arr_0']), torch.from_numpy(npz_file['arr_1'])\n",
        "    n_classes = len(target_classes)\n",
        "    \n",
        "else:\n",
        "    mask = None\n",
        "    mask_test = None\n",
        "    n_classes = 10\n",
        "\n",
        "data_hparams = { \n",
        "    'intensity': 128,\n",
        "    'crop_size': 22,\n",
        "    'round_input': False,\n",
        "}"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3TerGeJoFdzg"
      },
      "source": [
        "train_hparams = {\n",
        "    'n_train' : 4000,\n",
        "}\n",
        "\n",
        "network_hparams = {\n",
        "    # net structure\n",
        "    'channels': [100],\n",
        "    'filters': [15],\n",
        "    'strides': [4],\n",
        "    'input_channels': 1,\n",
        "    # time & Phase\n",
        "    'time': 250,\n",
        "    'dt' : 1,\n",
        "    # Nodes\n",
        "    'NodesType_LC': AdaptiveLIFNodes,\n",
        "    'theta_plus': 0.05,\n",
        "    'tc_theta_decay': 1e6,\n",
        "    'tc_trace':20,\n",
        "    'trace_additive' : False,\n",
        "    # Learning\n",
        "    'update_rule_LC': PostPre,\n",
        "    'update_rule_inh_LC' : None,\n",
        "    'nu_LC': (1e-4, 1e-2),\n",
        "    'soft_bound': False,\n",
        "    'wmin': 0,\n",
        "    'wmax': 1,\n",
        "    # Inhibition\n",
        "    'nu_inh_LC': 0.0,\n",
        "    'inh_LC': True,\n",
        "    'inh_LC_type': 'lateral',\n",
        "    'inh_factor_LC': 100,\n",
        "    # Normalization\n",
        "    'norm_factor_LC': 0.25*15*15,\n",
        "    'norm_factor_inh': None,\n",
        "    'norm_factor_inh_LC': None,\n",
        "    # clamp\n",
        "    'clamp_intensity': None,\n",
        "    # Save\n",
        "    'LC_weights_path': None,#'/content/drive/My Drive/LCNet/BioLCNet_layer1_Shallow_f15_s4_inh100_norm25_ch100.pth',\n",
        "    'save_path': '/content/drive/My Drive/LCNet/LCNet_phase2_baseline2_gpu_tmp.pth',\n",
        "    'load_path': None,#'/content/drive/My Drive/LCNet/LCNet_phase2_baseline2_gpu.pth',\n",
        "    # Plot:\n",
        "    'confusion_matrix' : False,\n",
        "    'lc_weights_vis': False,\n",
        "    'out_weights_vis': False,\n",
        "    'lc_convergence_vis': False,\n",
        "    'out_convergence_vis': False,\n",
        "}\n"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Rz1wmKKYuVg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "68d7f029-fb86-44f9-f881-d5466a0e372f"
      },
      "source": [
        "dataloader, val_loader = load_datasets(network_hparams, data_hparams, mask, mask_test)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
            "Using downloaded and verified file: ../../data/MNIST/TorchvisionDatasetWrapper/raw/train-images-idx3-ubyte.gz\n",
            "Extracting ../../data/MNIST/TorchvisionDatasetWrapper/raw/train-images-idx3-ubyte.gz to ../../data/MNIST/TorchvisionDatasetWrapper/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
            "Using downloaded and verified file: ../../data/MNIST/TorchvisionDatasetWrapper/raw/train-labels-idx1-ubyte.gz\n",
            "Extracting ../../data/MNIST/TorchvisionDatasetWrapper/raw/train-labels-idx1-ubyte.gz to ../../data/MNIST/TorchvisionDatasetWrapper/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
            "Using downloaded and verified file: ../../data/MNIST/TorchvisionDatasetWrapper/raw/t10k-images-idx3-ubyte.gz\n",
            "Extracting ../../data/MNIST/TorchvisionDatasetWrapper/raw/t10k-images-idx3-ubyte.gz to ../../data/MNIST/TorchvisionDatasetWrapper/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
            "Using downloaded and verified file: ../../data/MNIST/TorchvisionDatasetWrapper/raw/t10k-labels-idx1-ubyte.gz\n",
            "Extracting ../../data/MNIST/TorchvisionDatasetWrapper/raw/t10k-labels-idx1-ubyte.gz to ../../data/MNIST/TorchvisionDatasetWrapper/raw\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SokdidkrV2Z5"
      },
      "source": [
        "# Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Venb2KhSYrT_"
      },
      "source": [
        "# from google.colab import drive\n",
        "# if network_hparams['save_path'] or network_hparams['LC_weights_path']:    \n",
        "#     drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IavVVHrvYVoI"
      },
      "source": [
        "manual_seed(SEED)\n",
        "net = LCNet(**network_hparams, **train_hparams, **data_hparams)\n",
        "net.fit(dataloader = dataloader, **train_hparams)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bRanR4n-gHjB"
      },
      "source": [
        "## Visualization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pwMJvA1_ffKQ"
      },
      "source": [
        "def plot_semantic_pooling(lc : object,\n",
        "    input_channel: int = 0,\n",
        "    output_channel: int = None,\n",
        "    lines: bool = True,\n",
        "    figsize: Tuple[int, int] = (5, 5),\n",
        "    cmap: str = \"hot_r\",\n",
        "    color: str='r',\n",
        "    ) -> AxesImage:\n",
        "    # language=rst\n",
        "    \"\"\"\n",
        "    Plot a connection weight matrix of a :code:`Connection` with `locally connected\n",
        "    structure <http://yann.lecun.com/exdb/publis/pdf/gregor-nips-11.pdf>_.\n",
        "    :param lc: LC connection object of LCNet\n",
        "    :param input_channel: indicates weights which connected to this channel of input \n",
        "    :param output_channel: indicates weights of specific channel in the output layer\n",
        "    :param lines: Whether or not to draw horizontal and vertical lines separating input regions.\n",
        "    :param figsize: Horizontal, vertical figure size in inches.\n",
        "    :param cmap: Matplotlib colormap.\n",
        "    :return: Used for re-drawing the weights plot.\n",
        "    \"\"\"\n",
        "\n",
        "    n_sqrt = int(np.ceil(np.sqrt(lc.out_channels)))\n",
        "    sel_slice = lc.w.view(lc.in_channels, lc.out_channels, lc.conv_size[0], lc.conv_size[1], lc.kernel_size[0], lc.kernel_size[1]).cpu()\n",
        "    input_size = _pair(int(np.sqrt(lc.source.n)))\n",
        "    \n",
        "    if output_channel is None:\n",
        "        sel_slice = sel_slice[input_channel, ...]\n",
        "        reshaped = reshape_LC_weights(sel_slice, lc.out_channels, lc.kernel_size, lc.conv_size, input_size)\n",
        "    else:\n",
        "        sel_slice = sel_slice[input_channel, output_channel, ...]\n",
        "        sel_slice = sel_slice.unsqueeze(0)\n",
        "        reshaped = reshape_LC_weights(sel_slice, 1, lc.kernel_size, lc.conv_size, input_size)\n",
        "        print(reshaped.shape)\n",
        "\n",
        "    fig, ax = plt.subplots(figsize=figsize)\n",
        "\n",
        "    im = ax.imshow(reshaped.cpu(), cmap=cmap, vmin=lc.wmin, vmax=lc.wmax)\n",
        "    div = make_axes_locatable(ax)\n",
        "    cax = div.append_axes(\"right\", size=\"5%\", pad=0.05)\n",
        "\n",
        "    if lines and  output_channel is None:\n",
        "        for i in range(\n",
        "            n_sqrt * lc.kernel_size[0],\n",
        "            n_sqrt * lc.conv_size[0] * lc.kernel_size[0],\n",
        "            n_sqrt * lc.kernel_size[0],\n",
        "        ):\n",
        "            ax.axhline(i - 0.5, color=color, linestyle=\"--\")\n",
        "\n",
        "        for i in range(\n",
        "            n_sqrt * lc.kernel_size[1],\n",
        "            n_sqrt * lc.conv_size[1] * lc.kernel_size[1],\n",
        "            n_sqrt * lc.kernel_size[1],\n",
        "        ):\n",
        "            ax.axvline(i - 0.5, color=color, linestyle=\"--\")\n",
        "\n",
        "    ax.set_xticks(())\n",
        "    ax.set_yticks(())\n",
        "    ax.set_aspect(\"auto\")\n",
        "\n",
        "    plt.colorbar(im, cax=cax)\n",
        "    fig.tight_layout()\n",
        "\n",
        "    return im"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jUCxrmCSp7ct"
      },
      "source": [
        "def reshape_LC_weights(\n",
        "    w: torch.Tensor,\n",
        "    n_filters: int,\n",
        "    kernel_size: Union[int, Tuple[int, int]],\n",
        "    conv_size: Union[int, Tuple[int, int]],\n",
        "    input_sqrt: Union[int, Tuple[int, int]],\n",
        ") -> torch.Tensor:\n",
        "    # language=rst\n",
        "    \"\"\"\n",
        "    Get the weights from a locally connected layer and reshape them to be two-dimensional and square.\n",
        "    :param w: Weights from a locally connected layer.\n",
        "    :param n_filters: No. of neuron filters.\n",
        "    :param kernel_size: Side length(s) of convolutional kernel.\n",
        "    :param conv_size: Side length(s) of convolution population.\n",
        "    :param input_sqrt: Sides length(s) of input neurons.\n",
        "    :return: Locally connected weights reshaped as a collection of spatially ordered square grids.\n",
        "    \"\"\"\n",
        "    k1, k2 = kernel_size\n",
        "    c1, c2 = conv_size\n",
        "    i1, i2 = input_sqrt\n",
        "    c1sqrt, c2sqrt = int(math.ceil(math.sqrt(c1))), int(math.ceil(math.sqrt(c2)))\n",
        "    fs = int(math.ceil(math.sqrt(n_filters)))\n",
        "\n",
        "    w_ = torch.zeros((n_filters * k1, k2 * c1 * c2))\n",
        "\n",
        "    for n1 in range(c1):\n",
        "        for n2 in range(c2):\n",
        "            for feature in range(n_filters):\n",
        "                n = n1 * c2 + n2\n",
        "                filter_ = w[feature, n1, n2, :, :\n",
        "                ].view(k1, k2)\n",
        "                w_[feature * k1 : (feature + 1) * k1, n * k2 : (n + 1) * k2] = filter_\n",
        "\n",
        "    if c1 == 1 and c2 == 1:\n",
        "        square = torch.zeros((i1 * fs, i2 * fs))\n",
        "\n",
        "        for n in range(n_filters):\n",
        "            square[\n",
        "                (n // fs) * i1 : ((n // fs) + 1) * i2,\n",
        "                (n % fs) * i2 : ((n % fs) + 1) * i2,\n",
        "            ] = w_[n * i1 : (n + 1) * i2]\n",
        "\n",
        "        return square\n",
        "    else:\n",
        "        square = torch.zeros((k1 * fs * c1, k2 * fs * c2))\n",
        "\n",
        "        for n1 in range(c1):\n",
        "            for n2 in range(c2):\n",
        "                for f1 in range(fs):\n",
        "                    for f2 in range(fs):\n",
        "                        if f1 * fs + f2 < n_filters:\n",
        "                            square[\n",
        "                                k1 * (n1 * fs + f1) : k1 * (n1 * fs + f1 + 1),\n",
        "                                k2 * (n2 * fs + f2) : k2 * (n2 * fs + f2 + 1),\n",
        "                            ] = w_[\n",
        "                                (f1 * fs + f2) * k1 : (f1 * fs + f2 + 1) * k1,\n",
        "                                (n1 * c2 + n2) * k2 : (n1 * c2 + n2 + 1) * k2,\n",
        "                            ]\n",
        "\n",
        "        return square"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bw4pMTVwqFsK"
      },
      "source": [
        "def plot_LC_weights(lc : object,\n",
        "    input_channel: int = 0,\n",
        "    output_channel: int = None,\n",
        "    lines: bool = True,\n",
        "    figsize: Tuple[int, int] = (5, 5),\n",
        "    cmap: str = \"hot_r\",\n",
        "    color: str='r',\n",
        "    ) -> AxesImage:\n",
        "    # language=rst\n",
        "    \"\"\"\n",
        "    Plot a connection weight matrix of a :code:`Connection` with `locally connected\n",
        "    structure <http://yann.lecun.com/exdb/publis/pdf/gregor-nips-11.pdf>_.\n",
        "    :param lc: LC connection object of LCNet\n",
        "    :param input_channel: indicates weights which connected to this channel of input \n",
        "    :param output_channel: indicates weights of specific channel in the output layer\n",
        "    :param lines: Whether or not to draw horizontal and vertical lines separating input regions.\n",
        "    :param figsize: Horizontal, vertical figure size in inches.\n",
        "    :param cmap: Matplotlib colormap.\n",
        "    :return: Used for re-drawing the weights plot.\n",
        "    \"\"\"\n",
        "\n",
        "    n_sqrt = int(np.ceil(np.sqrt(lc.out_channels)))\n",
        "    sel_slice = lc.w.view(lc.in_channels, lc.out_channels, lc.conv_size[0], lc.conv_size[1], lc.kernel_size[0], lc.kernel_size[1]).cpu()\n",
        "    input_size = _pair(int(np.sqrt(lc.source.n)))\n",
        "    \n",
        "    if output_channel is None:\n",
        "        sel_slice = sel_slice[input_channel, ...]\n",
        "        reshaped = reshape_LC_weights(sel_slice, lc.out_channels, lc.kernel_size, lc.conv_size, input_size)\n",
        "    else:\n",
        "        sel_slice = sel_slice[input_channel, output_channel, ...]\n",
        "        sel_slice = sel_slice.unsqueeze(0)\n",
        "        reshaped = reshape_LC_weights(sel_slice, 1, lc.kernel_size, lc.conv_size, input_size)\n",
        "        print(reshaped.shape)\n",
        "\n",
        "    fig, ax = plt.subplots(figsize=figsize)\n",
        "\n",
        "    im = ax.imshow(reshaped.cpu(), cmap=cmap, vmin=lc.wmin, vmax=lc.wmax)\n",
        "    div = make_axes_locatable(ax)\n",
        "    cax = div.append_axes(\"right\", size=\"5%\", pad=0.05)\n",
        "\n",
        "    if lines and  output_channel is None:\n",
        "        for i in range(\n",
        "            n_sqrt * lc.kernel_size[0],\n",
        "            n_sqrt * lc.conv_size[0] * lc.kernel_size[0],\n",
        "            n_sqrt * lc.kernel_size[0],\n",
        "        ):\n",
        "            ax.axhline(i - 0.5, color=color, linestyle=\"--\")\n",
        "\n",
        "        for i in range(\n",
        "            n_sqrt * lc.kernel_size[1],\n",
        "            n_sqrt * lc.conv_size[1] * lc.kernel_size[1],\n",
        "            n_sqrt * lc.kernel_size[1],\n",
        "        ):\n",
        "            ax.axvline(i - 0.5, color=color, linestyle=\"--\")\n",
        "\n",
        "    ax.set_xticks(())\n",
        "    ax.set_yticks(())\n",
        "    ax.set_aspect(\"auto\")\n",
        "\n",
        "    plt.colorbar(im, cax=cax)\n",
        "    fig.tight_layout()\n",
        "\n",
        "    return im"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tHh7sYKxuO8E"
      },
      "source": [
        "def plot_LC_activation_map(lc : object,\n",
        "    spikes: torch.tensor,\n",
        "    input_channel: int = 0,\n",
        "    scale_factor: float = 1.0,\n",
        "    lines: bool = True,\n",
        "    figsize: Tuple[int, int] = (5, 5),\n",
        "    cmap: str = \"hot_r\",\n",
        "    color: str='r'\n",
        "    ) -> AxesImage:\n",
        "    # language=rst\n",
        "    \"\"\"\n",
        "    Plot an activation map of a :code:`Connection` with `locally connected\n",
        "    structure <http://yann.lecun.com/exdb/publis/pdf/gregor-nips-11.pdf>_.\n",
        "    :param lc: LC connection object of LCNet\n",
        "    :param input_channel: indicates weights which connected to this channel of input \n",
        "    :param scale_factor: determines intensity of activation map \n",
        "    :param lines: Whether or not to draw horizontal and vertical lines separating input regions.\n",
        "    :param figsize: Horizontal, vertical figure size in inches.\n",
        "    :param cmap: Matplotlib colormap.\n",
        "    :return: Used for re-drawing the weights plot.\n",
        "    \"\"\"\n",
        "    spikes = spikes.sum(0).squeeze().view(lc.conv_size[0]*int(np.sqrt(lc.out_channels)),lc.conv_size[1]*int(np.sqrt(lc.out_channels)))\n",
        "    x = scale_factor * spikes / torch.max(spikes)\n",
        "    x = x.clip(lc.wmin,lc.wmax).repeat_interleave(lc.kernel_size[0], dim=0).repeat_interleave(lc.kernel_size[1], dim=1).cpu()\n",
        "    n_sqrt = int(np.ceil(np.sqrt(lc.out_channels)))\n",
        "\n",
        "    sel_slice = lc.w.view(lc.in_channels, lc.out_channels, lc.conv_size[0], lc.conv_size[1], lc.kernel_size[0], lc.kernel_size[1]).cpu()\n",
        "    sel_slice = sel_slice[input_channel, ...]\n",
        "    input_size = _pair(int(np.sqrt(lc.source.n)))\n",
        "    reshaped = reshape_LC_weights(sel_slice, lc.out_channels, lc.kernel_size, lc.conv_size, input_size)\n",
        "\n",
        "    fig, ax = plt.subplots(figsize=figsize)\n",
        "\n",
        "    im = ax.imshow(reshaped.cpu()*x, cmap=cmap, vmin=lc.wmin, vmax=lc.wmax)\n",
        "    div = make_axes_locatable(ax)\n",
        "    cax = div.append_axes(\"right\", size=\"5%\", pad=0.05)\n",
        "\n",
        "    if lines:\n",
        "        for i in range(\n",
        "            n_sqrt * lc.kernel_size[0],\n",
        "            n_sqrt * lc.conv_size[0] * lc.kernel_size[0],\n",
        "            n_sqrt * lc.kernel_size[0],\n",
        "        ):\n",
        "            ax.axhline(i - 0.5, color=color, linestyle=\"--\")\n",
        "\n",
        "        for i in range(\n",
        "            n_sqrt * lc.kernel_size[1],\n",
        "            n_sqrt * lc.conv_size[1] * lc.kernel_size[1],\n",
        "            n_sqrt * lc.kernel_size[1],\n",
        "        ):\n",
        "            ax.axvline(i - 0.5, color=color, linestyle=\"--\")\n",
        "\n",
        "    ax.set_xticks(())\n",
        "    ax.set_yticks(())\n",
        "    ax.set_aspect(\"auto\")\n",
        "\n",
        "    plt.colorbar(im, cax=cax)\n",
        "    fig.tight_layout()\n",
        "\n",
        "    return im"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fJSZLgU-wRrI"
      },
      "source": [
        "def plot_FC_response_map(lc: object,\n",
        "    fc: object,\n",
        "    ind_neuron_in_group: int,\n",
        "    label: int,\n",
        "    n_per_class: int,\n",
        "    input_channel: int = 0,\n",
        "    scale_factor: float = 1.0,\n",
        "    lines: bool = True,\n",
        "    figsize: Tuple[int, int] = (5, 5),\n",
        "    cmap: str = \"hot_r\",\n",
        "    color: str='r'\n",
        "    ) -> AxesImage:\n",
        "    # language=rst\n",
        "    \"\"\"\n",
        "    Plot a connection weight matrix of a :code:`Connection` with `locally connected\n",
        "    structure <http://yann.lecun.com/exdb/publis/pdf/gregor-nips-11.pdf>_.\n",
        "    :param lc: LC connection object of LCNet\n",
        "    :param fc: FC connection object of LCNet\n",
        "    :param input_channel: indicates weights which connected to this channel of input\n",
        "    :param scale_factor: determines intensity of activation map  \n",
        "    :param lines: Whether or not to draw horizontal and vertical lines separating input regions.\n",
        "    :param figsize: Horizontal, vertical figure size in inches.\n",
        "    :param cmap: Matplotlib colormap.\n",
        "    :return: Used for re-drawing the weights plot.\n",
        "    \"\"\"\n",
        "\n",
        "    n_sqrt = int(np.ceil(np.sqrt(lc.out_channels)))\n",
        "\n",
        "    sel_slice = lc.w.view(lc.in_channels, lc.out_channels, lc.conv_size[0], lc.conv_size[1], lc.kernel_size[0], lc.kernel_size[1]).cpu()\n",
        "    sel_slice = sel_slice[input_channel, ...]\n",
        "    input_size = _pair(int(np.sqrt(lc.source.n)))\n",
        "    reshaped = reshape_LC_weights(sel_slice, lc.out_channels, lc.kernel_size, lc.conv_size, input_size)\n",
        "\t\n",
        "    ind_neuron = label * n_per_class + ind_neuron_in_group\n",
        "    w = fc.w[:,ind_neuron].view(reshaped.shape[0]//lc.kernel_size[0],reshaped.shape[1]//lc.kernel_size[1])\n",
        "    w = w.clip(lc.wmin,lc.wmax).repeat_interleave(lc.kernel_size[0], dim=0).repeat_interleave(lc.kernel_size[1], dim=1).cpu()\n",
        "\n",
        "    fig, ax = plt.subplots(figsize=figsize)\n",
        "\n",
        "    im = ax.imshow(reshaped.cpu()*w, cmap=cmap, vmin=lc.wmin, vmax=lc.wmax)\n",
        "    div = make_axes_locatable(ax)\n",
        "    cax = div.append_axes(\"right\", size=\"5%\", pad=0.05)\n",
        "\n",
        "    if lines:\n",
        "        for i in range(\n",
        "            n_sqrt * lc.kernel_size[0],\n",
        "            n_sqrt * lc.conv_size[0] * lc.kernel_size[0],\n",
        "            n_sqrt * lc.kernel_size[0],\n",
        "        ):\n",
        "            ax.axhline(i - 0.5, color=color, linestyle=\"--\")\n",
        "\n",
        "        for i in range(\n",
        "            n_sqrt * lc.kernel_size[1],\n",
        "            n_sqrt * lc.conv_size[1] * lc.kernel_size[1],\n",
        "            n_sqrt * lc.kernel_size[1],\n",
        "        ):\n",
        "            ax.axvline(i - 0.5, color=color, linestyle=\"--\")\n",
        "\n",
        "    ax.set_xticks(())\n",
        "    ax.set_yticks(())\n",
        "    ax.set_aspect(\"auto\")\n",
        "\n",
        "    plt.colorbar(im, cax=cax)\n",
        "    fig.tight_layout()\n",
        "\n",
        "    return im"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_XcD7_8oJr6Z"
      },
      "source": [
        "# net.fit(dataloader = dataloader, **train_hparams)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kJnZUs0dOKcz"
      },
      "source": [
        "# net.spikes['layer1'].get('s').sum()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Ctbn9lcCP0X"
      },
      "source": [
        "# net.connections"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YLxsgMEVBl8z"
      },
      "source": [
        "plot_locally_connected_weights_one_chan(net.connections[('layer0', 'layer1')].w,100,1,0,22,15,2,0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jfAgDU35Uc1D"
      },
      "source": [
        "# net.connections[('layer1_inh_neuron', 'layer1')].w"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LTNxCad5BVqH"
      },
      "source": [
        "# **Following lines are NOT edited yet!**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FCCSMmjvaxeo"
      },
      "source": [
        "### Visualization "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SfTcVmMNCSUo"
      },
      "source": [
        "plot_locally_connected_weights_meh(net.connections[('layer0', 'layer1')].w,100,1,0,22,15,2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lqb0Ph5AYdRv"
      },
      "source": [
        "dataloader_iterator = iter(dataloader)\n",
        "for i in range(10):\n",
        "    datum = next(dataloader_iterator)\n",
        "    plt.imshow(datum['encoded_image'].squeeze().sum(0))\n",
        "    plt.show()\n",
        "    net.one_step(datum)\n",
        "    spikes = net.spikes[\"main1\"].get(\"s\").sum(0).squeeze().view(4*5,4*5)\n",
        "    print(torch.max(spikes))\n",
        "    plot_locally_connected_weights_meh2(net.connections[('input', 'main1')].w,spikes,25,1,0,22,13,4)\n",
        "    plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vkz7skWaiWj7"
      },
      "source": [
        "## Tensorboard"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nAKGbShj7kOM"
      },
      "source": [
        "# Load the TensorBoard notebook extension\n",
        "%load_ext tensorboard"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EXv13w317oW7"
      },
      "source": [
        "%tensorboard --logdir '/content/runs'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IGe49P4-q9zp"
      },
      "source": [
        "## Save/Load Sessions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1Ow_r8C5qzwu"
      },
      "source": [
        "Save tensorBoard Session "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XdHDClhIqUlW"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "!cp -a /content/runs/. /content/drive/MyDrive/LCNet/logs/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6NOgZCamq4p-"
      },
      "source": [
        "Read Saved Sessions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DlItthjkq4S5"
      },
      "source": [
        "%tensorboard --logdir /content/drive/MyDrive/LCNet/logs/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QZtdOQm4oAu1"
      },
      "source": [
        "## Optuna"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hcLDxcrwithj"
      },
      "source": [
        "install and import optuna"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CGik9_MeQfaI"
      },
      "source": [
        "!pip install optuna\n",
        "import optuna"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bsM-Y9CZiv15"
      },
      "source": [
        "Define objective function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yj_n_rwWDrdV"
      },
      "source": [
        "STUDY_NAME  = ''\n",
        "DATA_PATH = ''\n",
        "N_TRIALS = ''\n",
        "\n",
        "def objective(trial):\n",
        "    ### Suggest parameters: \n",
        "    num_layers = trial.suggest_int('Number of Layers', 1, 4)\n",
        "    dropout_rate  = trial.suggest_float('Dropout', 0, .99)\n",
        "    activation = trial.suggest_categorical('activation', ['relu', 'selu', 'sigmoid', 'elu'])\n",
        "    lr = trial.suggest_float('Learning rate', 1e-6, 1)\n",
        "    network_hparams.update({\n",
        "        \n",
        "    })\n",
        "    ###Define your model\n",
        "    manual_seed(SEED)\n",
        "    hparams = {**reward_hparams, **network_hparams, **train_hparams, **data_hparams}\n",
        "    net = LCNet(**hparams, reward_fn = DynamicDopamineInjection)\n",
        "    va_acc = net.fit(dataloader = dataloader, val_loader = val_loader, reward_hparams = reward_hparams, **train_hparams)\n",
        "    ### Define objective value\n",
        "    objective_value = min(va_acc)\n",
        "\n",
        "    \n",
        "    return objective_value\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ywzhxB84jthL"
      },
      "source": [
        "Run the study"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IDFttXEkjv4f"
      },
      "source": [
        "study = optuna.create_study(study_name = STUDY_NAME , storage=f\"sqlite:////content/drive/MyDrive/LCNet/optuna/optuna_study.db\", load_if_exists=True)\n",
        "study.optimize(objective, n_trials=N_TRIALS )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Azp25HveO77U",
        "outputId": "5fdaa0eb-5888-4acd-acdb-d059bfcd5b87"
      },
      "source": [
        "study.best_params"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'Dropout': 0.09142336347778651,\n",
              " 'Layer 1': 82,\n",
              " 'Layer 2': 508,\n",
              " 'Layer 3': 286,\n",
              " 'Learning rate': 0.008771184760927113,\n",
              " 'Number of Layers': 3,\n",
              " 'activation': 'relu',\n",
              " 'l1': None}"
            ]
          },
          "execution_count": 94,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "epsdhrYujy4f"
      },
      "source": [
        "Visualization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "lYgaOQZBaPJk",
        "outputId": "4cc05a5d-636a-4fa0-e260-cecd4b73845a"
      },
      "source": [
        "plot_parallel_coordinate(study, params=[\"Learning rate\", \"Number of Layers\", \"Dropout\"])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>\n",
              "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
              "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
              "            <div id=\"4da333a4-edd7-4ced-a4de-49a20b8ed5f8\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
              "            <script type=\"text/javascript\">\n",
              "                \n",
              "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
              "                    \n",
              "                if (document.getElementById(\"4da333a4-edd7-4ced-a4de-49a20b8ed5f8\")) {\n",
              "                    Plotly.newPlot(\n",
              "                        '4da333a4-edd7-4ced-a4de-49a20b8ed5f8',\n",
              "                        [{\"dimensions\": [{\"label\": \"Objective Value\", \"range\": [1.2982209920883179, 14561242.0], \"values\": [3.41186261177063, 3.41186261177063, 1573.8834228515625, 40182.33984375, 1528.2066650390625, 3.41186261177063, 5028653.5, 1902.826171875, 4562251.5, 945.5894165039062, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.5610835552215576, 1.2982209920883179, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1996253.125, 3.41186261177063, 1.3412549495697021, 3.41186261177063, 1.3456010818481445, 208.45639038085938, 1.3210997581481934, 3.41186261177063, 1.6091002225875854, 1.3304294347763062, 297.8522033691406, 1.4888664484024048, 1.4887771606445312, 1.3360676765441895, 1.3630410432815552, 3.41186261177063, 1.6488354206085205, 1.3287392854690552, 1.4895340204238892, 3.41186261177063, 3.41186261177063, 1.5344265699386597, 3.41186261177063, 3.41186261177063, 1.440764307975769, 1.488864779472351, 1.488818883895874, 1.466086745262146, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.3408962488174438, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.3349838256835938, 3.41186261177063, 7.432951927185059, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.3247721195220947, 1.4888302087783813, 1.322178602218628, 1.4888978004455566, 1.4888209104537964, 3.41186261177063, 2.3582754135131836, 1.3386236429214478, 3.41186261177063, 1.49014151096344, 1.488823652267456, 3.41186261177063, 1.3245543241500854, 1.323976993560791, 1.4888335466384888, 3.41186261177063, 3.9560186862945557, 3.41186261177063, 2.909315347671509, 1.3193601369857788, 3.41186261177063, 1.511308193206787, 1.3136441707611084, 3.41186261177063, 3.41186261177063, 1.4888402223587036, 1.4888317584991455, 3.41186261177063, 3.41186261177063, 1.3589725494384766, 2.0364224910736084, 3.41186261177063, 1.3351562023162842, 1.333082675933838, 3.41186261177063, 1.345519781112671, 3.41186261177063, 3.41186261177063, 1.3363773822784424, 3.41186261177063, 1.3386261463165283, 3.41186261177063, 1.3352710008621216, 1.4969419240951538, 3.41186261177063, 1.3299638032913208, 1.4888179302215576, 499.7135009765625, 1.488878846168518, 1.4904865026474, 1.3302356004714966, 1.3491047620773315, 3.41186261177063, 1.3281259536743164, 1.4885554313659668, 1.4888970851898193, 3.41186261177063, 1.34572434425354, 1.3183053731918335, 1.4915050268173218, 3.41186261177063, 1.490088939666748, 1.4888391494750977, 1.3113908767700195, 3.41186261177063, 2.1569273471832275, 1.325134038925171, 1.4888319969177246, 3.41186261177063, 2.6820173263549805, 1.4889917373657227, 1.3089126348495483, 1.308632254600525, 3.41186261177063, 1.4888687133789062, 1.3131179809570312, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.4888315200805664, 1.4690502882003784, 1.5091662406921387, 1.3258752822875977, 3.41186261177063, 1.4888314008712769, 1.4705278873443604, 4818.158203125, 1.488825798034668, 1.484520673751831, 3.41186261177063, 1.48581063747406, 1.4793022871017456, 3.41186261177063, 2.6853699684143066, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.5001596212387085, 1.3194295167922974, 1.327432632446289, 1.3218836784362793, 1.3434199094772339, 3.41186261177063, 1.3185940980911255, 1.4976372718811035, 1.3232676982879639, 3.41186261177063, 3.41186261177063, 1.315920352935791, 3.41186261177063, 3.41186261177063, 1.3266558647155762, 3.41186261177063, 1.3477894067764282, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.4888219833374023, 1.373160719871521, 1.3271931409835815, 3.41186261177063, 1.4715697765350342, 1.3213223218917847, 3.41186261177063, 1.308161973953247, 1.4888219833374023, 1.488818645477295, 1.4889020919799805, 1.4888558387756348, 2.69138503074646, 1.3204686641693115, 1.3125131130218506, 3.41186261177063, 1.4888322353363037, 1.3331835269927979, 415.56671142578125, 1.581402063369751, 3.41186261177063, 1.3211685419082642, 1.3021266460418701, 1.4888198375701904, 1.325307011604309, 1.329646110534668, 1.4888436794281006, 1.3281971216201782, 1.4888861179351807, 3.41186261177063, 1.305359959602356, 1.3265137672424316, 1.3640973567962646, 1.3347446918487549, 3.41186261177063, 1.3504081964492798, 1.3332096338272095, 3.41186261177063, 1.4888545274734497, 3.4115991592407227, 3.41186261177063, 3.41186261177063, 1.3193230628967285, 1.4888228178024292, 1.3274098634719849, 1.3126106262207031, 1.3173880577087402, 1.3275312185287476, 1.322373628616333, 1.489281415939331, 3.41186261177063, 1.310310959815979, 1.4888172149658203, 1.3176379203796387, 3.41186261177063, 3.41186261177063, 1.329596757888794, 1.32630455493927, 2.0398926734924316, 1.488019585609436, 1.3448506593704224, 1.48855721950531, 1.4942303895950317, 1.4888358116149902, 311.04400634765625, 1.3175146579742432, 1.3329596519470215, 1.488847255706787, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.4967288970947266, 1.4313173294067383, 1.4887478351593018, 3.41186261177063, 1.8477803468704224, 1.4888261556625366, 1.4888958930969238, 1.3229091167449951, 1.332504153251648, 1.3610163927078247, 3.41186261177063, 1.3288545608520508, 1.3301451206207275, 3.41186261177063, 3.41186261177063, 1.4888267517089844, 3.41186261177063, 3.4117937088012695, 1.3516267538070679, 1.3335763216018677, 3.41186261177063, 199.16970825195312, 1.489451289176941, 1.3276795148849487, 1.334709882736206, 1.3188152313232422, 3.41186261177063, 3.41186261177063, 1.3202966451644897, 1.3358441591262817, 3.41186261177063, 1.4888269901275635, 3.41186261177063, 1.3066781759262085, 3.41186261177063, 1.3137255907058716, 3.41186261177063, 3.41186261177063, 1.3406171798706055, 3.41186261177063, 1.2999054193496704, 1.3272432088851929, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.3045551776885986, 3.41186261177063, 1.4952431917190552, 2675.79052734375, 1.3019428253173828, 1.48887038230896, 1.3364481925964355, 1.32117760181427, 3.41186261177063, 1.3121179342269897, 3.41186261177063, 1.4888906478881836, 3.41186261177063, 1.3226450681686401, 3.41186261177063, 1.4888287782669067, 1.320836067199707, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.3085274696350098, 1.3093986511230469, 3.41186261177063, 1.488820195198059, 1.317246675491333, 1.4921292066574097, 1.326299786567688, 1.3326469659805298, 3.41186261177063, 1.488829255104065, 3.41186261177063, 1.3266675472259521, 1.491834282875061, 1.3059812784194946, 3.41186261177063, 696.8388061523438, 1.4321969747543335, 1.4888207912445068, 3.41186261177063, 1.311875820159912, 3.41186261177063, 1.3288514614105225, 1.3099689483642578, 1.313618540763855, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.488821268081665, 1.3155453205108643, 3.411557912826538, 3.41186261177063, 1.3217459917068481, 3.41186261177063, 1.4906315803527832, 3.41186261177063, 1.3453859090805054, 1.4888182878494263, 3.41186261177063, 1.314800500869751, 3.41186261177063, 1.3281633853912354, 3.41186261177063, 1.321211814880371, 1102.6551513671875, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.3377939462661743, 3.41186261177063, 3.41186261177063, 1.3136341571807861, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.3284739255905151, 3.41186261177063, 1.3212107419967651, 1.3475548028945923, 3.41186261177063, 3.41186261177063, 1.3276013135910034, 3.41186261177063, 1.3147908449172974, 1.4900482892990112, 1.4910539388656616, 1.4888460636138916, 1.3363206386566162, 3.41186261177063, 3.41186261177063, 1.4887561798095703, 1.3011691570281982, 57.89400863647461, 3.41186261177063, 3.41186261177063, 2.024686813354492, 1.4956120252609253, 1.4888505935668945, 1.488817572593689, 1.4888553619384766, 1.3047354221343994, 3.41186261177063, 1.488818645477295, 1.3213609457015991, 2.043469190597534, 1.3281235694885254, 1.3031138181686401, 1.4892010688781738, 14561242.0, 3.41186261177063, 1.2996032238006592, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.488633155822754, 3.41186261177063, 1.488843560218811, 1.4442107677459717, 1.415286660194397, 3.41186261177063, 1.4373067617416382, 3181.039794921875, 3.41186261177063, 3.41186261177063, 1.3226823806762695, 1.3124511241912842, 1.3017604351043701, 3.41186261177063, 1.6777530908584595, 1.4891026020050049, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.3141649961471558, 3.41186261177063, 3.41186261177063, 1.3052650690078735, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.4536466598510742, 1.3093748092651367, 1.316675066947937, 3.41186261177063, 3.41186261177063, 1.313673973083496, 4.355731964111328, 3.41186261177063, 3.41186261177063, 1.319928765296936, 3.41186261177063, 1.4888230562210083, 1.3414974212646484, 3.41186261177063, 1.488836646080017, 3.41186261177063, 2.672865629196167, 3.41186261177063, 3.41186261177063, 1.4835532903671265, 3.41186261177063, 1.3185737133026123, 1.503530740737915, 1.4888479709625244, 1.3155088424682617, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 2.0430803298950195, 1.3163092136383057, 3.41186261177063, 1.492780327796936, 3.41186261177063, 1.4853918552398682, 1.3153973817825317, 226.0158233642578]}, {\"label\": \"Dropout\", \"range\": [3.443103036387257e-05, 0.9842606560857956], \"values\": [0.18977214464009692, 0.7227315480926801, 0.6815362495608449, 0.5185686676008853, 0.9571123927309707, 0.9413976023486653, 0.9240297064647457, 0.8832423793421973, 0.24820480999218064, 0.22803488983480102, 0.03705698311217703, 0.6687979784843106, 0.2995763177694883, 0.013776342616227705, 0.7506263336085158, 0.46831020858962125, 0.4699295907735423, 0.7784624726260249, 0.126345073419686, 0.4033925539510268, 0.8100547663369849, 0.11997535769142222, 0.34517494036659885, 0.5609482806976116, 0.12266602597561153, 0.14107593864823564, 0.5710514219566252, 0.5784965594320876, 0.09142336347778651, 0.6032814544493361, 0.39142095676496175, 0.5769203239656192, 0.6516745218245438, 0.4028877538782995, 0.5187279938991173, 0.2042794916392674, 0.8247660706030391, 0.06986220574368689, 0.32785661170132746, 0.2983801710400664, 0.19267254632572553, 0.5191530269533787, 0.06790657486476076, 0.0912079503768097, 0.004813769026088974, 0.02323644800214135, 0.05801799181828775, 0.06654630194044663, 0.1666155407376949, 0.021752858239725212, 0.020394699664888126, 0.06269230669812192, 0.054764765785951214, 0.003260979031988895, 0.0010386661049646946, 0.0042514240575314555, 0.25692568926512044, 0.1046480091242594, 0.0936971406943859, 0.1593576837933141, 0.10060409892962652, 0.039881893955648826, 0.009131805200186065, 0.15825048489742177, 0.22795929224033357, 0.10328946678074592, 0.042645892947305065, 0.1300934156651391, 0.08137342020826593, 0.03255850397317858, 0.0032687211019654383, 0.0054011288573812795, 0.9842606560857956, 0.05241566677384037, 0.002022585909989674, 0.11208759600690653, 0.1178981600381534, 0.17176756453516412, 0.14382620352860775, 0.08002638436773904, 0.21665627576686247, 0.03430415450884129, 0.10863876962768966, 0.030115227553549145, 0.03588715693992984, 0.13729906518175178, 0.029199815203026433, 0.07966638871111814, 0.051836270847720585, 0.1906048350017894, 0.052127455144741465, 0.021879004551463396, 0.10722345283414045, 0.07211031931240228, 0.07263815083250716, 0.08682192217721761, 0.14274896340832918, 0.25978152456704495, 0.17556005013273301, 0.06500372738241907, 0.11079873774496185, 0.12166532010138775, 0.07666670846091683, 0.09294591114907316, 0.09945427922991029, 0.1436334728930368, 0.11745473975062676, 0.045141366801021654, 0.15738204180492116, 0.06396837828601888, 0.20450263253346906, 0.029951649004927874, 0.08539379720540134, 0.09754246888896347, 0.0944676340805405, 0.12790599888765714, 0.17946120846687202, 0.06368394847934356, 0.10661137990586715, 0.03718775312778729, 0.14924491872609683, 0.0930492261823937, 0.07439242994802123, 0.017260289893501327, 0.11763801390512933, 0.053661581396183974, 0.0927608235226057, 0.021326752148343786, 0.13127253890822146, 0.08071128808912653, 0.04747427149672451, 0.09799345728234844, 0.10262951470757437, 0.0739678764097289, 0.0008586693541276641, 0.01183804134064103, 0.0014356267148336547, 0.04224617681712018, 0.06138064527594206, 0.02552558850865591, 0.029606589985803084, 0.08804472738367561, 0.0003866889637423043, 0.05529174990778434, 0.02312440920650563, 0.02317213300112871, 0.03231882679344715, 0.002904293927166124, 0.0699004406403539, 0.00448326943437846, 0.048294424062963706, 0.016547035310487877, 0.0423895536083662, 0.06591671644615017, 0.04538835653479515, 0.07214399480856416, 0.03677882620553821, 0.03734087072507006, 0.0005399644029429574, 0.059174074777862344, 0.02175796750859533, 0.07841378095748251, 0.053676210841932734, 0.1229197435675967, 0.1212336573590234, 0.03153559086610436, 0.004505217805839373, 0.725009661957866, 0.11085439762023731, 0.06613954042567897, 0.15181784187943353, 0.08959052592625524, 0.08553933950919818, 0.045222467191919684, 0.13334769083000425, 0.4274785527106425, 0.0018561014024722964, 0.11252773843343927, 0.06733250397411983, 0.027161499608146324, 0.10145758292584943, 0.09788506936364058, 0.1247132785959726, 0.12205694512388535, 0.13697556034747282, 0.07515658552798707, 0.16493885952892962, 0.08049438235332498, 0.07343802314425626, 0.05568483294249682, 0.08312290232123834, 0.08187119892853142, 0.0441305867191334, 0.10405993300797822, 0.06422543732879281, 0.08608027941409539, 0.032712041482421204, 0.11789450753129224, 0.054165854629040576, 0.07442380720347613, 0.02137283982025037, 0.09518961875538334, 0.10602189156194396, 0.13826071757602598, 0.09823442686458109, 0.06530341290112941, 0.04581060389667832, 0.04519175876333156, 0.03972971749058086, 0.08252547219594705, 0.05757215785254003, 0.017542253050450242, 0.12149509799674728, 0.08553386026934692, 0.9048680751591708, 0.08956478890152726, 0.10137083204218914, 0.07945824023174168, 0.06513631323270413, 0.11092307616006822, 0.13346016050874387, 0.13272779553843028, 0.15843922017900025, 0.13385333797619936, 0.12475726304576015, 0.14592706172323708, 0.11323256323796947, 0.0908648492436995, 0.17597461623511873, 0.07529828749794304, 0.10189731540715245, 0.07109789877878567, 0.0820197012623823, 0.1281669140835429, 0.05465642862289666, 0.09799304431299133, 0.07133513914285625, 0.04672547275949813, 0.11668458465970423, 0.0870997576806521, 0.1431490101425841, 0.03576019658665775, 0.03752225056590631, 0.06184614590715558, 0.0752585503000527, 0.03418688298787724, 0.028634785685719363, 0.04400434385393255, 0.02248668327889719, 0.6325023504316492, 0.04185470251173645, 0.05389071784865134, 0.01784256865428973, 0.012472059314996472, 0.031710234141571106, 0.10390799040205387, 0.05681723822354634, 0.018083139703332547, 0.04340793203611792, 0.12099615126762288, 0.0012123950881871778, 0.09170236089458397, 0.07608627310921343, 0.06127578605141091, 0.1567887120342379, 0.15891195945698844, 0.038027575480039065, 0.10569912999577061, 0.058821226644631804, 0.14192372318087684, 0.3299064064086893, 0.08343104793506871, 0.02090251344093779, 0.0950415427466656, 0.5479339359027675, 0.06968289443677259, 0.04141827732751504, 0.000570584848104308, 0.10667096602486088, 0.19867047307767022, 0.13372486375023235, 0.052135849328373465, 0.0757173600579813, 0.2801135965972183, 0.030400900258563793, 0.11669726830065223, 0.08888725177314286, 0.016802875622266136, 0.05365730544446178, 0.07244109470701623, 0.37650259095448346, 0.09801232034917286, 0.1628413870155264, 0.034998878722532795, 0.11992200423695587, 0.06550405288064576, 0.04920234409864489, 0.003958329431651572, 0.06687416246866033, 0.05910665840408718, 0.02493245797214909, 0.07433863608415064, 0.04179387385129526, 0.06272949249072046, 0.07066471139078256, 0.06128176727489264, 0.014397822761264672, 0.04840784470298225, 0.06270251023692296, 0.03145252895786916, 0.08512320118211228, 0.08854798734239794, 0.8297435221086382, 0.0785216394512167, 0.04333047541430265, 0.4724268717765803, 0.0009237965290651851, 0.08490152125501811, 0.022325420161716954, 0.000352831704061439, 0.025808108443689103, 0.014253689149947492, 0.04437567788810526, 0.03702159236130973, 0.04891078828951246, 0.0009908864628801264, 0.022743874061923835, 0.05032228725377848, 0.061126731743288545, 0.05686315826836345, 0.03940061074414607, 0.02031842201071487, 0.06445085997902945, 0.04941282659282947, 0.07587288075369919, 0.020373090270359297, 0.055735997777530466, 0.08144651789480722, 0.0010050816382745596, 0.002013508468486122, 0.011344956028949409, 0.016326901982388106, 0.003381909491668025, 0.002376789144923369, 0.002984806271809526, 0.0011421540830595135, 0.03227896870373849, 0.0009872263847036586, 0.03522150530904351, 0.001876648751473811, 0.04001681354483606, 0.021935589587147992, 3.443103036387257e-05, 0.042409908491311314, 0.025450163349166152, 0.05528613418275456, 0.08950520367959361, 0.048265256860903286, 0.039971750010183216, 0.05640397313179488, 0.021004433926737603, 0.018804511153529522, 0.015989226938690374, 0.06596196265485539, 0.02587192434132664, 0.04500901802784196, 0.07739932405569355, 0.06463783888795176, 0.020056198767562535, 0.04917553285352942, 0.09232545137122536, 9.842401400229905e-05, 0.07688283093582121, 0.03289452352740781, 0.05624996911429003, 0.024305966166802737, 0.06811483876267768, 0.04238046169427501, 0.058885905732796534, 0.01981289662822524, 0.10214356163140752, 0.0017185422089690703, 0.06968508743339868, 0.036604449942349696, 0.045933276655229886, 0.07638778300457917, 0.015757393540010906, 0.09899664650449055, 0.0637614071795838, 0.050714498049248775, 0.031318824294545966, 5.007173838544876e-05, 0.08946034190501388, 0.05607111165610284, 0.022349163411206813, 0.03905149746434803, 0.08660342712761897, 0.059894605248555215, 0.017814331642879374, 0.03757389881530937, 0.07555184794795243, 0.001866192684735056, 0.10998387119941301, 0.0011387396109528401, 0.058582094842012634, 0.03543870089805992, 0.09529616866559847, 0.0003891799634519015, 0.07198127252807866, 0.0890584912244459, 0.07514843271206964, 0.04774444513644024, 0.021697666052363698, 0.10883006504940862, 0.048608302421363256, 0.07958706506048667, 0.02324236591884332, 0.06771101378015973, 0.06182776442907803, 0.03137438983428246, 0.048191754701293596, 0.012192317766585671, 0.06858913642686684, 0.037114301624740054, 0.01758311538438813, 0.03714380589765104, 0.0026554761408616664, 0.03136617361625645, 0.04279455899101382, 0.03821089413613963, 0.06069461393902334, 0.08263154256148016, 0.028886985625439437, 0.04715867084502808, 0.0141401258317767, 0.07004922161202476, 0.09811309445403787, 0.04707526852334387, 0.030903027061292408, 0.07622066265338005, 0.0018572433475626815, 0.0556359041156962, 0.10427322445950628, 0.026994069431266598, 0.004155113356530498, 0.0031013235196995434, 0.43156161185613384, 0.01555485328852984, 0.021914683417306718, 0.9805178380869909, 0.0030231885615408065, 0.028413741785973626, 0.003281262951288888, 0.03720104005601631, 0.003674994388898021, 0.00261923990754568, 0.0029676245113087266, 0.017906656339263374, 0.2452523315804691, 0.67708770503428, 0.02612417283916115, 0.04531049066974576, 0.004637315543127127, 0.013526854482325155, 0.0030398571876038265, 0.003935793273641438, 0.00011147460648451915, 0.04784672934459856, 0.0009924063792427773, 0.0308750336739222, 0.053919319575250646, 0.0013838864553914962, 0.02260437815196577, 0.04290171639333729, 0.0005994551596217451, 0.06215828032759168, 0.031406293072650436, 0.05779459070988171, 0.01615746453546504, 0.03916746309493418, 0.771355916526557, 0.0004434535661207023, 0.059683995693768616, 0.07609962368472586, 0.024391975533022945, 0.045078353997614784, 0.06986488665283336, 0.0214930933921556, 0.04285829518180141, 0.01877261375194468, 0.08699617281962638, 0.05547172468515345, 0.00047376795584941627, 0.0001956695935350361, 0.03798531711097779, 0.06870642092035026, 0.1035445891095131, 0.027148385411983125, 0.08649193502290965, 0.050004943961580686]}, {\"label\": \"Learning rate\", \"range\": [4.839092386705679e-06, 0.999345881690639], \"values\": [0.8318111488629595, 0.5825190685066907, 0.8026175082579672, 0.6970440623810661, 0.3718848814571064, 0.9194361419958322, 0.6203094078517274, 0.3552388697724913, 0.4276298249589038, 0.044580280884625914, 0.970940037395826, 0.5901029835471306, 0.7850291054202887, 0.1880601162630498, 0.8490109726052444, 0.5085890263628017, 0.6660617866540687, 0.8839902023100054, 0.21539453881286114, 0.7132230699402093, 0.9635280722837865, 0.23701479130654213, 0.7573731362385578, 0.999345881690639, 0.009805414827726844, 0.2262018768767552, 0.9939229598154355, 0.0005647326270766251, 0.008771184760927113, 0.13746334034929766, 0.11510997037238159, 0.04088806407666547, 0.08345094274377901, 0.11652154408979622, 0.2988501763969884, 0.7274573603250489, 0.9008562366485751, 0.2956187496041139, 0.520493928904637, 0.4401003119778594, 0.8072312699347531, 0.819573894354146, 0.0038562530732292755, 0.02868917334823695, 0.0066328812289193195, 0.010777294021001433, 0.0005026089887436532, 0.07018339557195341, 0.1569976610569958, 0.00011998139166572927, 0.07489030743527873, 0.05119949949940067, 0.06472811371731842, 0.003638930111458142, 0.002799875313480671, 0.1702828572315992, 0.09863628535301724, 7.536833608501325e-05, 0.13638142118669183, 0.03748671413913193, 0.18197735102681784, 0.006575684098518466, 0.033976745180029404, 0.1093242513307107, 0.0015533218647126464, 0.08255258339830071, 0.046648239428554956, 0.006117449112957771, 0.13736961307686082, 0.20678274872244926, 0.26072449645801843, 0.001804019407857957, 0.020667049747028265, 0.060973332016613924, 0.09894953059915138, 0.005009273284468599, 0.040869575211491926, 0.0005106437167951795, 0.08807903074181601, 0.12923208540114764, 0.06318368476551045, 0.00046545414732343873, 0.025326689619468153, 0.000836307857516224, 0.04682815809985486, 0.026096937838657575, 0.0821556605848515, 0.15404082948907472, 0.0004609839602319583, 0.37780178166151307, 0.10601748847672118, 0.024195652321651488, 0.06559479895703142, 0.0008278344577075933, 0.0006263382346023283, 0.04231944383144484, 0.6199098957578392, 0.02517602266172283, 0.05407579592812331, 0.08584897176475585, 0.00018656772848294379, 0.024198028745843078, 0.004503205451593816, 0.0004392748746655684, 0.05903817570064794, 0.5447735247632177, 0.04249084884283121, 0.022016121675431093, 0.11615508888831637, 0.07165078331223045, 0.003171236616504612, 0.020937086021157927, 0.04145644212433934, 0.00034831258999738074, 0.00024867633838791846, 0.05758447226199979, 1.530797035000437e-05, 0.09704370650534337, 0.022151707558353245, 0.00018962612560447894, 0.04204894309009263, 0.0027952489288840692, 0.0761684118992445, 0.000114276094693333, 0.021772036018191675, 0.038849786038969394, 0.0006001452444256708, 0.051978477563983494, 0.02013727253005528, 0.07358177466113344, 0.4467724084884145, 0.0032833509140603716, 7.09956810860733e-05, 0.03255131223512051, 3.606647282256226e-05, 0.05569446607220057, 0.021049024507253326, 0.03485501080228253, 9.469444130095345e-05, 0.00042076570458325906, 0.09062881811017992, 0.015945012116275574, 0.05993098579073255, 0.013570053587776267, 0.0015964858632057878, 0.03361534081065006, 0.04531106303592312, 0.0024510427688730465, 0.015831034405731706, 0.0716349585074928, 0.0016152111930970562, 0.03299261525432884, 0.00039750005293959315, 0.0003735123694220152, 0.022954684835970445, 0.05030750834425449, 0.0019456392138276438, 0.03636853045826222, 0.017641678995441265, 0.05553940340159806, 0.017740118481972378, 0.0018485961636392208, 0.003644648238668417, 0.0008905678352506823, 0.027921368867372797, 0.0425612233853034, 0.0017823998823285313, 0.07314653593250144, 0.021350698054067366, 0.0014578074201163668, 0.6869616904774993, 0.0018174574296485665, 0.00254190704159633, 0.04786929311322935, 0.0011421035682364898, 0.03194497537763594, 0.01909453388600864, 0.059908197211907266, 0.03340643650992153, 0.002560702283418414, 0.00047973318343545566, 3.428312596082551e-05, 0.0002621838796837353, 0.0006848323529311741, 0.019987745225089337, 0.0010200128772356876, 0.03606771370723579, 0.0002110597233802087, 0.020449786947493095, 0.05139164646076383, 0.0002768828002315304, 0.018231075948590064, 0.8672271619313537, 0.00025975098994360685, 0.03391207657075111, 0.0014886727804582435, 0.0185633767611625, 0.7767051575038381, 0.04092783700242046, 0.01732202128809746, 4.839092386705679e-06, 9.250249068863388e-05, 0.9468883072631986, 0.0013709578451224268, 0.000299448190639499, 0.02583869534986426, 0.0006589696904070356, 0.04738346334479894, 0.02034327834590992, 0.06475412429969495, 0.02006078515589438, 0.0038582777392556318, 0.00038058179938438763, 0.00044436504219958324, 0.03516725854281543, 0.01886451605024933, 0.0007141120682533505, 0.03399878985704272, 0.32670260609012014, 0.018343659424244783, 0.00021620854488433282, 0.019600361830822803, 0.015829874656146214, 0.002042446018317671, 0.001380493285773636, 0.04287775883688133, 0.0018594929095926919, 0.03304388215031253, 0.0176194148353488, 0.0019793001043673887, 0.0002234535250527764, 3.8187742437063805e-05, 0.00018275630908954593, 0.021243757037771104, 0.0005455542353190162, 0.00017461130867614476, 0.03854773589447761, 0.020835541184319396, 0.054227922207086734, 0.019837023419036204, 0.03570054537482614, 0.002664028966735375, 0.014622896166524818, 6.190764292296827e-05, 0.0020020589569840286, 0.0007428723574550438, 0.0010230302497837957, 0.0006402148287644081, 0.026947944878872445, 0.019591669722353232, 0.0006354729643167035, 0.04701484324177392, 0.001116046081058396, 0.020961850118742326, 0.03701763274370863, 0.00036125055463698697, 0.0009983878913666319, 0.01991791248581757, 0.057140131629773515, 6.490671006018837e-05, 0.03477775045206333, 0.40535723770474574, 0.018391357944358925, 0.017468259895893963, 1.6337379373740162e-05, 2.4275611411332405e-05, 0.04521134143996523, 0.03199729110331968, 0.06521636725184048, 0.017195208942716606, 0.0004852933705535605, 0.0015605175032298198, 0.03468196537957979, 0.2618252677392967, 0.0007734925840008856, 0.018525261947555407, 0.051289532775630066, 0.01875266703501147, 0.0006428107515578265, 0.0002969712375334517, 0.040156271154686105, 0.0005272010903337764, 9.923612903044296e-06, 0.4924530550230862, 0.031011174052843374, 0.019160568953309098, 0.0723802758494964, 0.05171238602106266, 0.018757726276329795, 0.0008900266959972472, 0.03289050004766596, 0.018820792078667818, 0.04100349661200286, 0.0011935629447178597, 0.0009347307225850039, 0.0008078155709304219, 0.02124873032076251, 0.06241904138094003, 0.00041781949424815374, 8.136484272236368e-05, 0.03553173899429297, 0.017727684254190853, 0.04845224030298898, 0.0006769261764987614, 0.638178433448891, 0.0005457508688992035, 0.019435548164763845, 0.03056704540699154, 7.508419154701003e-05, 0.08578648864135227, 0.0008819475400870596, 0.0003304492161692224, 0.020681782488756708, 0.05445371224659366, 0.03416892717068001, 0.02134166362153642, 0.01685350843413476, 0.041022072288961056, 0.0005759146007853044, 0.018336859269085407, 0.0008982257708812491, 0.06768937612732671, 0.0010533713867024353, 0.03938048426744762, 0.0008266021528289115, 0.0010448501238005006, 0.018145010853096565, 0.0002995882361112422, 0.05231516839949336, 0.032392650945967924, 0.01702546991488874, 0.0002942316335957289, 0.03474450766041615, 0.015348776662527364, 0.00027269847933289297, 0.0536858079070212, 0.01976538618810699, 0.036087164878083625, 0.0002827213937042818, 0.0015741200620933435, 0.08012561111167861, 0.017302362980815143, 0.0010400591616107789, 0.5608418405327812, 0.00011304829506689944, 0.00018579677481631713, 0.05230514036084195, 0.02909717629588059, 0.020255502508929196, 6.779688621230283e-05, 0.1979235140215808, 0.0003918360389160052, 0.038234671920108515, 0.0648772575391006, 0.0007414501936795313, 0.019384775180918985, 0.039904104044949956, 0.0006839603038786923, 0.021520193029948394, 0.0011345692354542934, 0.0003523715605829366, 0.00022719604343753762, 0.03400622127657148, 0.0544755872582909, 0.01841942571259952, 0.01585306813913724, 0.000895802241765959, 0.03584010936264005, 0.01995692910962721, 2.8411756343403323e-05, 0.05031701728599021, 0.07296107997361252, 0.016331784333223704, 0.000354943304533424, 0.03656480591031412, 0.01874706391101637, 0.001791009442819385, 0.03520417008345619, 0.0014026435551135446, 0.05579384823640002, 0.00028497881259451486, 0.021223368827225336, 0.09975430421623677, 0.7445915829684586, 0.019701393489866457, 0.00013601067862287642, 0.03945870917402249, 0.4753148348889939, 0.0007064693205584117, 0.01907932192961654, 0.06697944418920354, 0.03508214385987181, 0.0006737979932668642, 0.017025425866391903, 0.0002824027070356082, 0.00014348347017702667, 0.048829640019877885, 0.0311663083412486, 0.0002911588928624626, 0.02058280298567976, 0.00045442284690733604, 0.24101336562042008, 0.048612464298794844, 0.01791757022560716, 1.8929290669759307e-05, 0.35811131174784067, 0.030513876781038347, 0.07172710719984579, 0.0008570216410594862, 0.0005966385127998599, 0.033194357249811964, 0.01834671626233139, 0.0008652467309240649, 0.05712498760706458, 0.03441620914177599, 0.01829768702177668, 0.016715581255036836, 0.00035923209422856263, 0.04692155351938929, 0.016046353024569745, 1.9999299314673452e-05, 0.0012703788445227653, 0.00010034699311291354, 0.0003325753751028354, 0.0863015147371859, 0.8412255140737062, 0.035314435433658004, 0.0003466879198883485, 0.04672437847140255, 0.020598942526623396, 0.03138215277414517, 0.06311656294015146, 0.16915758389571572, 0.018329434307616934, 0.018429149167798732, 0.0009573404116975137, 0.0007279542336458705, 0.045718197314420575, 0.0007698264248074498, 0.033791280109997116, 0.02585155394540583, 0.01693227764988244, 3.351934514526797e-05, 0.0005442495333666092, 0.0005558447602022427, 0.0563074904530597, 0.0008948678738089797, 0.03141200921360738, 0.01826122412017685, 0.0008949051168063129, 0.04784121126826306, 0.0003676829089073754, 0.032389933714145475, 0.018435791521706987, 0.0005704638146096265, 0.9180648957028958, 0.0169190790709879, 0.06689865071678494, 0.039898404950601776, 0.2996709638075558, 0.01652373700233391, 0.001166835769141716, 0.0005462461895284236, 0.0004040484392920495, 0.03640123882483865, 0.08387538940415022, 0.00036652575553067766, 0.0002107701935435868, 0.05422445207145151, 0.021034812640492482, 0.00017804436105929653, 0.03201672624258654, 0.017753273028320728, 0.00015393342061492655, 0.037715444240410415, 0.01944877003370576, 0.05564548384656694, 0.0007176007617984702, 0.01881215232901902, 0.04203171954219195, 0.0004931854463415218, 0.01883012226726168, 0.001030064517039493, 0.12882515770857506, 0.06856564600298082, 0.00040195562967428413, 0.03164688043140422, 0.020057567399541308, 0.04161125333344681, 0.01760584281377843, 0.04574589273977703, 0.020957922666900004, 0.0021496883411404014, 0.001200892996070054, 0.030733631262990193, 0.40917614895591103, 0.06263014898591063, 0.0012013042799923124, 0.00015718333163665399, 0.0192413593080739]}, {\"label\": \"Number of Layers\", \"range\": [1, 4], \"values\": [1, 1, 1, 2, 2, 3, 3, 2, 3, 3, 4, 1, 1, 1, 1, 2, 1, 2, 2, 4, 2, 2, 4, 4, 2, 3, 4, 3, 3, 3, 3, 4, 3, 3, 4, 4, 3, 2, 3, 2, 3, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 2, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 2, 4, 4, 2, 4, 4, 3, 3, 4, 4, 3, 3, 1, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]}], \"labelangle\": 30, \"labelside\": \"bottom\", \"line\": {\"color\": [3.41186261177063, 3.41186261177063, 1573.8834228515625, 40182.33984375, 1528.2066650390625, 3.41186261177063, 5028653.5, 1902.826171875, 4562251.5, 945.5894165039062, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.5610835552215576, 1.2982209920883179, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1996253.125, 3.41186261177063, 1.3412549495697021, 3.41186261177063, 1.3456010818481445, 208.45639038085938, 1.3210997581481934, 3.41186261177063, 1.6091002225875854, 1.3304294347763062, 297.8522033691406, 1.4888664484024048, 1.4887771606445312, 1.3360676765441895, 1.3630410432815552, 3.41186261177063, 1.6488354206085205, 1.3287392854690552, 1.4895340204238892, 3.41186261177063, 3.41186261177063, 1.5344265699386597, 3.41186261177063, 3.41186261177063, 1.440764307975769, 1.488864779472351, 1.488818883895874, 1.466086745262146, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.3408962488174438, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.3349838256835938, 3.41186261177063, 7.432951927185059, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.3247721195220947, 1.4888302087783813, 1.322178602218628, 1.4888978004455566, 1.4888209104537964, 3.41186261177063, 2.3582754135131836, 1.3386236429214478, 3.41186261177063, 1.49014151096344, 1.488823652267456, 3.41186261177063, 1.3245543241500854, 1.323976993560791, 1.4888335466384888, 3.41186261177063, 3.9560186862945557, 3.41186261177063, 2.909315347671509, 1.3193601369857788, 3.41186261177063, 1.511308193206787, 1.3136441707611084, 3.41186261177063, 3.41186261177063, 1.4888402223587036, 1.4888317584991455, 3.41186261177063, 3.41186261177063, 1.3589725494384766, 2.0364224910736084, 3.41186261177063, 1.3351562023162842, 1.333082675933838, 3.41186261177063, 1.345519781112671, 3.41186261177063, 3.41186261177063, 1.3363773822784424, 3.41186261177063, 1.3386261463165283, 3.41186261177063, 1.3352710008621216, 1.4969419240951538, 3.41186261177063, 1.3299638032913208, 1.4888179302215576, 499.7135009765625, 1.488878846168518, 1.4904865026474, 1.3302356004714966, 1.3491047620773315, 3.41186261177063, 1.3281259536743164, 1.4885554313659668, 1.4888970851898193, 3.41186261177063, 1.34572434425354, 1.3183053731918335, 1.4915050268173218, 3.41186261177063, 1.490088939666748, 1.4888391494750977, 1.3113908767700195, 3.41186261177063, 2.1569273471832275, 1.325134038925171, 1.4888319969177246, 3.41186261177063, 2.6820173263549805, 1.4889917373657227, 1.3089126348495483, 1.308632254600525, 3.41186261177063, 1.4888687133789062, 1.3131179809570312, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.4888315200805664, 1.4690502882003784, 1.5091662406921387, 1.3258752822875977, 3.41186261177063, 1.4888314008712769, 1.4705278873443604, 4818.158203125, 1.488825798034668, 1.484520673751831, 3.41186261177063, 1.48581063747406, 1.4793022871017456, 3.41186261177063, 2.6853699684143066, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.5001596212387085, 1.3194295167922974, 1.327432632446289, 1.3218836784362793, 1.3434199094772339, 3.41186261177063, 1.3185940980911255, 1.4976372718811035, 1.3232676982879639, 3.41186261177063, 3.41186261177063, 1.315920352935791, 3.41186261177063, 3.41186261177063, 1.3266558647155762, 3.41186261177063, 1.3477894067764282, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.4888219833374023, 1.373160719871521, 1.3271931409835815, 3.41186261177063, 1.4715697765350342, 1.3213223218917847, 3.41186261177063, 1.308161973953247, 1.4888219833374023, 1.488818645477295, 1.4889020919799805, 1.4888558387756348, 2.69138503074646, 1.3204686641693115, 1.3125131130218506, 3.41186261177063, 1.4888322353363037, 1.3331835269927979, 415.56671142578125, 1.581402063369751, 3.41186261177063, 1.3211685419082642, 1.3021266460418701, 1.4888198375701904, 1.325307011604309, 1.329646110534668, 1.4888436794281006, 1.3281971216201782, 1.4888861179351807, 3.41186261177063, 1.305359959602356, 1.3265137672424316, 1.3640973567962646, 1.3347446918487549, 3.41186261177063, 1.3504081964492798, 1.3332096338272095, 3.41186261177063, 1.4888545274734497, 3.4115991592407227, 3.41186261177063, 3.41186261177063, 1.3193230628967285, 1.4888228178024292, 1.3274098634719849, 1.3126106262207031, 1.3173880577087402, 1.3275312185287476, 1.322373628616333, 1.489281415939331, 3.41186261177063, 1.310310959815979, 1.4888172149658203, 1.3176379203796387, 3.41186261177063, 3.41186261177063, 1.329596757888794, 1.32630455493927, 2.0398926734924316, 1.488019585609436, 1.3448506593704224, 1.48855721950531, 1.4942303895950317, 1.4888358116149902, 311.04400634765625, 1.3175146579742432, 1.3329596519470215, 1.488847255706787, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.4967288970947266, 1.4313173294067383, 1.4887478351593018, 3.41186261177063, 1.8477803468704224, 1.4888261556625366, 1.4888958930969238, 1.3229091167449951, 1.332504153251648, 1.3610163927078247, 3.41186261177063, 1.3288545608520508, 1.3301451206207275, 3.41186261177063, 3.41186261177063, 1.4888267517089844, 3.41186261177063, 3.4117937088012695, 1.3516267538070679, 1.3335763216018677, 3.41186261177063, 199.16970825195312, 1.489451289176941, 1.3276795148849487, 1.334709882736206, 1.3188152313232422, 3.41186261177063, 3.41186261177063, 1.3202966451644897, 1.3358441591262817, 3.41186261177063, 1.4888269901275635, 3.41186261177063, 1.3066781759262085, 3.41186261177063, 1.3137255907058716, 3.41186261177063, 3.41186261177063, 1.3406171798706055, 3.41186261177063, 1.2999054193496704, 1.3272432088851929, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.3045551776885986, 3.41186261177063, 1.4952431917190552, 2675.79052734375, 1.3019428253173828, 1.48887038230896, 1.3364481925964355, 1.32117760181427, 3.41186261177063, 1.3121179342269897, 3.41186261177063, 1.4888906478881836, 3.41186261177063, 1.3226450681686401, 3.41186261177063, 1.4888287782669067, 1.320836067199707, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.3085274696350098, 1.3093986511230469, 3.41186261177063, 1.488820195198059, 1.317246675491333, 1.4921292066574097, 1.326299786567688, 1.3326469659805298, 3.41186261177063, 1.488829255104065, 3.41186261177063, 1.3266675472259521, 1.491834282875061, 1.3059812784194946, 3.41186261177063, 696.8388061523438, 1.4321969747543335, 1.4888207912445068, 3.41186261177063, 1.311875820159912, 3.41186261177063, 1.3288514614105225, 1.3099689483642578, 1.313618540763855, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.488821268081665, 1.3155453205108643, 3.411557912826538, 3.41186261177063, 1.3217459917068481, 3.41186261177063, 1.4906315803527832, 3.41186261177063, 1.3453859090805054, 1.4888182878494263, 3.41186261177063, 1.314800500869751, 3.41186261177063, 1.3281633853912354, 3.41186261177063, 1.321211814880371, 1102.6551513671875, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.3377939462661743, 3.41186261177063, 3.41186261177063, 1.3136341571807861, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.3284739255905151, 3.41186261177063, 1.3212107419967651, 1.3475548028945923, 3.41186261177063, 3.41186261177063, 1.3276013135910034, 3.41186261177063, 1.3147908449172974, 1.4900482892990112, 1.4910539388656616, 1.4888460636138916, 1.3363206386566162, 3.41186261177063, 3.41186261177063, 1.4887561798095703, 1.3011691570281982, 57.89400863647461, 3.41186261177063, 3.41186261177063, 2.024686813354492, 1.4956120252609253, 1.4888505935668945, 1.488817572593689, 1.4888553619384766, 1.3047354221343994, 3.41186261177063, 1.488818645477295, 1.3213609457015991, 2.043469190597534, 1.3281235694885254, 1.3031138181686401, 1.4892010688781738, 14561242.0, 3.41186261177063, 1.2996032238006592, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.488633155822754, 3.41186261177063, 1.488843560218811, 1.4442107677459717, 1.415286660194397, 3.41186261177063, 1.4373067617416382, 3181.039794921875, 3.41186261177063, 3.41186261177063, 1.3226823806762695, 1.3124511241912842, 1.3017604351043701, 3.41186261177063, 1.6777530908584595, 1.4891026020050049, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.3141649961471558, 3.41186261177063, 3.41186261177063, 1.3052650690078735, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 1.4536466598510742, 1.3093748092651367, 1.316675066947937, 3.41186261177063, 3.41186261177063, 1.313673973083496, 4.355731964111328, 3.41186261177063, 3.41186261177063, 1.319928765296936, 3.41186261177063, 1.4888230562210083, 1.3414974212646484, 3.41186261177063, 1.488836646080017, 3.41186261177063, 2.672865629196167, 3.41186261177063, 3.41186261177063, 1.4835532903671265, 3.41186261177063, 1.3185737133026123, 1.503530740737915, 1.4888479709625244, 1.3155088424682617, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 3.41186261177063, 2.0430803298950195, 1.3163092136383057, 3.41186261177063, 1.492780327796936, 3.41186261177063, 1.4853918552398682, 1.3153973817825317, 226.0158233642578], \"colorbar\": {\"title\": {\"text\": \"Objective Value\"}}, \"colorscale\": [[0.0, \"rgb(247,251,255)\"], [0.125, \"rgb(222,235,247)\"], [0.25, \"rgb(198,219,239)\"], [0.375, \"rgb(158,202,225)\"], [0.5, \"rgb(107,174,214)\"], [0.625, \"rgb(66,146,198)\"], [0.75, \"rgb(33,113,181)\"], [0.875, \"rgb(8,81,156)\"], [1.0, \"rgb(8,48,107)\"]], \"reversescale\": true, \"showscale\": true}, \"type\": \"parcoords\"}],\n",
              "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Parallel Coordinate Plot\"}},\n",
              "                        {\"responsive\": true}\n",
              "                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('4da333a4-edd7-4ced-a4de-49a20b8ed5f8');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })\n",
              "                };\n",
              "                \n",
              "            </script>\n",
              "        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "output_type": "display_data"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AfLpb8KTJwjK"
      },
      "source": [
        "# SVM 1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1jxVvaHmJxuU"
      },
      "source": [
        "network_hparams.update({'LC_weights_path':'/content/drive/My Drive/LCNet/BioLCNet_layer1_Shallow_f13_s3_inh100_norm3.pth'})\n",
        "train_hparams.update({'n_train' : 1,})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NNRRaZWJKlia",
        "outputId": "dcfe6168-0a4d-40f6-accd-c49982b01d2e"
      },
      "source": [
        "from google.colab import drive\n",
        "if network_hparams['save_path'] or network_hparams['LC_weights_path']:    \n",
        "    drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168,
          "referenced_widgets": [
            "72836c8033994ae08c6542d426b2746d",
            "59a07da32df4407caef87ed495e52506",
            "54ca608f2b134bc99cbe8552617aae11",
            "9a69bcb494be4448ac833bf86e647937",
            "363ac6728f3c421cbfc738c8695c2d13",
            "8fa60c3185ed4196b91d8f78d758da21",
            "92de25edbf6a46ef8118dd42b4902551",
            "7bec0305213245cbb64121e1cd6957b7",
            "b9cb8a95bb8d4034a362b1995b2e3607",
            "cbaa198b9b184cbf9212f1726a295d08",
            "17f30272a75e4f948a94926e052e6317",
            "42f941495e98470aabd084db3e9a3eb9",
            "a0dd44abe23543cd96a3cd55dccee40b",
            "c7ca306e81134bb0913b099265b23f8b",
            "5117bc33a6514d2cb652c2202275bb77",
            "f089e106cc564148be9738fff31a506a",
            "5edd981b809e4dd18120b74d41ee02bc",
            "45627b0c9c8c4c11bd13d7237ea1f606",
            "7c481db05ae94f3097a55194076efa08",
            "fb77a66894e0404c9252853fb8d413aa",
            "46dd37f63e0f4c22910ed49c4a113caf",
            "c65f8dd3a44648999c32829d724e86d2"
          ]
        },
        "id": "KGX4j9JkKlib",
        "outputId": "6b62bc29-03e3-4a71-a7cb-21646b63f782"
      },
      "source": [
        "manual_seed(SEED)\n",
        "hparams = {**reward_hparams, **network_hparams, **train_hparams, **data_hparams}\n",
        "net = LCNet(**hparams, reward_fn = DynamicDopamineInjection)\n",
        "net.fit(dataloader = dataloader, val_loader = val_loader, reward_hparams = reward_hparams, **train_hparams)\n",
        "\n",
        "S, V, y = net.predict(\n",
        "    val_loader= val_loader,\n",
        "    n_pred= 9999,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Weights loaded ...\n",
            "Previous model not found! Training from the beginning...\n",
            "\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "72836c8033994ae08c6542d426b2746d",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "output tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) pred_label: 0 GT: 0 , Acc Rew: 0.0 Pos dps: 1.00000, Neg dps: 1.00000, Rew base: 1.00000, Pun base: 1.00000, RPe: 0.000 input_mean_fire_freq: 31.6,main_mean_fire_freq:8.5 output_mean_fire_freq:19.6 mean_lc1_w: 0.25000, mean_fc_w:0.49627 std_lc1_w: 0.23600, std_fc_w:0.28884 convergence_lc1: 0.29705, convergence_fc: 0.83342"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "42f941495e98470aabd084db3e9a3eb9",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/9999 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pw1Vnfy9MaSH",
        "outputId": "907b846d-4a2a-4e43-a3bb-eb850411af5e"
      },
      "source": [
        "def SVM(layer, monitor):\n",
        "    X = np.array([i[layer].squeeze(1).view(i[layer].shape[0],-1).sum(0).numpy()  for i in monitor])\n",
        "    y_array = np.array(y)\n",
        "\n",
        "    X_train, X_test, y_train, y_test = train_test_split(\n",
        "        X, y_array, test_size=0.2, random_state=42,\n",
        "    )\n",
        "\n",
        "    classifier = svm.SVC()\n",
        "    classifier.fit(X_train, y_train)\n",
        "    \n",
        "    return accuracy_score(y_test, classifier.predict(X_test))\n",
        "\n",
        "for l in range(len(S[0])):\n",
        "    print(f'* Layer {l}')\n",
        "    print(f\"Accuracy for spikes:\", SVM(l, S))\n",
        "    print(f\"Accuracy for voltage:\", SVM(l, V))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "* Layer 0\n",
            "Accuracy for spikes: 0.8755\n",
            "Accuracy for voltage: 0.814\n",
            "* Layer 1\n",
            "Accuracy for spikes: 0.444\n",
            "Accuracy for voltage: 0.4195\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KNOECmaLatEA"
      },
      "source": [
        "# SVM 2 "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lf9NiKrratEK"
      },
      "source": [
        "network_hparams.update({'LC_weights_path':'/content/drive/My Drive/LCNet/BioLCNet_layer1_Shallow_f15_s4_inh100_norm25_ch100.pth',})\n",
        "train_hparams.update({'n_train' : 0,})"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WJlY1mVHatEL",
        "outputId": "8a44b0eb-1985-4b86-fd40-bf0df32d649d"
      },
      "source": [
        "from google.colab import drive\n",
        "if network_hparams['save_path'] or network_hparams['LC_weights_path']:    \n",
        "    drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168,
          "referenced_widgets": [
            "8cb31b589fa54ec8aac36b3d743c4279",
            "7f964331909644939f4588898f8d8615",
            "57c3e62c98c94722a1f9706baa0c9b41",
            "583f8b41e1654c33919b91a426a1972d",
            "9f966103708045cfad08fb46eeaf66fc",
            "f9328e8aa83f4d47a4efb20a0653170b",
            "dc744a7d0f584d16be34b7eb84777b29",
            "d4bd844ebcda4602ba647dd7c49b8eb8",
            "a560cd9b689c40908c436f60aef51c45",
            "bf5ec280c0154d9a9b5b6ae90510a50d",
            "e56dc87bab894f24a33992ca26cb4a53",
            "e685c0338a924fd99854fe746c465fbd",
            "b8784123251b494e8168539d729b03e0",
            "c6bb9806056449aa9268e17a450a6160",
            "57e9d087a1334938baf4a41746c0a7f7",
            "97ec30ae621548cbb5a719627b013db5",
            "133177cdcdfc45b39cd55b352111701c",
            "c6b4f8b150d4485a820d4157d7ec8f01",
            "bbd852e8b05b42459358a7831c04eaf1",
            "e04253b377ab48509b15eabf5b33fc16",
            "35a0d2183ad84959a3be7cb6825dd9b4",
            "3f0b2e4761e7498eb378f4685b65daf1"
          ]
        },
        "id": "gzCb1V1-atEL",
        "outputId": "b939ec01-f143-45fa-ef9b-65126c0e1785"
      },
      "source": [
        "manual_seed(SEED)\n",
        "hparams = {**reward_hparams, **network_hparams, **train_hparams, **data_hparams}\n",
        "net = LCNet(**hparams, reward_fn = DynamicDopamineInjection)\n",
        "net.fit(dataloader = dataloader, val_loader = val_loader, reward_hparams = reward_hparams, **train_hparams)\n",
        "\n",
        "S, V, y = net.predict(\n",
        "    val_loader= val_loader,\n",
        "    n_pred= 9999,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Weights loaded ...\n",
            "Previous model not found! Training from the beginning...\n",
            "\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8cb31b589fa54ec8aac36b3d743c4279",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "0it [00:00, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\routput tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) pred_label: 0 GT: 5 , Acc Rew: 0.0 Pos dps: 1.00000, Neg dps: 1.00000, Rew base: 1.00000, Pun base: 1.00000, RPe: 0.000 input_mean_fire_freq: 28.0,main_mean_fire_freq:4.0 output_mean_fire_freq:11.6 mean_lc1_w: 0.25000, mean_fc_w:0.50152 std_lc1_w: 0.23792, std_fc_w:0.28744 convergence_lc1: 0.30191, convergence_fc: 0.83260"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e685c0338a924fd99854fe746c465fbd",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/9999 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I85639K_atEL",
        "outputId": "79c641c0-5674-46ed-a83b-d7cf1dbca271"
      },
      "source": [
        "def SVM(layer, monitor):\n",
        "    X = np.array([i[layer].squeeze(1).view(i[layer].shape[0],-1).sum(0).numpy()  for i in monitor])\n",
        "    y_array = np.array(y)\n",
        "\n",
        "    X_train, X_test, y_train, y_test = train_test_split(\n",
        "        X, y_array, test_size=0.2, random_state=42,\n",
        "    )\n",
        "\n",
        "    classifier = svm.SVC()\n",
        "    classifier.fit(X_train, y_train)\n",
        "    \n",
        "    return accuracy_score(y_test, classifier.predict(X_test))\n",
        "\n",
        "for l in range(len(S[0])):\n",
        "    print(f'* Layer {l}')\n",
        "    print(f\"Accuracy for spikes:\", SVM(l, S))\n",
        "    print(f\"Accuracy for voltage:\", SVM(l, V))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "* Layer 0\n",
            "Accuracy for spikes: 0.833\n",
            "Accuracy for voltage: 0.7495\n"
          ]
        }
      ]
    }
  ]
}