Ray_with_default_hp_space.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3",
      "language": "python"
    },
    "language_info": {
      "name": "python",
      "version": "3.7.9",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "colab": {
      "name": "Ray_with_default_hp_space.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "47ee740c9e6e41c48746b9ea8f32cb50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_313b173eed994d0788ae25468d425e08",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_df40fcb64a364fbc99bc34e531bb525b",
              "IPY_MODEL_8101f90fa66c4bdca10d69da7d8c9893"
            ]
          }
        },
        "313b173eed994d0788ae25468d425e08": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "df40fcb64a364fbc99bc34e531bb525b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7805b487814548a4a244f1369e791883",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1404983,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1404983,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bfcbb0bcf4e34c73b967990814f8ca59"
          }
        },
        "8101f90fa66c4bdca10d69da7d8c9893": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_71bbdc8aaf2a472584745ebbfae5c482",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.40M/1.40M [00:01&lt;00:00, 959kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b824f7b026da43ee9fd3cece5b9aa8fd"
          }
        },
        "7805b487814548a4a244f1369e791883": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bfcbb0bcf4e34c73b967990814f8ca59": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "71bbdc8aaf2a472584745ebbfae5c482": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b824f7b026da43ee9fd3cece5b9aa8fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b7dc47aaa05a412a98b419c144de9f00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fba952b3f9e54cb7917a29a4a9c3122b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_11db1e857b0e4c24ac2583a5271ab4ce",
              "IPY_MODEL_70ad65925d8c4878bd5a59050768f649"
            ]
          }
        },
        "fba952b3f9e54cb7917a29a4a9c3122b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "11db1e857b0e4c24ac2583a5271ab4ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_14d758a39bd441f5a2bead90fa37e4aa",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c9191d0f8c044a2e850d69bf7edfa1ba"
          }
        },
        "70ad65925d8c4878bd5a59050768f649": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9cafe182531340158a7394af7d180356",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [00:00&lt;00:00, 243B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fb7b5c6b598d43aca839c1359a64362f"
          }
        },
        "14d758a39bd441f5a2bead90fa37e4aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c9191d0f8c044a2e850d69bf7edfa1ba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9cafe182531340158a7394af7d180356": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fb7b5c6b598d43aca839c1359a64362f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7f7d90acfb40415fa263c0089787d6f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_abeb92aae10c47a8b71737d7803a4ab1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4516b2cdddf14186b0dcbdcea3166417",
              "IPY_MODEL_80347e74c3754ad28bbbf7c8427f0c5b"
            ]
          }
        },
        "abeb92aae10c47a8b71737d7803a4ab1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4516b2cdddf14186b0dcbdcea3166417": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2409888af9ee4875a6584a1452a61461",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 24,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 24,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_68801766d05d4bf7988a38d80240d757"
          }
        },
        "80347e74c3754ad28bbbf7c8427f0c5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_65ab289dcf4844899047460e60b4d6c5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 24.0/24.0 [00:00&lt;00:00, 73.1B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b3a25c9d7fb44230a43333c63283f8a9"
          }
        },
        "2409888af9ee4875a6584a1452a61461": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "68801766d05d4bf7988a38d80240d757": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "65ab289dcf4844899047460e60b4d6c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b3a25c9d7fb44230a43333c63283f8a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b43a08e44cb645d7bb9f2427959ef67d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5255caebd72c4784ac61cdcda037ee2c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_956560d259134797b23daf4bdb05731e",
              "IPY_MODEL_828b52073dbd438f8cb565d117625b2b"
            ]
          }
        },
        "5255caebd72c4784ac61cdcda037ee2c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "956560d259134797b23daf4bdb05731e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9070f3b4d33540f4bcc915fa77a5ccc8",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 642,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 642,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_504c5714ef12411597cadef250a88eea"
          }
        },
        "828b52073dbd438f8cb565d117625b2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_eeb55c80e104493fabf3cf0118f8b6d6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 642/642 [00:32&lt;00:00, 19.6B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5393f973878145d289a8a4895608a1f3"
          }
        },
        "9070f3b4d33540f4bcc915fa77a5ccc8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "504c5714ef12411597cadef250a88eea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "eeb55c80e104493fabf3cf0118f8b6d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5393f973878145d289a8a4895608a1f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e293bb85fd2840b5ba53ef023aee4ae3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_90d9e0c8b6494bfca2f46d46bfd4987a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_92c6508849724690943691d18c9c9c80",
              "IPY_MODEL_454dd9b892014461ba9f07df5577b879"
            ]
          }
        },
        "90d9e0c8b6494bfca2f46d46bfd4987a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "92c6508849724690943691d18c9c9c80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_62072b2e73f34736a971df34a889f316",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 711456782,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 711456782,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c0fb8eaf5b724e52988b6cb4682afeec"
          }
        },
        "454dd9b892014461ba9f07df5577b879": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2beddee102f745b3a095426bf3d8f6ef",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 711M/711M [00:17&lt;00:00, 41.3MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f8ebb44126c645748ec8e2e45ad79c6d"
          }
        },
        "62072b2e73f34736a971df34a889f316": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c0fb8eaf5b724e52988b6cb4682afeec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2beddee102f745b3a095426bf3d8f6ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f8ebb44126c645748ec8e2e45ad79c6d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/olgasem10/Unsafe/blob/main/Ray_with_default_hp_space.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "40eBULRBxCJ-"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "URyjDhkhzPAZ"
      },
      "source": [
        "!pip install ray[tune]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "8W4GCnPhwYRU"
      },
      "source": [
        "from transformers import BertTokenizer, BertForSequenceClassification, Trainer, TrainingArguments\n",
        "import torch\n",
        "from torch.utils.data import Dataset\n",
        "from sklearn.metrics import accuracy_score, precision_recall_fscore_support, roc_auc_score\n",
        "import numpy as np\n",
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K0jUe33twYRb"
      },
      "source": [
        "### Данные (с добавлением спец токена)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AD13wCHKw0-Z",
        "outputId": "4bac51a8-4f31-4b65-d798-26a05f0f957b"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "gDkrwx_fwYRc"
      },
      "source": [
        "train_df = pd.read_csv('/content/drive/MyDrive/internship/data/train_randst0.csv')\n",
        "test_df = pd.read_csv('/content/drive/MyDrive/internship/data/val_randst0.csv')\n",
        "#train_df = train_df[[\"text\", \"unsafe\"]]\n",
        "#test_df = test_df[[\"text\", \"unsafe\"]]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "NQfSJcEfwYRc"
      },
      "source": [
        "train_df = train_df.loc[(train_df['unsafe'] >= 0.8) | (train_df['unsafe'] <= 0.2)]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "ageVXh5cwYRd"
      },
      "source": [
        "translator = {\n",
        "    'crime_real' : 'криминал',\n",
        "    'crime_web' : 'веб-криминал',\n",
        "    'drugs' : 'наркотики',\n",
        "    'gambling' : 'азартные игры',\n",
        "    'pornography' : 'порнография',\n",
        "    'prostitution' : 'проституция',\n",
        "    'slavery' : 'рабство',\n",
        "    'suicide' : 'суицид',\n",
        "    'terrorism' : 'терроризм',\n",
        "    'weapons' : 'оружие', \n",
        "    'body_shaming' : 'уродства',\n",
        "    'halth_shaming' : 'инвалидность',\n",
        "    'politics' : 'политика',\n",
        "    'racism' : 'расизм',\n",
        "    'religion' : 'религия',\n",
        "    'sex_minorities' : 'гомосексуализм',\n",
        "    'sexism' : 'сексизм',\n",
        "    'social' : 'социальное',\n",
        "    'none' : ''\n",
        "}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "IMSsdZ23wYRd"
      },
      "source": [
        "col_names = ['crime_real',\n",
        " 'crime_web',\n",
        " 'drugs',\n",
        " 'gambling',\n",
        " 'pornography',\n",
        " 'prostitution',\n",
        " 'slavery',\n",
        " 'suicide',\n",
        " 'terrorism',\n",
        " 'weapons',\n",
        " 'body_shaming',\n",
        " 'halth_shaming',\n",
        " 'politics',\n",
        " 'racism',\n",
        " 'religion',\n",
        " 'sex_minorities',\n",
        " 'sexism',\n",
        " 'social']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "Qqal9_MswYRd"
      },
      "source": [
        "def binary(val):\n",
        "    return round(val)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "Ecz9np7AwYRe"
      },
      "source": [
        "for name in col_names:\n",
        "    train_df[name] = train_df[name].apply(binary)\n",
        "    test_df[name] = test_df[name].apply(binary)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "TClIDbMgwYRe"
      },
      "source": [
        "train_df['unsafe'] = train_df['unsafe'].apply(binary)\n",
        "test_df['unsafe'] = test_df['unsafe'].apply(binary)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "8rSn7SWAwYRe"
      },
      "source": [
        "def add_special_tokens(dataframe):\n",
        "    new_dataframe = dataframe.copy()\n",
        "    for column in col_names:\n",
        "        indices = dataframe.index[dataframe[column] == 1]\n",
        "        new_dataframe.loc[indices, ['text']] = '<' + translator[column] + '> ' + dataframe.loc[indices, ['text']].astype(str)\n",
        "    return new_dataframe"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "ZGgZBs4VwYRf"
      },
      "source": [
        "train_df = add_special_tokens(train_df)\n",
        "test_df = add_special_tokens(test_df)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "HFilo2apwYRf"
      },
      "source": [
        "train_df = train_df.sample(frac=1)\n",
        "test_df = test_df.sample(frac=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "ckpk_F6bwYRg"
      },
      "source": [
        "x_train = train_df['text'].tolist()#[:15000]\n",
        "y_train = train_df['unsafe'].tolist()#[:15000]\n",
        "x_test = test_df['text'].tolist()#[:2000]\n",
        "y_test = test_df['unsafe'].tolist()#[:2000]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "Kh1AUG-bwYRh"
      },
      "source": [
        "class UnsafeData(Dataset):\n",
        "\n",
        "    def __init__(self, texts, targets, tokenizer, max_len):\n",
        "        \n",
        "        super().__init__()\n",
        "        \n",
        "        self.texts = texts\n",
        "        self.targets = targets        \n",
        "        self.max_len = max_len\n",
        "        self.tokenizer = tokenizer\n",
        "\n",
        "    def __len__(self):\n",
        "        \n",
        "        return len(self.texts)\n",
        "\n",
        "    \n",
        "    def __getitem__(self, index):\n",
        "        x = self.texts[index]\n",
        "\n",
        "        enc_dict = self.tokenizer(x, truncation=True, max_length=self.max_len, padding='max_length')\n",
        "      \n",
        "        item = {key: torch.tensor(val).long() for key, val in enc_dict.items()}\n",
        "        item['labels'] = torch.tensor(self.targets[index]).long()\n",
        "\n",
        "        return item "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZCKuigpQwYRi"
      },
      "source": [
        "### Обучение"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "xSsL1BnjwYRi"
      },
      "source": [
        "def compute_metrics(pred):\n",
        "    labels = pred.label_ids\n",
        "    preds = pred.predictions.argmax(-1)\n",
        "    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='weighted')\n",
        "    acc = accuracy_score(labels, preds)\n",
        "    return {\n",
        "        'accuracy': acc,\n",
        "        'f1': f1,\n",
        "        'precision': precision,\n",
        "        'recall': recall\n",
        "    }"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "tjl4ELuIwYRj"
      },
      "source": [
        "model_name = 'DeepPavlov/rubert-base-cased-conversational'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "OHc35n4owYRj",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164,
          "referenced_widgets": [
            "47ee740c9e6e41c48746b9ea8f32cb50",
            "313b173eed994d0788ae25468d425e08",
            "df40fcb64a364fbc99bc34e531bb525b",
            "8101f90fa66c4bdca10d69da7d8c9893",
            "7805b487814548a4a244f1369e791883",
            "bfcbb0bcf4e34c73b967990814f8ca59",
            "71bbdc8aaf2a472584745ebbfae5c482",
            "b824f7b026da43ee9fd3cece5b9aa8fd",
            "b7dc47aaa05a412a98b419c144de9f00",
            "fba952b3f9e54cb7917a29a4a9c3122b",
            "11db1e857b0e4c24ac2583a5271ab4ce",
            "70ad65925d8c4878bd5a59050768f649",
            "14d758a39bd441f5a2bead90fa37e4aa",
            "c9191d0f8c044a2e850d69bf7edfa1ba",
            "9cafe182531340158a7394af7d180356",
            "fb7b5c6b598d43aca839c1359a64362f",
            "7f7d90acfb40415fa263c0089787d6f6",
            "abeb92aae10c47a8b71737d7803a4ab1",
            "4516b2cdddf14186b0dcbdcea3166417",
            "80347e74c3754ad28bbbf7c8427f0c5b",
            "2409888af9ee4875a6584a1452a61461",
            "68801766d05d4bf7988a38d80240d757",
            "65ab289dcf4844899047460e60b4d6c5",
            "b3a25c9d7fb44230a43333c63283f8a9"
          ]
        },
        "outputId": "f2b3b457-b3d4-4b33-c35a-455d8fa39b96"
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained(model_name)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "47ee740c9e6e41c48746b9ea8f32cb50",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1404983.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b7dc47aaa05a412a98b419c144de9f00",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=112.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7f7d90acfb40415fa263c0089787d6f6",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=24.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "L8eiVMBBwYRh"
      },
      "source": [
        "train_dataset = UnsafeData(x_train, y_train, tokenizer, max_len = 60)\n",
        "test_dataset = UnsafeData(x_test, y_test, tokenizer, max_len = 60)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VWf8M4B9wYRi",
        "outputId": "e53b2690-92aa-45bb-a195-1b4bdadcdb72"
      },
      "source": [
        "len(train_dataset), len(test_dataset)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(120120, 24501)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c8Jo4T66wYRi",
        "outputId": "e3c58af2-af39-4a46-9f68-a10f4e8db4d4"
      },
      "source": [
        "train_dataset[10000]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "         1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n",
              " 'input_ids': tensor([  101,   160, 21749,   130, 48931,   164, 40051,   477,   507,   465,\n",
              "         58496,   439, 28019, 14646, 32260,  8297, 41387,   128,  1007,  1151,\n",
              "         19644,   340,   362,  5703,   166,   102,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0]),\n",
              " 'labels': tensor(1),\n",
              " 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "Gy8nTG0LwYRj"
      },
      "source": [
        "def model_init():\n",
        "    return BertForSequenceClassification.from_pretrained(model_name, return_dict=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 149,
          "referenced_widgets": [
            "b43a08e44cb645d7bb9f2427959ef67d",
            "5255caebd72c4784ac61cdcda037ee2c",
            "956560d259134797b23daf4bdb05731e",
            "828b52073dbd438f8cb565d117625b2b",
            "9070f3b4d33540f4bcc915fa77a5ccc8",
            "504c5714ef12411597cadef250a88eea",
            "eeb55c80e104493fabf3cf0118f8b6d6",
            "5393f973878145d289a8a4895608a1f3",
            "e293bb85fd2840b5ba53ef023aee4ae3",
            "90d9e0c8b6494bfca2f46d46bfd4987a",
            "92c6508849724690943691d18c9c9c80",
            "454dd9b892014461ba9f07df5577b879",
            "62072b2e73f34736a971df34a889f316",
            "c0fb8eaf5b724e52988b6cb4682afeec",
            "2beddee102f745b3a095426bf3d8f6ef",
            "f8ebb44126c645748ec8e2e45ad79c6d"
          ]
        },
        "id": "r55TgRD4wYRk",
        "outputId": "926afbbf-48c5-48b2-a012-a079127e587d"
      },
      "source": [
        "training_args = TrainingArguments(\n",
        "    output_dir='/bert1',\n",
        "    logging_steps = 600,\n",
        "    evaluation_strategy = 'steps',\n",
        "    logging_dir='/bert1/logs',\n",
        "    do_eval=True,\n",
        "    save_steps = 10000,\n",
        "    save_total_limit = 2,\n",
        "    disable_tqdm=True\n",
        ")\n",
        "\n",
        "trainer = Trainer(\n",
        "    args=training_args,\n",
        "    tokenizer=tokenizer,\n",
        "    train_dataset=train_dataset,\n",
        "    eval_dataset=test_dataset,\n",
        "    model_init=model_init,\n",
        "    compute_metrics=compute_metrics,\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b43a08e44cb645d7bb9f2427959ef67d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=642.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e293bb85fd2840b5ba53ef023aee4ae3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=711456782.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "o6DUgdQewYRk"
      },
      "source": [
        "### По какой метрике оптимизировать\n",
        "\n",
        "def my_func(metrics):\n",
        "    return metrics['eval_f1']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WMEkcPS49inB"
      },
      "source": [
        "import ray"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o8hysqOS91jQ"
      },
      "source": [
        "from ray.tune.suggest.hyperopt import HyperOptSearch\n",
        "from ray.tune.schedulers import AsyncHyperBandScheduler"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ip7B4iRsfGg8",
        "outputId": "b69566e1-8bfd-44d5-a1fb-bc332d6221d3"
      },
      "source": [
        "best_run = trainer.hyperparameter_search(\n",
        "    n_trials=20,\n",
        "    compute_objective = my_func,\n",
        "    direction=\"maximize\",\n",
        "    backend = 'ray',\n",
        "    search_alg=HyperOptSearch(metric = 'eval_f1', mode = 'max'),\n",
        "    scheduler=AsyncHyperBandScheduler(metric = 'eval_f1', mode = 'max')\n",
        "    )"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-03-18 08:38:57,950\tINFO services.py:1174 -- View the Ray dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265\u001b[39m\u001b[22m\n",
            "2021-03-18 08:39:03,399\tWARNING function_runner.py:541 -- Function checkpointing is disabled. This may result in unexpected behavior when using checkpointing features or certain schedulers. To enable, set the train function arguments to be `func(config, checkpoint_dir=None)`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "== Status ==\n",
            "Memory usage on this node: 3.8/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=0\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: None\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 1/20 (1 RUNNING)\n",
            "+-----------------+----------+-------+-----------------+--------------------+-------------------------------+--------+\n",
            "| Trial name      | status   | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |   seed |\n",
            "|-----------------+----------+-------+-----------------+--------------------+-------------------------------+--------|\n",
            "| _inner_64e82530 | RUNNING  |       |     9.69583e-05 |                  1 |                            64 | 34.923 |\n",
            "+-----------------+----------+-------+-----------------+--------------------+-------------------------------+--------+\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=366)\u001b[0m 2021-03-18 08:39:04.677449: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=366)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "\u001b[2m\u001b[36m(pid=366)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=366)\u001b[0m {'loss': 0.416, 'learning_rate': 6.596470393691822e-05, 'epoch': 0.32}\n",
            "Result for _inner_64e82530:\n",
            "  date: 2021-03-18_08-48-56\n",
            "  done: false\n",
            "  epoch: 0.32\n",
            "  eval_accuracy: 0.7911513815762622\n",
            "  eval_f1: 0.785431839440515\n",
            "  eval_loss: 0.4543006420135498\n",
            "  eval_precision: 0.7839843424239313\n",
            "  eval_recall: 0.7911513815762622\n",
            "  eval_runtime: 106.523\n",
            "  eval_samples_per_second: 230.007\n",
            "  experiment_id: fc3e59f0abfc4481a821dc7f9db32f3a\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.785431839440515\n",
            "  pid: 366\n",
            "  time_since_restore: 592.135365486145\n",
            "  time_this_iter_s: 592.135365486145\n",
            "  time_total_s: 592.135365486145\n",
            "  timestamp: 1616057336\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 64e82530\n",
            "  \n",
            "\u001b[2m\u001b[36m(pid=366)\u001b[0m {'eval_loss': 0.4543006420135498, 'eval_accuracy': 0.7911513815762622, 'eval_f1': 0.785431839440515, 'eval_precision': 0.7839843424239313, 'eval_recall': 0.7911513815762622, 'eval_runtime': 106.523, 'eval_samples_per_second': 230.007, 'epoch': 0.32}\n",
            "== Status ==\n",
            "Memory usage on this node: 5.6/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=0\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.785431839440515\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 2/20 (1 PENDING, 1 RUNNING)\n",
            "+-----------------+----------+----------------+-----------------+--------------------+-------------------------------+---------+-------------+\n",
            "| Trial name      | status   | loc            |   learning_rate |   num_train_epochs |   per_device_train_batch_size |    seed |   objective |\n",
            "|-----------------+----------+----------------+-----------------+--------------------+-------------------------------+---------+-------------|\n",
            "| _inner_64e82530 | RUNNING  | 172.28.0.2:366 |     9.69583e-05 |                  1 |                            64 | 34.923  |    0.785432 |\n",
            "| _inner_64ed099c | PENDING  |                |     4.3896e-06  |                  2 |                            64 | 20.7908 |             |\n",
            "+-----------------+----------+----------------+-----------------+--------------------+-------------------------------+---------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=366)\u001b[0m {'loss': 0.3799, 'learning_rate': 3.497110772536698e-05, 'epoch': 0.64}\n",
            "Result for _inner_64e82530:\n",
            "  date: 2021-03-18_08-58-16\n",
            "  done: false\n",
            "  epoch: 0.64\n",
            "  eval_accuracy: 0.798661279131464\n",
            "  eval_f1: 0.7945523213139225\n",
            "  eval_loss: 0.45182445645332336\n",
            "  eval_precision: 0.7929966575767785\n",
            "  eval_recall: 0.798661279131464\n",
            "  eval_runtime: 106.4955\n",
            "  eval_samples_per_second: 230.066\n",
            "  experiment_id: fc3e59f0abfc4481a821dc7f9db32f3a\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 2\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7945523213139225\n",
            "  pid: 366\n",
            "  time_since_restore: 1152.1794655323029\n",
            "  time_this_iter_s: 560.0441000461578\n",
            "  time_total_s: 1152.1794655323029\n",
            "  timestamp: 1616057896\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 2\n",
            "  trial_id: 64e82530\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.6/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=0\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.785431839440515\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 2/20 (1 PENDING, 1 RUNNING)\n",
            "+-----------------+----------+----------------+-----------------+--------------------+-------------------------------+---------+-------------+\n",
            "| Trial name      | status   | loc            |   learning_rate |   num_train_epochs |   per_device_train_batch_size |    seed |   objective |\n",
            "|-----------------+----------+----------------+-----------------+--------------------+-------------------------------+---------+-------------|\n",
            "| _inner_64e82530 | RUNNING  | 172.28.0.2:366 |     9.69583e-05 |                  1 |                            64 | 34.923  |    0.794552 |\n",
            "| _inner_64ed099c | PENDING  |                |     4.3896e-06  |                  2 |                            64 | 20.7908 |             |\n",
            "+-----------------+----------+----------------+-----------------+--------------------+-------------------------------+---------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=366)\u001b[0m {'eval_loss': 0.45182445645332336, 'eval_accuracy': 0.798661279131464, 'eval_f1': 0.7945523213139225, 'eval_precision': 0.7929966575767785, 'eval_recall': 0.798661279131464, 'eval_runtime': 106.4955, 'eval_samples_per_second': 230.066, 'epoch': 0.64}\n",
            "\u001b[2m\u001b[36m(pid=366)\u001b[0m {'loss': 0.3556, 'learning_rate': 3.977511513815742e-06, 'epoch': 0.96}\n",
            "Result for _inner_64e82530:\n",
            "  date: 2021-03-18_09-07-35\n",
            "  done: false\n",
            "  epoch: 0.96\n",
            "  eval_accuracy: 0.8077221337904575\n",
            "  eval_f1: 0.8023533089303417\n",
            "  eval_loss: 0.42341551184654236\n",
            "  eval_precision: 0.8015727778008387\n",
            "  eval_recall: 0.8077221337904575\n",
            "  eval_runtime: 105.7755\n",
            "  eval_samples_per_second: 231.632\n",
            "  experiment_id: fc3e59f0abfc4481a821dc7f9db32f3a\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 3\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.8023533089303417\n",
            "  pid: 366\n",
            "  time_since_restore: 1711.6883635520935\n",
            "  time_this_iter_s: 559.5088980197906\n",
            "  time_total_s: 1711.6883635520935\n",
            "  timestamp: 1616058455\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 3\n",
            "  trial_id: 64e82530\n",
            "  \n",
            "\u001b[2m\u001b[36m(pid=366)\u001b[0m {'eval_loss': 0.42341551184654236, 'eval_accuracy': 0.8077221337904575, 'eval_f1': 0.8023533089303417, 'eval_precision': 0.8015727778008387, 'eval_recall': 0.8077221337904575, 'eval_runtime': 105.7755, 'eval_samples_per_second': 231.632, 'epoch': 0.96}\n",
            "== Status ==\n",
            "Memory usage on this node: 5.6/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=0\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.785431839440515\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 2/20 (1 PENDING, 1 RUNNING)\n",
            "+-----------------+----------+----------------+-----------------+--------------------+-------------------------------+---------+-------------+\n",
            "| Trial name      | status   | loc            |   learning_rate |   num_train_epochs |   per_device_train_batch_size |    seed |   objective |\n",
            "|-----------------+----------+----------------+-----------------+--------------------+-------------------------------+---------+-------------|\n",
            "| _inner_64e82530 | RUNNING  | 172.28.0.2:366 |     9.69583e-05 |                  1 |                            64 | 34.923  |    0.802353 |\n",
            "| _inner_64ed099c | PENDING  |                |     4.3896e-06  |                  2 |                            64 | 20.7908 |             |\n",
            "+-----------------+----------+----------------+-----------------+--------------------+-------------------------------+---------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=366)\u001b[0m {'train_runtime': 1737.4156, 'train_samples_per_second': 1.08, 'epoch': 1.0}\n",
            "Result for _inner_64e82530:\n",
            "  date: 2021-03-18_09-07-35\n",
            "  done: true\n",
            "  epoch: 0.96\n",
            "  eval_accuracy: 0.8077221337904575\n",
            "  eval_f1: 0.8023533089303417\n",
            "  eval_loss: 0.42341551184654236\n",
            "  eval_precision: 0.8015727778008387\n",
            "  eval_recall: 0.8077221337904575\n",
            "  eval_runtime: 105.7755\n",
            "  eval_samples_per_second: 231.632\n",
            "  experiment_id: fc3e59f0abfc4481a821dc7f9db32f3a\n",
            "  experiment_tag: 1_learning_rate=9.6958e-05,num_train_epochs=1,per_device_train_batch_size=64,seed=34.923\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 3\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.8023533089303417\n",
            "  pid: 366\n",
            "  time_since_restore: 1711.6883635520935\n",
            "  time_this_iter_s: 559.5088980197906\n",
            "  time_total_s: 1711.6883635520935\n",
            "  timestamp: 1616058455\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 3\n",
            "  trial_id: 64e82530\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=0\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.785431839440515\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 2/20 (1 PENDING, 1 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+---------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |    seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+---------+-------------|\n",
            "| _inner_64ed099c | PENDING    |       |     4.3896e-06  |                  2 |                            64 | 20.7908 |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923  |    0.802353 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+---------+-------------+\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=365)\u001b[0m 2021-03-18 09:08:35.052139: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=365)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=365)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=365)\u001b[0m {'loss': 0.4555, 'learning_rate': 3.6880129419720295e-06, 'epoch': 0.32}\n",
            "Result for _inner_64ed099c:\n",
            "  date: 2021-03-18_09-18-06\n",
            "  done: true\n",
            "  epoch: 0.32\n",
            "  eval_accuracy: 0.779274315334068\n",
            "  eval_f1: 0.775543251821145\n",
            "  eval_loss: 0.4566640853881836\n",
            "  eval_precision: 0.773544832350675\n",
            "  eval_recall: 0.779274315334068\n",
            "  eval_runtime: 106.0888\n",
            "  eval_samples_per_second: 230.948\n",
            "  experiment_id: eb2f0b32d64d450c8e264d788cf33e40\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.775543251821145\n",
            "  pid: 365\n",
            "  time_since_restore: 572.2759852409363\n",
            "  time_this_iter_s: 572.2759852409363\n",
            "  time_total_s: 572.2759852409363\n",
            "  timestamp: 1616059086\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 64ed099c\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7829596925356725\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 3/20 (1 PENDING, 2 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_840f43a4 | PENDING    |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=365)\u001b[0m {'eval_loss': 0.4566640853881836, 'eval_accuracy': 0.779274315334068, 'eval_f1': 0.775543251821145, 'eval_precision': 0.773544832350675, 'eval_recall': 0.779274315334068, 'eval_runtime': 106.0888, 'eval_samples_per_second': 230.948, 'epoch': 0.32}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m 2021-03-18 09:18:08.105663: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'loss': 0.4306, 'learning_rate': 8.97041979294247e-06, 'epoch': 0.32}\n",
            "Result for _inner_840f43a4:\n",
            "  date: 2021-03-18_09-27-39\n",
            "  done: false\n",
            "  epoch: 0.32\n",
            "  eval_accuracy: 0.7954777356026285\n",
            "  eval_f1: 0.7955374245293948\n",
            "  eval_loss: 0.43590760231018066\n",
            "  eval_precision: 0.7955977535603731\n",
            "  eval_recall: 0.7954777356026285\n",
            "  eval_runtime: 105.9471\n",
            "  eval_samples_per_second: 231.257\n",
            "  experiment_id: c774e48d278340ea894e2a1eb511ff2a\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7955374245293948\n",
            "  pid: 622\n",
            "  time_since_restore: 572.3672208786011\n",
            "  time_this_iter_s: 572.3672208786011\n",
            "  time_total_s: 572.3672208786011\n",
            "  timestamp: 1616059659\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 840f43a4\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7904846319849549\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 4/20 (1 PENDING, 1 RUNNING, 2 TERMINATED)\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc            |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_840f43a4 | RUNNING    | 172.28.0.2:622 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.795537 |\n",
            "| _inner_d969198c | PENDING    |                |     1.2724e-05  |                  2 |                             4 | 12.8396  |             |\n",
            "| _inner_64e82530 | TERMINATED |                |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'eval_loss': 0.43590760231018066, 'eval_accuracy': 0.7954777356026285, 'eval_f1': 0.7955374245293948, 'eval_precision': 0.7955977535603731, 'eval_recall': 0.7954777356026285, 'eval_runtime': 105.9471, 'eval_samples_per_second': 231.257, 'epoch': 0.32}\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'loss': 0.3683, 'learning_rate': 8.191286632003633e-06, 'epoch': 0.64}\n",
            "Result for _inner_840f43a4:\n",
            "  date: 2021-03-18_09-36-59\n",
            "  done: false\n",
            "  epoch: 0.64\n",
            "  eval_accuracy: 0.8026611158728215\n",
            "  eval_f1: 0.8033545641045866\n",
            "  eval_loss: 0.42182549834251404\n",
            "  eval_precision: 0.8041459570740566\n",
            "  eval_recall: 0.8026611158728215\n",
            "  eval_runtime: 105.8306\n",
            "  eval_samples_per_second: 231.511\n",
            "  experiment_id: c774e48d278340ea894e2a1eb511ff2a\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 2\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.8033545641045866\n",
            "  pid: 622\n",
            "  time_since_restore: 1131.5592699050903\n",
            "  time_this_iter_s: 559.1920490264893\n",
            "  time_total_s: 1131.5592699050903\n",
            "  timestamp: 1616060219\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 2\n",
            "  trial_id: 840f43a4\n",
            "  \n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'eval_loss': 0.42182549834251404, 'eval_accuracy': 0.8026611158728215, 'eval_f1': 0.8033545641045866, 'eval_precision': 0.8041459570740566, 'eval_recall': 0.8026611158728215, 'eval_runtime': 105.8306, 'eval_samples_per_second': 231.511, 'epoch': 0.64}\n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7904846319849549\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 4/20 (1 PENDING, 1 RUNNING, 2 TERMINATED)\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc            |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_840f43a4 | RUNNING    | 172.28.0.2:622 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.803355 |\n",
            "| _inner_d969198c | PENDING    |                |     1.2724e-05  |                  2 |                             4 | 12.8396  |             |\n",
            "| _inner_64e82530 | TERMINATED |                |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'loss': 0.3499, 'learning_rate': 7.4121534710647974e-06, 'epoch': 0.96}\n",
            "Result for _inner_840f43a4:\n",
            "  date: 2021-03-18_09-46-18\n",
            "  done: false\n",
            "  epoch: 0.96\n",
            "  eval_accuracy: 0.8114362679074324\n",
            "  eval_f1: 0.8064079256446829\n",
            "  eval_loss: 0.419907808303833\n",
            "  eval_precision: 0.8056388590991841\n",
            "  eval_recall: 0.8114362679074324\n",
            "  eval_runtime: 106.3667\n",
            "  eval_samples_per_second: 230.345\n",
            "  experiment_id: c774e48d278340ea894e2a1eb511ff2a\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 3\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.8064079256446829\n",
            "  pid: 622\n",
            "  time_since_restore: 1690.728686094284\n",
            "  time_this_iter_s: 559.1694161891937\n",
            "  time_total_s: 1690.728686094284\n",
            "  timestamp: 1616060778\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 3\n",
            "  trial_id: 840f43a4\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7904846319849549\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 4/20 (1 PENDING, 1 RUNNING, 2 TERMINATED)\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc            |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_840f43a4 | RUNNING    | 172.28.0.2:622 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.806408 |\n",
            "| _inner_d969198c | PENDING    |                |     1.2724e-05  |                  2 |                             4 | 12.8396  |             |\n",
            "| _inner_64e82530 | TERMINATED |                |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'eval_loss': 0.419907808303833, 'eval_accuracy': 0.8114362679074324, 'eval_f1': 0.8064079256446829, 'eval_precision': 0.8056388590991841, 'eval_recall': 0.8114362679074324, 'eval_runtime': 106.3667, 'eval_samples_per_second': 230.345, 'epoch': 0.96}\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'loss': 0.285, 'learning_rate': 6.633020310125961e-06, 'epoch': 1.28}\n",
            "Result for _inner_840f43a4:\n",
            "  date: 2021-03-18_09-55-37\n",
            "  done: false\n",
            "  epoch: 1.28\n",
            "  eval_accuracy: 0.8017223786784213\n",
            "  eval_f1: 0.8052292738674598\n",
            "  eval_loss: 0.44215127825737\n",
            "  eval_precision: 0.8118165906887549\n",
            "  eval_recall: 0.8017223786784213\n",
            "  eval_runtime: 106.1293\n",
            "  eval_samples_per_second: 230.86\n",
            "  experiment_id: c774e48d278340ea894e2a1eb511ff2a\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 4\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.8052292738674598\n",
            "  pid: 622\n",
            "  time_since_restore: 2250.2449655532837\n",
            "  time_this_iter_s: 559.5162794589996\n",
            "  time_total_s: 2250.2449655532837\n",
            "  timestamp: 1616061337\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 4\n",
            "  trial_id: 840f43a4\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7904846319849549\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 4/20 (1 PENDING, 1 RUNNING, 2 TERMINATED)\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc            |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_840f43a4 | RUNNING    | 172.28.0.2:622 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.805229 |\n",
            "| _inner_d969198c | PENDING    |                |     1.2724e-05  |                  2 |                             4 | 12.8396  |             |\n",
            "| _inner_64e82530 | TERMINATED |                |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'eval_loss': 0.44215127825737, 'eval_accuracy': 0.8017223786784213, 'eval_f1': 0.8052292738674598, 'eval_precision': 0.8118165906887549, 'eval_recall': 0.8017223786784213, 'eval_runtime': 106.1293, 'eval_samples_per_second': 230.86, 'epoch': 1.28}\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'loss': 0.2779, 'learning_rate': 5.853887149187125e-06, 'epoch': 1.6}\n",
            "Result for _inner_840f43a4:\n",
            "  date: 2021-03-18_10-04-56\n",
            "  done: false\n",
            "  epoch: 1.6\n",
            "  eval_accuracy: 0.8085384270029795\n",
            "  eval_f1: 0.8097445800300684\n",
            "  eval_loss: 0.442513108253479\n",
            "  eval_precision: 0.8112809292897267\n",
            "  eval_recall: 0.8085384270029795\n",
            "  eval_runtime: 106.0134\n",
            "  eval_samples_per_second: 231.112\n",
            "  experiment_id: c774e48d278340ea894e2a1eb511ff2a\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 5\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.8097445800300684\n",
            "  pid: 622\n",
            "  time_since_restore: 2809.128294467926\n",
            "  time_this_iter_s: 558.8833289146423\n",
            "  time_total_s: 2809.128294467926\n",
            "  timestamp: 1616061896\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 5\n",
            "  trial_id: 840f43a4\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7904846319849549\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 4/20 (1 PENDING, 1 RUNNING, 2 TERMINATED)\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc            |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_840f43a4 | RUNNING    | 172.28.0.2:622 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809745 |\n",
            "| _inner_d969198c | PENDING    |                |     1.2724e-05  |                  2 |                             4 | 12.8396  |             |\n",
            "| _inner_64e82530 | TERMINATED |                |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'eval_loss': 0.442513108253479, 'eval_accuracy': 0.8085384270029795, 'eval_f1': 0.8097445800300684, 'eval_precision': 0.8112809292897267, 'eval_recall': 0.8085384270029795, 'eval_runtime': 106.0134, 'eval_samples_per_second': 231.112, 'epoch': 1.6}\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'loss': 0.2768, 'learning_rate': 5.074753988248288e-06, 'epoch': 1.92}\n",
            "Result for _inner_840f43a4:\n",
            "  date: 2021-03-18_10-14-14\n",
            "  done: false\n",
            "  epoch: 1.92\n",
            "  eval_accuracy: 0.811273009264928\n",
            "  eval_f1: 0.8113390763113286\n",
            "  eval_loss: 0.4396231174468994\n",
            "  eval_precision: 0.8114060929991246\n",
            "  eval_recall: 0.811273009264928\n",
            "  eval_runtime: 106.0575\n",
            "  eval_samples_per_second: 231.016\n",
            "  experiment_id: c774e48d278340ea894e2a1eb511ff2a\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 6\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.8113390763113286\n",
            "  pid: 622\n",
            "  time_since_restore: 3367.4932436943054\n",
            "  time_this_iter_s: 558.3649492263794\n",
            "  time_total_s: 3367.4932436943054\n",
            "  timestamp: 1616062454\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 6\n",
            "  trial_id: 840f43a4\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7904846319849549\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 4/20 (1 PENDING, 1 RUNNING, 2 TERMINATED)\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc            |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_840f43a4 | RUNNING    | 172.28.0.2:622 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.811339 |\n",
            "| _inner_d969198c | PENDING    |                |     1.2724e-05  |                  2 |                             4 | 12.8396  |             |\n",
            "| _inner_64e82530 | TERMINATED |                |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'eval_loss': 0.4396231174468994, 'eval_accuracy': 0.811273009264928, 'eval_f1': 0.8113390763113286, 'eval_precision': 0.8114060929991246, 'eval_recall': 0.811273009264928, 'eval_runtime': 106.0575, 'eval_samples_per_second': 231.016, 'epoch': 1.92}\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'loss': 0.2227, 'learning_rate': 4.295620827309451e-06, 'epoch': 2.24}\n",
            "Result for _inner_840f43a4:\n",
            "  date: 2021-03-18_10-23-34\n",
            "  done: false\n",
            "  epoch: 2.24\n",
            "  eval_accuracy: 0.8130688543324762\n",
            "  eval_f1: 0.8096918413531685\n",
            "  eval_loss: 0.5000341534614563\n",
            "  eval_precision: 0.8084189675349479\n",
            "  eval_recall: 0.8130688543324762\n",
            "  eval_runtime: 106.5714\n",
            "  eval_samples_per_second: 229.902\n",
            "  experiment_id: c774e48d278340ea894e2a1eb511ff2a\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 7\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.8096918413531685\n",
            "  pid: 622\n",
            "  time_since_restore: 3927.0575239658356\n",
            "  time_this_iter_s: 559.5642802715302\n",
            "  time_total_s: 3927.0575239658356\n",
            "  timestamp: 1616063014\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 7\n",
            "  trial_id: 840f43a4\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7904846319849549\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 4/20 (1 PENDING, 1 RUNNING, 2 TERMINATED)\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc            |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_840f43a4 | RUNNING    | 172.28.0.2:622 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809692 |\n",
            "| _inner_d969198c | PENDING    |                |     1.2724e-05  |                  2 |                             4 | 12.8396  |             |\n",
            "| _inner_64e82530 | TERMINATED |                |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'eval_loss': 0.5000341534614563, 'eval_accuracy': 0.8130688543324762, 'eval_f1': 0.8096918413531685, 'eval_precision': 0.8084189675349479, 'eval_recall': 0.8130688543324762, 'eval_runtime': 106.5714, 'eval_samples_per_second': 229.902, 'epoch': 2.24}\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'loss': 0.204, 'learning_rate': 3.516487666370615e-06, 'epoch': 2.56}\n",
            "Result for _inner_840f43a4:\n",
            "  date: 2021-03-18_10-32-53\n",
            "  done: false\n",
            "  epoch: 2.56\n",
            "  eval_accuracy: 0.8128647810293458\n",
            "  eval_f1: 0.8105744414664698\n",
            "  eval_loss: 0.5079146027565002\n",
            "  eval_precision: 0.8093019922335257\n",
            "  eval_recall: 0.8128647810293458\n",
            "  eval_runtime: 106.192\n",
            "  eval_samples_per_second: 230.724\n",
            "  experiment_id: c774e48d278340ea894e2a1eb511ff2a\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 8\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.8105744414664698\n",
            "  pid: 622\n",
            "  time_since_restore: 4486.35379076004\n",
            "  time_this_iter_s: 559.2962667942047\n",
            "  time_total_s: 4486.35379076004\n",
            "  timestamp: 1616063573\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 8\n",
            "  trial_id: 840f43a4\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7904846319849549\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 4/20 (1 PENDING, 1 RUNNING, 2 TERMINATED)\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc            |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_840f43a4 | RUNNING    | 172.28.0.2:622 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.810574 |\n",
            "| _inner_d969198c | PENDING    |                |     1.2724e-05  |                  2 |                             4 | 12.8396  |             |\n",
            "| _inner_64e82530 | TERMINATED |                |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'eval_loss': 0.5079146027565002, 'eval_accuracy': 0.8128647810293458, 'eval_f1': 0.8105744414664698, 'eval_precision': 0.8093019922335257, 'eval_recall': 0.8128647810293458, 'eval_runtime': 106.192, 'eval_samples_per_second': 230.724, 'epoch': 2.56}\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'loss': 0.2031, 'learning_rate': 2.7373545054317787e-06, 'epoch': 2.88}\n",
            "Result for _inner_840f43a4:\n",
            "  date: 2021-03-18_10-42-12\n",
            "  done: false\n",
            "  epoch: 2.88\n",
            "  eval_accuracy: 0.8060079180441615\n",
            "  eval_f1: 0.8068818626817139\n",
            "  eval_loss: 0.5047711133956909\n",
            "  eval_precision: 0.8079198957752116\n",
            "  eval_recall: 0.8060079180441615\n",
            "  eval_runtime: 106.2091\n",
            "  eval_samples_per_second: 230.686\n",
            "  experiment_id: c774e48d278340ea894e2a1eb511ff2a\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 9\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.8068818626817139\n",
            "  pid: 622\n",
            "  time_since_restore: 5045.071979999542\n",
            "  time_this_iter_s: 558.718189239502\n",
            "  time_total_s: 5045.071979999542\n",
            "  timestamp: 1616064132\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 9\n",
            "  trial_id: 840f43a4\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7904846319849549\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 4/20 (1 PENDING, 1 RUNNING, 2 TERMINATED)\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc            |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_840f43a4 | RUNNING    | 172.28.0.2:622 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.806882 |\n",
            "| _inner_d969198c | PENDING    |                |     1.2724e-05  |                  2 |                             4 | 12.8396  |             |\n",
            "| _inner_64e82530 | TERMINATED |                |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'eval_loss': 0.5047711133956909, 'eval_accuracy': 0.8060079180441615, 'eval_f1': 0.8068818626817139, 'eval_precision': 0.8079198957752116, 'eval_recall': 0.8060079180441615, 'eval_runtime': 106.2091, 'eval_samples_per_second': 230.686, 'epoch': 2.88}\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'loss': 0.1763, 'learning_rate': 1.958221344492942e-06, 'epoch': 3.2}\n",
            "Result for _inner_840f43a4:\n",
            "  date: 2021-03-18_10-51-31\n",
            "  done: false\n",
            "  epoch: 3.2\n",
            "  eval_accuracy: 0.8114362679074324\n",
            "  eval_f1: 0.807863640425493\n",
            "  eval_loss: 0.5588476061820984\n",
            "  eval_precision: 0.8065801188416969\n",
            "  eval_recall: 0.8114362679074324\n",
            "  eval_runtime: 106.0946\n",
            "  eval_samples_per_second: 230.935\n",
            "  experiment_id: c774e48d278340ea894e2a1eb511ff2a\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 10\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.807863640425493\n",
            "  pid: 622\n",
            "  time_since_restore: 5604.325802087784\n",
            "  time_this_iter_s: 559.2538220882416\n",
            "  time_total_s: 5604.325802087784\n",
            "  timestamp: 1616064691\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 10\n",
            "  trial_id: 840f43a4\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7904846319849549\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 4/20 (1 PENDING, 1 RUNNING, 2 TERMINATED)\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc            |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_840f43a4 | RUNNING    | 172.28.0.2:622 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.807864 |\n",
            "| _inner_d969198c | PENDING    |                |     1.2724e-05  |                  2 |                             4 | 12.8396  |             |\n",
            "| _inner_64e82530 | TERMINATED |                |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'eval_loss': 0.5588476061820984, 'eval_accuracy': 0.8114362679074324, 'eval_f1': 0.807863640425493, 'eval_precision': 0.8065801188416969, 'eval_recall': 0.8114362679074324, 'eval_runtime': 106.0946, 'eval_samples_per_second': 230.935, 'epoch': 3.2}\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'loss': 0.1552, 'learning_rate': 1.1790881835541057e-06, 'epoch': 3.52}\n",
            "Result for _inner_840f43a4:\n",
            "  date: 2021-03-18_11-00-49\n",
            "  done: false\n",
            "  epoch: 3.52\n",
            "  eval_accuracy: 0.811273009264928\n",
            "  eval_f1: 0.80905342656767\n",
            "  eval_loss: 0.5660959482192993\n",
            "  eval_precision: 0.8077750767707143\n",
            "  eval_recall: 0.811273009264928\n",
            "  eval_runtime: 105.7962\n",
            "  eval_samples_per_second: 231.587\n",
            "  experiment_id: c774e48d278340ea894e2a1eb511ff2a\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 11\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.80905342656767\n",
            "  pid: 622\n",
            "  time_since_restore: 6162.055418729782\n",
            "  time_this_iter_s: 557.7296166419983\n",
            "  time_total_s: 6162.055418729782\n",
            "  timestamp: 1616065249\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 11\n",
            "  trial_id: 840f43a4\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7904846319849549\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 4/20 (1 PENDING, 1 RUNNING, 2 TERMINATED)\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc            |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_840f43a4 | RUNNING    | 172.28.0.2:622 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809053 |\n",
            "| _inner_d969198c | PENDING    |                |     1.2724e-05  |                  2 |                             4 | 12.8396  |             |\n",
            "| _inner_64e82530 | TERMINATED |                |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'eval_loss': 0.5660959482192993, 'eval_accuracy': 0.811273009264928, 'eval_f1': 0.80905342656767, 'eval_precision': 0.8077750767707143, 'eval_recall': 0.811273009264928, 'eval_runtime': 105.7962, 'eval_samples_per_second': 231.587, 'epoch': 3.52}\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'loss': 0.1529, 'learning_rate': 3.9995502261526936e-07, 'epoch': 3.84}\n",
            "Result for _inner_840f43a4:\n",
            "  date: 2021-03-18_11-10-08\n",
            "  done: false\n",
            "  epoch: 3.84\n",
            "  eval_accuracy: 0.811273009264928\n",
            "  eval_f1: 0.8092747207548171\n",
            "  eval_loss: 0.5693864822387695\n",
            "  eval_precision: 0.808048081742711\n",
            "  eval_recall: 0.811273009264928\n",
            "  eval_runtime: 106.6171\n",
            "  eval_samples_per_second: 229.804\n",
            "  experiment_id: c774e48d278340ea894e2a1eb511ff2a\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 12\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.8092747207548171\n",
            "  pid: 622\n",
            "  time_since_restore: 6720.966800451279\n",
            "  time_this_iter_s: 558.9113817214966\n",
            "  time_total_s: 6720.966800451279\n",
            "  timestamp: 1616065808\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 12\n",
            "  trial_id: 840f43a4\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7904846319849549\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 4/20 (1 PENDING, 1 RUNNING, 2 TERMINATED)\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc            |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_840f43a4 | RUNNING    | 172.28.0.2:622 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | PENDING    |                |     1.2724e-05  |                  2 |                             4 | 12.8396  |             |\n",
            "| _inner_64e82530 | TERMINATED |                |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "+-----------------+------------+----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'eval_loss': 0.5693864822387695, 'eval_accuracy': 0.811273009264928, 'eval_f1': 0.8092747207548171, 'eval_precision': 0.808048081742711, 'eval_recall': 0.811273009264928, 'eval_runtime': 106.6171, 'eval_samples_per_second': 229.804, 'epoch': 3.84}\n",
            "\u001b[2m\u001b[36m(pid=622)\u001b[0m {'train_runtime': 6940.1684, 'train_samples_per_second': 1.082, 'epoch': 4.0}\n",
            "Result for _inner_840f43a4:\n",
            "  date: 2021-03-18_11-10-08\n",
            "  done: true\n",
            "  epoch: 3.84\n",
            "  eval_accuracy: 0.811273009264928\n",
            "  eval_f1: 0.8092747207548171\n",
            "  eval_loss: 0.5693864822387695\n",
            "  eval_precision: 0.808048081742711\n",
            "  eval_recall: 0.811273009264928\n",
            "  eval_runtime: 106.6171\n",
            "  eval_samples_per_second: 229.804\n",
            "  experiment_id: c774e48d278340ea894e2a1eb511ff2a\n",
            "  experiment_tag: 3_learning_rate=9.7496e-06,num_train_epochs=4,per_device_train_batch_size=64,seed=2.6125\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 12\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.8092747207548171\n",
            "  pid: 622\n",
            "  time_since_restore: 6720.966800451279\n",
            "  time_this_iter_s: 558.9113817214966\n",
            "  time_total_s: 6720.966800451279\n",
            "  timestamp: 1616065808\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 12\n",
            "  trial_id: 840f43a4\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.4/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7904846319849549\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 4/20 (1 PENDING, 3 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_d969198c | PENDING    |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1005)\u001b[0m 2021-03-18 11:14:01.911007: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1005)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "\u001b[2m\u001b[36m(pid=1005)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1005)\u001b[0m {'loss': 0.547, 'learning_rate': 1.2596918810674001e-05, 'epoch': 0.02}\n",
            "Result for _inner_d969198c:\n",
            "  date: 2021-03-18_11-17-10\n",
            "  done: true\n",
            "  epoch: 0.02\n",
            "  eval_accuracy: 0.7520917513570875\n",
            "  eval_f1: 0.754770574051757\n",
            "  eval_loss: 0.5321484208106995\n",
            "  eval_precision: 0.7583827877567276\n",
            "  eval_recall: 0.7520917513570875\n",
            "  eval_runtime: 105.8019\n",
            "  eval_samples_per_second: 231.574\n",
            "  experiment_id: 93b29119b7fb428fb7c9b6e34d8fdff4\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.754770574051757\n",
            "  pid: 1005\n",
            "  time_since_restore: 188.94078063964844\n",
            "  time_this_iter_s: 188.94078063964844\n",
            "  time_total_s: 188.94078063964844\n",
            "  timestamp: 1616066230\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: d969198c\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=2\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7879582357127349\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 5/20 (1 PENDING, 4 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_0a467ba2 | PENDING    |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1005)\u001b[0m {'eval_loss': 0.5321484208106995, 'eval_accuracy': 0.7520917513570875, 'eval_f1': 0.754770574051757, 'eval_precision': 0.7583827877567276, 'eval_recall': 0.7520917513570875, 'eval_runtime': 105.8019, 'eval_samples_per_second': 231.574, 'epoch': 0.02}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1036)\u001b[0m 2021-03-18 11:17:11.748895: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1036)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "\u001b[2m\u001b[36m(pid=1036)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1036)\u001b[0m {'loss': 0.5045, 'learning_rate': 1.6740117933655895e-06, 'epoch': 0.32}\n",
            "Result for _inner_0a467ba2:\n",
            "  date: 2021-03-18_11-26-43\n",
            "  done: true\n",
            "  epoch: 0.32\n",
            "  eval_accuracy: 0.7638871882780295\n",
            "  eval_f1: 0.759186131580364\n",
            "  eval_loss: 0.48497700691223145\n",
            "  eval_precision: 0.7567443561168071\n",
            "  eval_recall: 0.7638871882780295\n",
            "  eval_runtime: 106.4962\n",
            "  eval_samples_per_second: 230.065\n",
            "  experiment_id: a95c985d22c34b1f87f94eab7c08308d\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.759186131580364\n",
            "  pid: 1036\n",
            "  time_since_restore: 572.4108521938324\n",
            "  time_this_iter_s: 572.4108521938324\n",
            "  time_total_s: 572.4108521938324\n",
            "  timestamp: 1616066803\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 0a467ba2\n",
            "  \n",
            "\u001b[2m\u001b[36m(pid=1036)\u001b[0m {'eval_loss': 0.48497700691223145, 'eval_accuracy': 0.7638871882780295, 'eval_f1': 0.759186131580364, 'eval_precision': 0.7567443561168071, 'eval_recall': 0.7638871882780295, 'eval_runtime': 106.4962, 'eval_samples_per_second': 230.065, 'epoch': 0.32}\n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=3\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.785431839440515\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 6/20 (1 PENDING, 5 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_7b639b3a | PENDING    |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1084)\u001b[0m 2021-03-18 11:26:44.973753: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1084)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=1084)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1084)\u001b[0m {'loss': 0.4568, 'learning_rate': 3.9899857373552205e-05, 'epoch': 0.08}\n",
            "Result for _inner_7b639b3a:\n",
            "  date: 2021-03-18_11-31-11\n",
            "  done: true\n",
            "  epoch: 0.08\n",
            "  eval_accuracy: 0.7758866985021019\n",
            "  eval_f1: 0.7726147654128548\n",
            "  eval_loss: 0.475066214799881\n",
            "  eval_precision: 0.7706538688558014\n",
            "  eval_recall: 0.7758866985021019\n",
            "  eval_runtime: 106.0514\n",
            "  eval_samples_per_second: 231.03\n",
            "  experiment_id: 65e0b7c19cb848cc9161487ebf55be0a\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7726147654128548\n",
            "  pid: 1084\n",
            "  time_since_restore: 266.87957429885864\n",
            "  time_this_iter_s: 266.87957429885864\n",
            "  time_total_s: 266.87957429885864\n",
            "  timestamp: 1616067071\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 7b639b3a\n",
            "  \n",
            "\u001b[2m\u001b[36m(pid=1084)\u001b[0m {'eval_loss': 0.475066214799881, 'eval_accuracy': 0.7758866985021019, 'eval_f1': 0.7726147654128548, 'eval_precision': 0.7706538688558014, 'eval_recall': 0.7758866985021019, 'eval_runtime': 106.0514, 'eval_samples_per_second': 231.03, 'epoch': 0.08}\n",
            "== Status ==\n",
            "Memory usage on this node: 5.6/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=4\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7829596925356725\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 7/20 (1 PENDING, 6 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_d11513c8 | PENDING    |       |     1.70282e-05 |                  4 |                            16 | 21.2015  |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1116)\u001b[0m 2021-03-18 11:31:12.719061: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1116)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=1116)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1116)\u001b[0m {'loss': 0.4544, 'learning_rate': 1.6687979290359695e-05, 'epoch': 0.08}\n",
            "Result for _inner_d11513c8:\n",
            "  date: 2021-03-18_11-35-38\n",
            "  done: true\n",
            "  epoch: 0.08\n",
            "  eval_accuracy: 0.7802946818497204\n",
            "  eval_f1: 0.7790611214713948\n",
            "  eval_loss: 0.4730099141597748\n",
            "  eval_precision: 0.7780422281905798\n",
            "  eval_recall: 0.7802946818497204\n",
            "  eval_runtime: 106.1958\n",
            "  eval_samples_per_second: 230.715\n",
            "  experiment_id: a948db9c2793408d84f6090bd00b51d5\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7790611214713948\n",
            "  pid: 1116\n",
            "  time_since_restore: 266.8555660247803\n",
            "  time_this_iter_s: 266.8555660247803\n",
            "  time_total_s: 266.8555660247803\n",
            "  timestamp: 1616067338\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: d11513c8\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=5\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7822464804559549\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 8/20 (1 PENDING, 7 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_70aa2522 | PENDING    |       |     6.27619e-05 |                  5 |                             8 |  4.40721 |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |       |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1116)\u001b[0m {'eval_loss': 0.4730099141597748, 'eval_accuracy': 0.7802946818497204, 'eval_f1': 0.7790611214713948, 'eval_precision': 0.7780422281905798, 'eval_recall': 0.7802946818497204, 'eval_runtime': 106.1958, 'eval_samples_per_second': 230.715, 'epoch': 0.08}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1149)\u001b[0m 2021-03-18 11:35:40.356777: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1149)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "\u001b[2m\u001b[36m(pid=1149)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1149)\u001b[0m {'loss': 0.5965, 'learning_rate': 6.226027627882286e-05, 'epoch': 0.04}\n",
            "Result for _inner_70aa2522:\n",
            "  date: 2021-03-18_11-39-09\n",
            "  done: true\n",
            "  epoch: 0.04\n",
            "  eval_accuracy: 0.7007469082894576\n",
            "  eval_f1: 0.5778172008879999\n",
            "  eval_loss: 0.6147259473800659\n",
            "  eval_precision: 0.7570543032671577\n",
            "  eval_recall: 0.7007469082894576\n",
            "  eval_runtime: 102.5888\n",
            "  eval_samples_per_second: 238.827\n",
            "  experiment_id: f7dbd53f065e48f28244faa56838d547\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.5778172008879999\n",
            "  pid: 1149\n",
            "  time_since_restore: 209.4970200061798\n",
            "  time_this_iter_s: 209.4970200061798\n",
            "  time_total_s: 209.4970200061798\n",
            "  timestamp: 1616067549\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 70aa2522\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=6\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7806538009636749\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 9/20 (1 PENDING, 8 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_103f6c14 | PENDING    |       |     2.01159e-06 |                  2 |                            16 |  5.50495 |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |       |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |       |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1149)\u001b[0m {'eval_loss': 0.6147259473800659, 'eval_accuracy': 0.7007469082894576, 'eval_f1': 0.5778172008879999, 'eval_precision': 0.7570543032671577, 'eval_recall': 0.7007469082894576, 'eval_runtime': 102.5888, 'eval_samples_per_second': 238.827, 'epoch': 0.04}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1186)\u001b[0m 2021-03-18 11:39:10.698751: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1186)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=1186)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1186)\u001b[0m {'loss': 0.5509, 'learning_rate': 1.9312080754793694e-06, 'epoch': 0.08}\n",
            "Result for _inner_103f6c14:\n",
            "  date: 2021-03-18_11-43-37\n",
            "  done: true\n",
            "  epoch: 0.08\n",
            "  eval_accuracy: 0.7485816905432431\n",
            "  eval_f1: 0.7448698035837847\n",
            "  eval_loss: 0.5084359049797058\n",
            "  eval_precision: 0.7424167325156047\n",
            "  eval_recall: 0.7485816905432431\n",
            "  eval_runtime: 106.4323\n",
            "  eval_samples_per_second: 230.203\n",
            "  experiment_id: a0e9f97d18024f3a96e456045f0eb3a0\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7448698035837847\n",
            "  pid: 1186\n",
            "  time_since_restore: 267.78470039367676\n",
            "  time_this_iter_s: 267.78470039367676\n",
            "  time_total_s: 267.78470039367676\n",
            "  timestamp: 1616067817\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 103f6c14\n",
            "  \n",
            "\u001b[2m\u001b[36m(pid=1186)\u001b[0m {'eval_loss': 0.5084359049797058, 'eval_accuracy': 0.7485816905432431, 'eval_f1': 0.7448698035837847, 'eval_precision': 0.7424167325156047, 'eval_recall': 0.7485816905432431, 'eval_runtime': 106.4323, 'eval_samples_per_second': 230.203, 'epoch': 0.08}\n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=7\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7790611214713948\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 10/20 (1 PENDING, 9 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_8d952e60 | PENDING    |       |     3.63457e-06 |                  3 |                            16 | 13.7357  |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |       |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |       |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |       |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1217)\u001b[0m 2021-03-18 11:43:39.295213: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1217)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=1217)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1217)\u001b[0m {'loss': 0.4947, 'learning_rate': 3.5377470177865622e-06, 'epoch': 0.08}\n",
            "Result for _inner_8d952e60:\n",
            "  date: 2021-03-18_11-48-06\n",
            "  done: true\n",
            "  epoch: 0.08\n",
            "  eval_accuracy: 0.7685808742500306\n",
            "  eval_f1: 0.765774947383935\n",
            "  eval_loss: 0.4782803952693939\n",
            "  eval_precision: 0.763880865160413\n",
            "  eval_recall: 0.7685808742500306\n",
            "  eval_runtime: 106.1377\n",
            "  eval_samples_per_second: 230.842\n",
            "  experiment_id: db5d9a8cce07438492b1ffe1956338dd\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.765774947383935\n",
            "  pid: 1217\n",
            "  time_since_restore: 267.5511476993561\n",
            "  time_this_iter_s: 267.5511476993561\n",
            "  time_total_s: 267.5511476993561\n",
            "  timestamp: 1616068086\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 8d952e60\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=8\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7781816540588324\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 11/20 (1 PENDING, 10 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_2db3b948 | PENDING    |       |     4.18269e-06 |                  4 |                            64 |  4.09892 |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |       |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |       |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |       |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |       |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1217)\u001b[0m {'eval_loss': 0.4782803952693939, 'eval_accuracy': 0.7685808742500306, 'eval_f1': 0.765774947383935, 'eval_precision': 0.763880865160413, 'eval_recall': 0.7685808742500306, 'eval_runtime': 106.1377, 'eval_samples_per_second': 230.842, 'epoch': 0.08}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1258)\u001b[0m 2021-03-18 11:48:07.651874: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1258)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "\u001b[2m\u001b[36m(pid=1258)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1258)\u001b[0m {'loss': 0.46, 'learning_rate': 3.848430855353389e-06, 'epoch': 0.32}\n",
            "Result for _inner_2db3b948:\n",
            "  date: 2021-03-18_11-57-39\n",
            "  done: true\n",
            "  epoch: 0.32\n",
            "  eval_accuracy: 0.7800497938859638\n",
            "  eval_f1: 0.7695705605466825\n",
            "  eval_loss: 0.45824652910232544\n",
            "  eval_precision: 0.7700263564034067\n",
            "  eval_recall: 0.7800497938859638\n",
            "  eval_runtime: 105.9522\n",
            "  eval_samples_per_second: 231.246\n",
            "  experiment_id: e045409c9fe94be4818f29f5d29b821c\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7695705605466825\n",
            "  pid: 1258\n",
            "  time_since_restore: 572.0776553153992\n",
            "  time_this_iter_s: 572.0776553153992\n",
            "  time_total_s: 572.0776553153992\n",
            "  timestamp: 1616068659\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 2db3b948\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=9\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7773021866462699\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 12/20 (1 PENDING, 11 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_cda60438 | PENDING    |       |     1.06154e-05 |                  5 |                             4 |  1.37908 |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |       |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |       |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |       |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |       |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |       |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1258)\u001b[0m {'eval_loss': 0.45824652910232544, 'eval_accuracy': 0.7800497938859638, 'eval_f1': 0.7695705605466825, 'eval_precision': 0.7700263564034067, 'eval_recall': 0.7800497938859638, 'eval_runtime': 105.9522, 'eval_samples_per_second': 231.246, 'epoch': 0.32}\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1304)\u001b[0m 2021-03-18 11:57:40.880795: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1304)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=1304)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1304)\u001b[0m {'loss': 0.5616, 'learning_rate': 1.057300031684288e-05, 'epoch': 0.02}\n",
            "Result for _inner_cda60438:\n",
            "  date: 2021-03-18_12-00-49\n",
            "  done: true\n",
            "  epoch: 0.02\n",
            "  eval_accuracy: 0.7565813640259581\n",
            "  eval_f1: 0.7243293492416838\n",
            "  eval_loss: 0.6452959775924683\n",
            "  eval_precision: 0.7465306491241352\n",
            "  eval_recall: 0.7565813640259581\n",
            "  eval_runtime: 105.9581\n",
            "  eval_samples_per_second: 231.233\n",
            "  experiment_id: 55f0f6656124440082d0f8bda3f0088e\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7243293492416838\n",
            "  pid: 1304\n",
            "  time_since_restore: 189.7422375679016\n",
            "  time_this_iter_s: 189.7422375679016\n",
            "  time_total_s: 189.7422375679016\n",
            "  timestamp: 1616068849\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: cda60438\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=10\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7764227192337074\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 13/20 (1 PENDING, 12 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_231d52e4 | PENDING    |       |     6.44559e-05 |                  2 |                            64 | 26.1482  |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |       |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |       |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |       |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |       |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |       |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "| _inner_cda60438 | TERMINATED |       |     1.06154e-05 |                  5 |                             4 |  1.37908 |    0.724329 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1304)\u001b[0m {'eval_loss': 0.6452959775924683, 'eval_accuracy': 0.7565813640259581, 'eval_f1': 0.7243293492416838, 'eval_precision': 0.7465306491241352, 'eval_recall': 0.7565813640259581, 'eval_runtime': 105.9581, 'eval_samples_per_second': 231.233, 'epoch': 0.02}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1337)\u001b[0m 2021-03-18 12:00:51.338820: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1337)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=1337)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1337)\u001b[0m {'loss': 0.4243, 'learning_rate': 5.415392320287305e-05, 'epoch': 0.32}\n",
            "Result for _inner_231d52e4:\n",
            "  date: 2021-03-18_12-10-22\n",
            "  done: false\n",
            "  epoch: 0.32\n",
            "  eval_accuracy: 0.7915187135218971\n",
            "  eval_f1: 0.7841022796601412\n",
            "  eval_loss: 0.46036624908447266\n",
            "  eval_precision: 0.7834757075029353\n",
            "  eval_recall: 0.7915187135218971\n",
            "  eval_runtime: 106.456\n",
            "  eval_samples_per_second: 230.151\n",
            "  experiment_id: e6c2ce85b95a4a4fad44194653c2ae7f\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7841022796601412\n",
            "  pid: 1337\n",
            "  time_since_restore: 571.8770935535431\n",
            "  time_this_iter_s: 571.8770935535431\n",
            "  time_total_s: 571.8770935535431\n",
            "  timestamp: 1616069422\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 231d52e4\n",
            "  \n",
            "\u001b[2m\u001b[36m(pid=1337)\u001b[0m {'eval_loss': 0.46036624908447266, 'eval_accuracy': 0.7915187135218971, 'eval_f1': 0.7841022796601412, 'eval_precision': 0.7834757075029353, 'eval_recall': 0.7915187135218971, 'eval_runtime': 106.456, 'eval_samples_per_second': 230.151, 'epoch': 0.32}\n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=10\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7790611214713948\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 14/20 (1 PENDING, 1 RUNNING, 12 TERMINATED)\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc             |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_231d52e4 | RUNNING    | 172.28.0.2:1337 |     6.44559e-05 |                  2 |                            64 | 26.1482  |    0.784102 |\n",
            "| _inner_94d158ae | PENDING    |                 |     1.66709e-05 |                  5 |                             8 | 29.3587  |             |\n",
            "| _inner_64e82530 | TERMINATED |                 |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                 |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |                 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |                 |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |                 |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |                 |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |                 |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |                 |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |                 |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |                 |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |                 |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "| _inner_cda60438 | TERMINATED |                 |     1.06154e-05 |                  5 |                             4 |  1.37908 |    0.724329 |\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1337)\u001b[0m {'loss': 0.3766, 'learning_rate': 4.385197205457761e-05, 'epoch': 0.64}\n",
            "Result for _inner_231d52e4:\n",
            "  date: 2021-03-18_12-19-41\n",
            "  done: false\n",
            "  epoch: 0.64\n",
            "  eval_accuracy: 0.7980082445614465\n",
            "  eval_f1: 0.7897340898449634\n",
            "  eval_loss: 0.44886481761932373\n",
            "  eval_precision: 0.7902752495930084\n",
            "  eval_recall: 0.7980082445614465\n",
            "  eval_runtime: 106.3754\n",
            "  eval_samples_per_second: 230.326\n",
            "  experiment_id: e6c2ce85b95a4a4fad44194653c2ae7f\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 2\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7897340898449634\n",
            "  pid: 1337\n",
            "  time_since_restore: 1130.8505651950836\n",
            "  time_this_iter_s: 558.9734716415405\n",
            "  time_total_s: 1130.8505651950836\n",
            "  timestamp: 1616069981\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 2\n",
            "  trial_id: 231d52e4\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=10\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7790611214713948\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 14/20 (1 PENDING, 1 RUNNING, 12 TERMINATED)\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc             |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_231d52e4 | RUNNING    | 172.28.0.2:1337 |     6.44559e-05 |                  2 |                            64 | 26.1482  |    0.789734 |\n",
            "| _inner_94d158ae | PENDING    |                 |     1.66709e-05 |                  5 |                             8 | 29.3587  |             |\n",
            "| _inner_64e82530 | TERMINATED |                 |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                 |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |                 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |                 |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |                 |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |                 |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |                 |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |                 |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |                 |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |                 |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |                 |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "| _inner_cda60438 | TERMINATED |                 |     1.06154e-05 |                  5 |                             4 |  1.37908 |    0.724329 |\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1337)\u001b[0m {'eval_loss': 0.44886481761932373, 'eval_accuracy': 0.7980082445614465, 'eval_f1': 0.7897340898449634, 'eval_precision': 0.7902752495930084, 'eval_recall': 0.7980082445614465, 'eval_runtime': 106.3754, 'eval_samples_per_second': 230.326, 'epoch': 0.64}\n",
            "\u001b[2m\u001b[36m(pid=1337)\u001b[0m {'loss': 0.3664, 'learning_rate': 3.355002090628216e-05, 'epoch': 0.96}\n",
            "Result for _inner_231d52e4:\n",
            "  date: 2021-03-18_12-29-00\n",
            "  done: false\n",
            "  epoch: 0.96\n",
            "  eval_accuracy: 0.7996408309864903\n",
            "  eval_f1: 0.8020994830700735\n",
            "  eval_loss: 0.43206846714019775\n",
            "  eval_precision: 0.8059045071164533\n",
            "  eval_recall: 0.7996408309864903\n",
            "  eval_runtime: 106.2638\n",
            "  eval_samples_per_second: 230.568\n",
            "  experiment_id: e6c2ce85b95a4a4fad44194653c2ae7f\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 3\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.8020994830700735\n",
            "  pid: 1337\n",
            "  time_since_restore: 1689.4383108615875\n",
            "  time_this_iter_s: 558.5877456665039\n",
            "  time_total_s: 1689.4383108615875\n",
            "  timestamp: 1616070540\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 3\n",
            "  trial_id: 231d52e4\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=10\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8052292738674598 | Iter 1.000: 0.7790611214713948\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 14/20 (1 PENDING, 1 RUNNING, 12 TERMINATED)\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc             |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_231d52e4 | RUNNING    | 172.28.0.2:1337 |     6.44559e-05 |                  2 |                            64 | 26.1482  |    0.802099 |\n",
            "| _inner_94d158ae | PENDING    |                 |     1.66709e-05 |                  5 |                             8 | 29.3587  |             |\n",
            "| _inner_64e82530 | TERMINATED |                 |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                 |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |                 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |                 |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |                 |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |                 |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |                 |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |                 |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |                 |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |                 |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |                 |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "| _inner_cda60438 | TERMINATED |                 |     1.06154e-05 |                  5 |                             4 |  1.37908 |    0.724329 |\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1337)\u001b[0m {'eval_loss': 0.43206846714019775, 'eval_accuracy': 0.7996408309864903, 'eval_f1': 0.8020994830700735, 'eval_precision': 0.8059045071164533, 'eval_recall': 0.7996408309864903, 'eval_runtime': 106.2638, 'eval_samples_per_second': 230.568, 'epoch': 0.96}\n",
            "\u001b[2m\u001b[36m(pid=1337)\u001b[0m {'loss': 0.2082, 'learning_rate': 2.3248069757986722e-05, 'epoch': 1.28}\n",
            "Result for _inner_231d52e4:\n",
            "  date: 2021-03-18_12-38-19\n",
            "  done: true\n",
            "  epoch: 1.28\n",
            "  eval_accuracy: 0.8016407493571691\n",
            "  eval_f1: 0.7996379094997614\n",
            "  eval_loss: 0.5687971115112305\n",
            "  eval_precision: 0.798326755677992\n",
            "  eval_recall: 0.8016407493571691\n",
            "  eval_runtime: 106.0827\n",
            "  eval_samples_per_second: 230.961\n",
            "  experiment_id: e6c2ce85b95a4a4fad44194653c2ae7f\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 4\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7996379094997614\n",
            "  pid: 1337\n",
            "  time_since_restore: 2248.496516942978\n",
            "  time_this_iter_s: 559.0582060813904\n",
            "  time_total_s: 2248.496516942978\n",
            "  timestamp: 1616071099\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 4\n",
            "  trial_id: 231d52e4\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=11\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8038314327755351 | Iter 1.000: 0.7790611214713948\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 14/20 (1 PENDING, 13 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_94d158ae | PENDING    |       |     1.66709e-05 |                  5 |                             8 | 29.3587  |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |       |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |       |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |       |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |       |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |       |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "| _inner_cda60438 | TERMINATED |       |     1.06154e-05 |                  5 |                             4 |  1.37908 |    0.724329 |\n",
            "| _inner_231d52e4 | TERMINATED |       |     6.44559e-05 |                  2 |                            64 | 26.1482  |    0.799638 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1337)\u001b[0m {'eval_loss': 0.5687971115112305, 'eval_accuracy': 0.8016407493571691, 'eval_f1': 0.7996379094997614, 'eval_precision': 0.798326755677992, 'eval_recall': 0.8016407493571691, 'eval_runtime': 106.0827, 'eval_samples_per_second': 230.961, 'epoch': 1.28}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m 2021-03-18 12:38:20.749409: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m {'loss': 0.4966, 'learning_rate': 1.6537644939080908e-05, 'epoch': 0.04}\n",
            "Result for _inner_94d158ae:\n",
            "  date: 2021-03-18_12-41-54\n",
            "  done: true\n",
            "  epoch: 0.04\n",
            "  eval_accuracy: 0.7666217705399779\n",
            "  eval_f1: 0.7434886534248963\n",
            "  eval_loss: 0.5142428874969482\n",
            "  eval_precision: 0.7557496621585467\n",
            "  eval_recall: 0.7666217705399779\n",
            "  eval_runtime: 106.7762\n",
            "  eval_samples_per_second: 229.461\n",
            "  experiment_id: d946e178d1ac47d2867440300ffc8ec5\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7434886534248963\n",
            "  pid: 1477\n",
            "  time_since_restore: 214.55583930015564\n",
            "  time_this_iter_s: 214.55583930015564\n",
            "  time_total_s: 214.55583930015564\n",
            "  timestamp: 1616071314\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 94d158ae\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=12\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8038314327755351 | Iter 1.000: 0.7781816540588324\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 15/20 (1 PENDING, 14 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_d189e2e8 | PENDING    |       |     7.05205e-05 |                  2 |                            16 |  9.78073 |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |       |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |       |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |       |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |       |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |       |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "| _inner_cda60438 | TERMINATED |       |     1.06154e-05 |                  5 |                             4 |  1.37908 |    0.724329 |\n",
            "| _inner_231d52e4 | TERMINATED |       |     6.44559e-05 |                  2 |                            64 | 26.1482  |    0.799638 |\n",
            "| _inner_94d158ae | TERMINATED |       |     1.66709e-05 |                  5 |                             8 | 29.3587  |    0.743489 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1477)\u001b[0m {'eval_loss': 0.5142428874969482, 'eval_accuracy': 0.7666217705399779, 'eval_f1': 0.7434886534248963, 'eval_precision': 0.7557496621585467, 'eval_recall': 0.7666217705399779, 'eval_runtime': 106.7762, 'eval_samples_per_second': 229.461, 'epoch': 0.04}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1507)\u001b[0m 2021-03-18 12:41:56.441845: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1507)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "\u001b[2m\u001b[36m(pid=1507)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1507)\u001b[0m {'loss': 0.4986, 'learning_rate': 6.770264054726372e-05, 'epoch': 0.08}\n",
            "Result for _inner_d189e2e8:\n",
            "  date: 2021-03-18_12-46-22\n",
            "  done: true\n",
            "  epoch: 0.08\n",
            "  eval_accuracy: 0.739275947920493\n",
            "  eval_f1: 0.6911848243769984\n",
            "  eval_loss: 0.5387644171714783\n",
            "  eval_precision: 0.7285571647965147\n",
            "  eval_recall: 0.739275947920493\n",
            "  eval_runtime: 105.3018\n",
            "  eval_samples_per_second: 232.674\n",
            "  experiment_id: 6920c33a99a44fdb81dbdbe80fd4931d\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.6911848243769984\n",
            "  pid: 1507\n",
            "  time_since_restore: 266.92331314086914\n",
            "  time_this_iter_s: 266.92331314086914\n",
            "  time_total_s: 266.92331314086914\n",
            "  timestamp: 1616071582\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: d189e2e8\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=13\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8038314327755351 | Iter 1.000: 0.7773021866462699\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 16/20 (1 PENDING, 15 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_51f3e398 | PENDING    |       |     5.04851e-05 |                  2 |                            64 |  8.79624 |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |       |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |       |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |       |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |       |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |       |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "| _inner_cda60438 | TERMINATED |       |     1.06154e-05 |                  5 |                             4 |  1.37908 |    0.724329 |\n",
            "| _inner_231d52e4 | TERMINATED |       |     6.44559e-05 |                  2 |                            64 | 26.1482  |    0.799638 |\n",
            "| _inner_94d158ae | TERMINATED |       |     1.66709e-05 |                  5 |                             8 | 29.3587  |    0.743489 |\n",
            "| _inner_d189e2e8 | TERMINATED |       |     7.05205e-05 |                  2 |                            16 |  9.78073 |    0.691185 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1507)\u001b[0m {'eval_loss': 0.5387644171714783, 'eval_accuracy': 0.739275947920493, 'eval_f1': 0.6911848243769984, 'eval_precision': 0.7285571647965147, 'eval_recall': 0.739275947920493, 'eval_runtime': 105.3018, 'eval_samples_per_second': 232.674, 'epoch': 0.08}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1540)\u001b[0m 2021-03-18 12:46:24.336103: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1540)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=1540)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1540)\u001b[0m {'loss': 0.4206, 'learning_rate': 4.2416074619002006e-05, 'epoch': 0.32}\n",
            "Result for _inner_51f3e398:\n",
            "  date: 2021-03-18_12-55-56\n",
            "  done: false\n",
            "  epoch: 0.32\n",
            "  eval_accuracy: 0.7861311783192523\n",
            "  eval_f1: 0.7884561437243965\n",
            "  eval_loss: 0.4610021114349365\n",
            "  eval_precision: 0.7917898595453349\n",
            "  eval_recall: 0.7861311783192523\n",
            "  eval_runtime: 106.7266\n",
            "  eval_samples_per_second: 229.568\n",
            "  experiment_id: 06162aa090de4ccc8e15555bb20552c2\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7884561437243965\n",
            "  pid: 1540\n",
            "  time_since_restore: 572.8663115501404\n",
            "  time_this_iter_s: 572.8663115501404\n",
            "  time_total_s: 572.8663115501404\n",
            "  timestamp: 1616072156\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 51f3e398\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=13\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8038314327755351 | Iter 1.000: 0.7803214110185814\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 17/20 (1 PENDING, 1 RUNNING, 15 TERMINATED)\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc             |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_51f3e398 | RUNNING    | 172.28.0.2:1540 |     5.04851e-05 |                  2 |                            64 |  8.79624 |    0.788456 |\n",
            "| _inner_f1a88286 | PENDING    |                 |     1.61955e-05 |                  5 |                             4 |  8.24467 |             |\n",
            "| _inner_64e82530 | TERMINATED |                 |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                 |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |                 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |                 |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |                 |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |                 |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |                 |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |                 |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |                 |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |                 |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |                 |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "| _inner_cda60438 | TERMINATED |                 |     1.06154e-05 |                  5 |                             4 |  1.37908 |    0.724329 |\n",
            "| _inner_231d52e4 | TERMINATED |                 |     6.44559e-05 |                  2 |                            64 | 26.1482  |    0.799638 |\n",
            "| _inner_94d158ae | TERMINATED |                 |     1.66709e-05 |                  5 |                             8 | 29.3587  |    0.743489 |\n",
            "| _inner_d189e2e8 | TERMINATED |                 |     7.05205e-05 |                  2 |                            16 |  9.78073 |    0.691185 |\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1540)\u001b[0m {'eval_loss': 0.4610021114349365, 'eval_accuracy': 0.7861311783192523, 'eval_f1': 0.7884561437243965, 'eval_precision': 0.7917898595453349, 'eval_recall': 0.7861311783192523, 'eval_runtime': 106.7266, 'eval_samples_per_second': 229.568, 'epoch': 0.32}\n",
            "\u001b[2m\u001b[36m(pid=1540)\u001b[0m {'loss': 0.3721, 'learning_rate': 3.434706866738463e-05, 'epoch': 0.64}\n",
            "Result for _inner_51f3e398:\n",
            "  date: 2021-03-18_13-05-16\n",
            "  done: false\n",
            "  epoch: 0.64\n",
            "  eval_accuracy: 0.7877637647442961\n",
            "  eval_f1: 0.7925182862101663\n",
            "  eval_loss: 0.4608881175518036\n",
            "  eval_precision: 0.8025799737376106\n",
            "  eval_recall: 0.7877637647442961\n",
            "  eval_runtime: 106.5069\n",
            "  eval_samples_per_second: 230.042\n",
            "  experiment_id: 06162aa090de4ccc8e15555bb20552c2\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 2\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7925182862101663\n",
            "  pid: 1540\n",
            "  time_since_restore: 1132.9057517051697\n",
            "  time_this_iter_s: 560.0394401550293\n",
            "  time_total_s: 1132.9057517051697\n",
            "  timestamp: 1616072716\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 2\n",
            "  trial_id: 51f3e398\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=13\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8038314327755351 | Iter 1.000: 0.7803214110185814\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 17/20 (1 PENDING, 1 RUNNING, 15 TERMINATED)\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc             |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_51f3e398 | RUNNING    | 172.28.0.2:1540 |     5.04851e-05 |                  2 |                            64 |  8.79624 |    0.792518 |\n",
            "| _inner_f1a88286 | PENDING    |                 |     1.61955e-05 |                  5 |                             4 |  8.24467 |             |\n",
            "| _inner_64e82530 | TERMINATED |                 |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                 |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |                 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |                 |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |                 |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |                 |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |                 |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |                 |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |                 |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |                 |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |                 |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "| _inner_cda60438 | TERMINATED |                 |     1.06154e-05 |                  5 |                             4 |  1.37908 |    0.724329 |\n",
            "| _inner_231d52e4 | TERMINATED |                 |     6.44559e-05 |                  2 |                            64 | 26.1482  |    0.799638 |\n",
            "| _inner_94d158ae | TERMINATED |                 |     1.66709e-05 |                  5 |                             8 | 29.3587  |    0.743489 |\n",
            "| _inner_d189e2e8 | TERMINATED |                 |     7.05205e-05 |                  2 |                            16 |  9.78073 |    0.691185 |\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1540)\u001b[0m {'eval_loss': 0.4608881175518036, 'eval_accuracy': 0.7877637647442961, 'eval_f1': 0.7925182862101663, 'eval_precision': 0.8025799737376106, 'eval_recall': 0.7877637647442961, 'eval_runtime': 106.5069, 'eval_samples_per_second': 230.042, 'epoch': 0.64}\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1540)\u001b[0m {'loss': 0.3573, 'learning_rate': 2.6278062715767256e-05, 'epoch': 0.96}\n",
            "Result for _inner_51f3e398:\n",
            "  date: 2021-03-18_13-14-36\n",
            "  done: false\n",
            "  epoch: 0.96\n",
            "  eval_accuracy: 0.8062119913472919\n",
            "  eval_f1: 0.7971325865556089\n",
            "  eval_loss: 0.4481305181980133\n",
            "  eval_precision: 0.7993903876294426\n",
            "  eval_recall: 0.8062119913472919\n",
            "  eval_runtime: 106.3365\n",
            "  eval_samples_per_second: 230.41\n",
            "  experiment_id: 06162aa090de4ccc8e15555bb20552c2\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 3\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7971325865556089\n",
            "  pid: 1540\n",
            "  time_since_restore: 1692.5232758522034\n",
            "  time_this_iter_s: 559.6175241470337\n",
            "  time_total_s: 1692.5232758522034\n",
            "  timestamp: 1616073276\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 3\n",
            "  trial_id: 51f3e398\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=13\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8038314327755351 | Iter 1.000: 0.7803214110185814\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 17/20 (1 PENDING, 1 RUNNING, 15 TERMINATED)\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc             |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_51f3e398 | RUNNING    | 172.28.0.2:1540 |     5.04851e-05 |                  2 |                            64 |  8.79624 |    0.797133 |\n",
            "| _inner_f1a88286 | PENDING    |                 |     1.61955e-05 |                  5 |                             4 |  8.24467 |             |\n",
            "| _inner_64e82530 | TERMINATED |                 |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |                 |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |                 |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |                 |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |                 |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |                 |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |                 |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |                 |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |                 |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |                 |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |                 |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "| _inner_cda60438 | TERMINATED |                 |     1.06154e-05 |                  5 |                             4 |  1.37908 |    0.724329 |\n",
            "| _inner_231d52e4 | TERMINATED |                 |     6.44559e-05 |                  2 |                            64 | 26.1482  |    0.799638 |\n",
            "| _inner_94d158ae | TERMINATED |                 |     1.66709e-05 |                  5 |                             8 | 29.3587  |    0.743489 |\n",
            "| _inner_d189e2e8 | TERMINATED |                 |     7.05205e-05 |                  2 |                            16 |  9.78073 |    0.691185 |\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1540)\u001b[0m {'eval_loss': 0.4481305181980133, 'eval_accuracy': 0.8062119913472919, 'eval_f1': 0.7971325865556089, 'eval_precision': 0.7993903876294426, 'eval_recall': 0.8062119913472919, 'eval_runtime': 106.3365, 'eval_samples_per_second': 230.41, 'epoch': 0.96}\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1540)\u001b[0m {'loss': 0.2095, 'learning_rate': 1.820905676414988e-05, 'epoch': 1.28}\n",
            "Result for _inner_51f3e398:\n",
            "  date: 2021-03-18_13-23-55\n",
            "  done: true\n",
            "  epoch: 1.28\n",
            "  eval_accuracy: 0.806252806007918\n",
            "  eval_f1: 0.8025772058906058\n",
            "  eval_loss: 0.5125831365585327\n",
            "  eval_precision: 0.8011715542705021\n",
            "  eval_recall: 0.806252806007918\n",
            "  eval_runtime: 105.8505\n",
            "  eval_samples_per_second: 231.468\n",
            "  experiment_id: 06162aa090de4ccc8e15555bb20552c2\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 4\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.8025772058906058\n",
            "  pid: 1540\n",
            "  time_since_restore: 2251.761778354645\n",
            "  time_this_iter_s: 559.2385025024414\n",
            "  time_total_s: 2251.761778354645\n",
            "  timestamp: 1616073835\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 4\n",
            "  trial_id: 51f3e398\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=14\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8039032398790328 | Iter 1.000: 0.7803214110185814\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 17/20 (1 PENDING, 16 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_f1a88286 | PENDING    |       |     1.61955e-05 |                  5 |                             4 |  8.24467 |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |       |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |       |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |       |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |       |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |       |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "| _inner_cda60438 | TERMINATED |       |     1.06154e-05 |                  5 |                             4 |  1.37908 |    0.724329 |\n",
            "| _inner_231d52e4 | TERMINATED |       |     6.44559e-05 |                  2 |                            64 | 26.1482  |    0.799638 |\n",
            "| _inner_94d158ae | TERMINATED |       |     1.66709e-05 |                  5 |                             8 | 29.3587  |    0.743489 |\n",
            "| _inner_d189e2e8 | TERMINATED |       |     7.05205e-05 |                  2 |                            16 |  9.78073 |    0.691185 |\n",
            "| _inner_51f3e398 | TERMINATED |       |     5.04851e-05 |                  2 |                            64 |  8.79624 |    0.802577 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1540)\u001b[0m {'eval_loss': 0.5125831365585327, 'eval_accuracy': 0.806252806007918, 'eval_f1': 0.8025772058906058, 'eval_precision': 0.8011715542705021, 'eval_recall': 0.806252806007918, 'eval_runtime': 105.8505, 'eval_samples_per_second': 231.468, 'epoch': 1.28}\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1703)\u001b[0m 2021-03-18 13:23:57.143925: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1703)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "\u001b[2m\u001b[36m(pid=1703)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1703)\u001b[0m {'loss': 0.5618, 'learning_rate': 1.6130778812183638e-05, 'epoch': 0.02}\n",
            "Result for _inner_f1a88286:\n",
            "  date: 2021-03-18_13-27-06\n",
            "  done: true\n",
            "  epoch: 0.02\n",
            "  eval_accuracy: 0.7584588384147586\n",
            "  eval_f1: 0.7290304709224356\n",
            "  eval_loss: 0.5821283459663391\n",
            "  eval_precision: 0.7475364191783282\n",
            "  eval_recall: 0.7584588384147586\n",
            "  eval_runtime: 106.0176\n",
            "  eval_samples_per_second: 231.103\n",
            "  experiment_id: b5a65bf332364ad6a4c77b64215a9aee\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7290304709224356\n",
            "  pid: 1703\n",
            "  time_since_restore: 190.08754181861877\n",
            "  time_this_iter_s: 190.08754181861877\n",
            "  time_total_s: 190.08754181861877\n",
            "  timestamp: 1616074026\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: f1a88286\n",
            "  \n",
            "\u001b[2m\u001b[36m(pid=1703)\u001b[0m {'eval_loss': 0.5821283459663391, 'eval_accuracy': 0.7584588384147586, 'eval_f1': 0.7290304709224356, 'eval_precision': 0.7475364191783282, 'eval_recall': 0.7584588384147586, 'eval_runtime': 106.0176, 'eval_samples_per_second': 231.103, 'epoch': 0.02}\n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=15\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8039032398790328 | Iter 1.000: 0.7790611214713948\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 18/20 (1 PENDING, 17 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_306ce750 | PENDING    |       |     1.1225e-05  |                  5 |                             8 | 21.9505  |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |       |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |       |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |       |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |       |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |       |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "| _inner_cda60438 | TERMINATED |       |     1.06154e-05 |                  5 |                             4 |  1.37908 |    0.724329 |\n",
            "| _inner_231d52e4 | TERMINATED |       |     6.44559e-05 |                  2 |                            64 | 26.1482  |    0.799638 |\n",
            "| _inner_94d158ae | TERMINATED |       |     1.66709e-05 |                  5 |                             8 | 29.3587  |    0.743489 |\n",
            "| _inner_d189e2e8 | TERMINATED |       |     7.05205e-05 |                  2 |                            16 |  9.78073 |    0.691185 |\n",
            "| _inner_51f3e398 | TERMINATED |       |     5.04851e-05 |                  2 |                            64 |  8.79624 |    0.802577 |\n",
            "| _inner_f1a88286 | TERMINATED |       |     1.61955e-05 |                  5 |                             4 |  8.24467 |    0.72903  |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1732)\u001b[0m 2021-03-18 13:27:08.008963: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1732)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "\u001b[2m\u001b[36m(pid=1732)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1732)\u001b[0m {'loss': 0.4903, 'learning_rate': 1.113533852234219e-05, 'epoch': 0.04}\n",
            "Result for _inner_306ce750:\n",
            "  date: 2021-03-18_13-30-41\n",
            "  done: true\n",
            "  epoch: 0.04\n",
            "  eval_accuracy: 0.7685808742500306\n",
            "  eval_f1: 0.7639675219371501\n",
            "  eval_loss: 0.47730815410614014\n",
            "  eval_precision: 0.7616431172282723\n",
            "  eval_recall: 0.7685808742500306\n",
            "  eval_runtime: 106.6281\n",
            "  eval_samples_per_second: 229.78\n",
            "  experiment_id: 23106db7e2184e00ae8f3929e3e2c6b4\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7639675219371501\n",
            "  pid: 1732\n",
            "  time_since_restore: 214.4102017879486\n",
            "  time_this_iter_s: 214.4102017879486\n",
            "  time_total_s: 214.4102017879486\n",
            "  timestamp: 1616074241\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 306ce750\n",
            "  \n",
            "\u001b[2m\u001b[36m(pid=1732)\u001b[0m {'eval_loss': 0.47730815410614014, 'eval_accuracy': 0.7685808742500306, 'eval_f1': 0.7639675219371501, 'eval_precision': 0.7616431172282723, 'eval_recall': 0.7685808742500306, 'eval_runtime': 106.6281, 'eval_samples_per_second': 229.78, 'epoch': 0.04}\n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=16\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8039032398790328 | Iter 1.000: 0.7781816540588324\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 19/20 (1 PENDING, 18 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_a2598198 | PENDING    |       |     3.56164e-05 |                  5 |                             8 | 36.6783  |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |       |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |       |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |       |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |       |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |       |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "| _inner_cda60438 | TERMINATED |       |     1.06154e-05 |                  5 |                             4 |  1.37908 |    0.724329 |\n",
            "| _inner_231d52e4 | TERMINATED |       |     6.44559e-05 |                  2 |                            64 | 26.1482  |    0.799638 |\n",
            "| _inner_94d158ae | TERMINATED |       |     1.66709e-05 |                  5 |                             8 | 29.3587  |    0.743489 |\n",
            "| _inner_d189e2e8 | TERMINATED |       |     7.05205e-05 |                  2 |                            16 |  9.78073 |    0.691185 |\n",
            "| _inner_51f3e398 | TERMINATED |       |     5.04851e-05 |                  2 |                            64 |  8.79624 |    0.802577 |\n",
            "| _inner_f1a88286 | TERMINATED |       |     1.61955e-05 |                  5 |                             4 |  8.24467 |    0.72903  |\n",
            "| _inner_306ce750 | TERMINATED |       |     1.1225e-05  |                  5 |                             8 | 21.9505  |    0.763968 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1760)\u001b[0m 2021-03-18 13:30:43.290565: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1760)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "\u001b[2m\u001b[36m(pid=1760)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1760)\u001b[0m {'loss': 0.5924, 'learning_rate': 3.533178645919406e-05, 'epoch': 0.04}\n",
            "Result for _inner_a2598198:\n",
            "  date: 2021-03-18_13-34-14\n",
            "  done: true\n",
            "  epoch: 0.04\n",
            "  eval_accuracy: 0.7004612056650749\n",
            "  eval_f1: 0.5773008286881254\n",
            "  eval_loss: 0.619695782661438\n",
            "  eval_precision: 0.6404496922483778\n",
            "  eval_recall: 0.7004612056650749\n",
            "  eval_runtime: 104.149\n",
            "  eval_samples_per_second: 235.25\n",
            "  experiment_id: e33d229d3bfc40579b9d300208dfdad5\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.5773008286881254\n",
            "  pid: 1760\n",
            "  time_since_restore: 211.84330439567566\n",
            "  time_this_iter_s: 211.84330439567566\n",
            "  time_total_s: 211.84330439567566\n",
            "  timestamp: 1616074454\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: a2598198\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.6/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=17\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8039032398790328 | Iter 1.000: 0.7773021866462699\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 20/20 (1 PENDING, 19 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_22a5f49e | PENDING    |       |     6.24521e-05 |                  5 |                             8 | 35.5573  |             |\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |       |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |       |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |       |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |       |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |       |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "| _inner_cda60438 | TERMINATED |       |     1.06154e-05 |                  5 |                             4 |  1.37908 |    0.724329 |\n",
            "| _inner_231d52e4 | TERMINATED |       |     6.44559e-05 |                  2 |                            64 | 26.1482  |    0.799638 |\n",
            "| _inner_94d158ae | TERMINATED |       |     1.66709e-05 |                  5 |                             8 | 29.3587  |    0.743489 |\n",
            "| _inner_d189e2e8 | TERMINATED |       |     7.05205e-05 |                  2 |                            16 |  9.78073 |    0.691185 |\n",
            "| _inner_51f3e398 | TERMINATED |       |     5.04851e-05 |                  2 |                            64 |  8.79624 |    0.802577 |\n",
            "| _inner_f1a88286 | TERMINATED |       |     1.61955e-05 |                  5 |                             4 |  8.24467 |    0.72903  |\n",
            "| _inner_306ce750 | TERMINATED |       |     1.1225e-05  |                  5 |                             8 | 21.9505  |    0.763968 |\n",
            "| _inner_a2598198 | TERMINATED |       |     3.56164e-05 |                  5 |                             8 | 36.6783  |    0.577301 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1760)\u001b[0m {'eval_loss': 0.619695782661438, 'eval_accuracy': 0.7004612056650749, 'eval_f1': 0.5773008286881254, 'eval_precision': 0.6404496922483778, 'eval_recall': 0.7004612056650749, 'eval_runtime': 104.149, 'eval_samples_per_second': 235.25, 'epoch': 0.04}\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1789)\u001b[0m 2021-03-18 13:34:16.219802: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1789)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "\u001b[2m\u001b[36m(pid=1789)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1789)\u001b[0m {'loss': 0.611, 'learning_rate': 6.195295774667361e-05, 'epoch': 0.04}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1789)\u001b[0m /usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "\u001b[2m\u001b[36m(pid=1789)\u001b[0m   _warn_prf(average, modifier, msg_start, len(result))\n",
            "2021-03-18 13:37:47,937\tINFO tune.py:450 -- Total run time: 17930.90 seconds (17924.42 seconds for the tuning loop).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Result for _inner_22a5f49e:\n",
            "  date: 2021-03-18_13-37-47\n",
            "  done: true\n",
            "  epoch: 0.04\n",
            "  eval_accuracy: 0.7004612056650749\n",
            "  eval_f1: 0.5770739126622669\n",
            "  eval_loss: 0.6193544268608093\n",
            "  eval_precision: 0.4906459006417703\n",
            "  eval_recall: 0.7004612056650749\n",
            "  eval_runtime: 104.8177\n",
            "  eval_samples_per_second: 233.749\n",
            "  experiment_id: c3721b845dd04f21b730852c768a13ba\n",
            "  hostname: b5916f891e08\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.5770739126622669\n",
            "  pid: 1789\n",
            "  time_since_restore: 212.35546588897705\n",
            "  time_this_iter_s: 212.35546588897705\n",
            "  time_total_s: 212.35546588897705\n",
            "  timestamp: 1616074667\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 22a5f49e\n",
            "  \n",
            "\u001b[2m\u001b[36m(pid=1789)\u001b[0m {'eval_loss': 0.6193544268608093, 'eval_accuracy': 0.7004612056650749, 'eval_f1': 0.5770739126622669, 'eval_precision': 0.4906459006417703, 'eval_recall': 0.7004612056650749, 'eval_runtime': 104.8177, 'eval_samples_per_second': 233.749, 'epoch': 0.04}\n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=18\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8039032398790328 | Iter 1.000: 0.7764227192337074\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 20/20 (20 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |       |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |       |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |       |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |       |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |       |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "| _inner_cda60438 | TERMINATED |       |     1.06154e-05 |                  5 |                             4 |  1.37908 |    0.724329 |\n",
            "| _inner_231d52e4 | TERMINATED |       |     6.44559e-05 |                  2 |                            64 | 26.1482  |    0.799638 |\n",
            "| _inner_94d158ae | TERMINATED |       |     1.66709e-05 |                  5 |                             8 | 29.3587  |    0.743489 |\n",
            "| _inner_d189e2e8 | TERMINATED |       |     7.05205e-05 |                  2 |                            16 |  9.78073 |    0.691185 |\n",
            "| _inner_51f3e398 | TERMINATED |       |     5.04851e-05 |                  2 |                            64 |  8.79624 |    0.802577 |\n",
            "| _inner_f1a88286 | TERMINATED |       |     1.61955e-05 |                  5 |                             4 |  8.24467 |    0.72903  |\n",
            "| _inner_306ce750 | TERMINATED |       |     1.1225e-05  |                  5 |                             8 | 21.9505  |    0.763968 |\n",
            "| _inner_a2598198 | TERMINATED |       |     3.56164e-05 |                  5 |                             8 | 36.6783  |    0.577301 |\n",
            "| _inner_22a5f49e | TERMINATED |       |     6.24521e-05 |                  5 |                             8 | 35.5573  |    0.577074 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "== Status ==\n",
            "Memory usage on this node: 5.5/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=18\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: 0.8039032398790328 | Iter 1.000: 0.7764227192337074\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.32 GiB heap, 0.0/2.49 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-18_08-39-03\n",
            "Number of trials: 20/20 (20 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_64e82530 | TERMINATED |       |     9.69583e-05 |                  1 |                            64 | 34.923   |    0.802353 |\n",
            "| _inner_64ed099c | TERMINATED |       |     4.3896e-06  |                  2 |                            64 | 20.7908  |    0.775543 |\n",
            "| _inner_840f43a4 | TERMINATED |       |     9.74955e-06 |                  4 |                            64 |  2.61251 |    0.809275 |\n",
            "| _inner_d969198c | TERMINATED |       |     1.2724e-05  |                  2 |                             4 | 12.8396  |    0.754771 |\n",
            "| _inner_0a467ba2 | TERMINATED |       |     1.81941e-06 |                  4 |                            64 | 20.1972  |    0.759186 |\n",
            "| _inner_7b639b3a | TERMINATED |       |     4.05479e-05 |                  5 |                            16 | 24.4544  |    0.772615 |\n",
            "| _inner_d11513c8 | TERMINATED |       |     1.70282e-05 |                  4 |                            16 | 21.2015  |    0.779061 |\n",
            "| _inner_70aa2522 | TERMINATED |       |     6.27619e-05 |                  5 |                             8 |  4.40721 |    0.577817 |\n",
            "| _inner_103f6c14 | TERMINATED |       |     2.01159e-06 |                  2 |                            16 |  5.50495 |    0.74487  |\n",
            "| _inner_8d952e60 | TERMINATED |       |     3.63457e-06 |                  3 |                            16 | 13.7357  |    0.765775 |\n",
            "| _inner_2db3b948 | TERMINATED |       |     4.18269e-06 |                  4 |                            64 |  4.09892 |    0.769571 |\n",
            "| _inner_cda60438 | TERMINATED |       |     1.06154e-05 |                  5 |                             4 |  1.37908 |    0.724329 |\n",
            "| _inner_231d52e4 | TERMINATED |       |     6.44559e-05 |                  2 |                            64 | 26.1482  |    0.799638 |\n",
            "| _inner_94d158ae | TERMINATED |       |     1.66709e-05 |                  5 |                             8 | 29.3587  |    0.743489 |\n",
            "| _inner_d189e2e8 | TERMINATED |       |     7.05205e-05 |                  2 |                            16 |  9.78073 |    0.691185 |\n",
            "| _inner_51f3e398 | TERMINATED |       |     5.04851e-05 |                  2 |                            64 |  8.79624 |    0.802577 |\n",
            "| _inner_f1a88286 | TERMINATED |       |     1.61955e-05 |                  5 |                             4 |  8.24467 |    0.72903  |\n",
            "| _inner_306ce750 | TERMINATED |       |     1.1225e-05  |                  5 |                             8 | 21.9505  |    0.763968 |\n",
            "| _inner_a2598198 | TERMINATED |       |     3.56164e-05 |                  5 |                             8 | 36.6783  |    0.577301 |\n",
            "| _inner_22a5f49e | TERMINATED |       |     6.24521e-05 |                  5 |                             8 | 35.5573  |    0.577074 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dlYAeqKtzXoa"
      },
      "source": [
        "#### Пример запуска на части датасета"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yj-PoBiv61aJ",
        "outputId": "4b07c716-7dfb-447b-f855-b9d9cceb1011"
      },
      "source": [
        "best_run = trainer.hyperparameter_search(\n",
        "    n_trials=10,\n",
        "    compute_objective = my_func,\n",
        "    direction=\"maximize\",\n",
        "    backend = 'ray',\n",
        "    search_alg=HyperOptSearch(metric = 'eval_f1', mode = 'max'),\n",
        "    scheduler=AsyncHyperBandScheduler(metric = 'eval_f1', mode = 'max')\n",
        "    )"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "== Status ==\n",
            "Memory usage on this node: 4.3/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=0\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: None\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.42 GiB heap, 0.0/2.54 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-17_11-51-38\n",
            "Number of trials: 1/10 (1 RUNNING)\n",
            "+-----------------+----------+-------+-----------------+--------------------+-------------------------------+---------+\n",
            "| Trial name      | status   | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |    seed |\n",
            "|-----------------+----------+-------+-----------------+--------------------+-------------------------------+---------|\n",
            "| _inner_2182b94e | RUNNING  |       |     9.39003e-05 |                  1 |                            32 | 36.0425 |\n",
            "+-----------------+----------+-------+-----------------+--------------------+-------------------------------+---------+\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=947)\u001b[0m 2021-03-17 11:51:39.072799: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=947)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=947)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=947)\u001b[0m {'train_runtime': 183.8865, 'train_samples_per_second': 2.55, 'epoch': 1.0}\n",
            "\u001b[2m\u001b[36m(pid=947)\u001b[0m {'eval_loss': 0.4715965688228607, 'eval_accuracy': 0.789, 'eval_f1': 0.7880768352084142, 'eval_precision': 0.7872916009785267, 'eval_recall': 0.789, 'eval_runtime': 7.9616, 'eval_samples_per_second': 251.206, 'epoch': 1.0}\n",
            "Result for _inner_2182b94e:\n",
            "  date: 2021-03-17_11-55-03\n",
            "  done: true\n",
            "  epoch: 1.0\n",
            "  eval_accuracy: 0.789\n",
            "  eval_f1: 0.7880768352084142\n",
            "  eval_loss: 0.4715965688228607\n",
            "  eval_mem_cpu_alloc_delta: 137517\n",
            "  eval_mem_cpu_peaked_delta: 231704\n",
            "  eval_mem_gpu_alloc_delta: 0\n",
            "  eval_mem_gpu_peaked_delta: 16966144\n",
            "  eval_precision: 0.7872916009785267\n",
            "  eval_recall: 0.789\n",
            "  eval_runtime: 7.9616\n",
            "  eval_samples_per_second: 251.206\n",
            "  experiment_id: 69e5b319493a458d9adb9246d52b9882\n",
            "  hostname: 30101e747662\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7880768352084142\n",
            "  pid: 947\n",
            "  time_since_restore: 204.9747622013092\n",
            "  time_this_iter_s: 204.9747622013092\n",
            "  time_total_s: 204.9747622013092\n",
            "  timestamp: 1615982103\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 2182b94e\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 6.1/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=0\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7880768352084142\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.42 GiB heap, 0.0/2.54 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-17_11-51-38\n",
            "Number of trials: 2/10 (1 PENDING, 1 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_21872f74 | PENDING    |       |     1.91335e-06 |                  3 |                            32 |  8.09127 |             |\n",
            "| _inner_2182b94e | TERMINATED |       |     9.39003e-05 |                  1 |                            32 | 36.0425  |    0.788077 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=946)\u001b[0m 2021-03-17 11:55:04.712475: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=946)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=946)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=946)\u001b[0m {'loss': 0.5517, 'learning_rate': 1.097425249072027e-06, 'epoch': 1.28}\n",
            "Result for _inner_21872f74:\n",
            "  date: 2021-03-17_11-59-22\n",
            "  done: true\n",
            "  epoch: 1.28\n",
            "  eval_accuracy: 0.735\n",
            "  eval_f1: 0.7408069877883311\n",
            "  eval_loss: 0.5122585296630859\n",
            "  eval_precision: 0.7513574845827476\n",
            "  eval_recall: 0.735\n",
            "  eval_runtime: 7.9894\n",
            "  eval_samples_per_second: 250.33\n",
            "  experiment_id: e3691bd74f54474aaed34c8e2dd52f43\n",
            "  hostname: 30101e747662\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7408069877883311\n",
            "  pid: 946\n",
            "  time_since_restore: 258.65727972984314\n",
            "  time_this_iter_s: 258.65727972984314\n",
            "  time_total_s: 258.65727972984314\n",
            "  timestamp: 1615982362\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 21872f74\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 6.1/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7762593733533935\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.42 GiB heap, 0.0/2.54 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-17_11-51-38\n",
            "Number of trials: 3/10 (1 PENDING, 2 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_9be36dc8 | PENDING    |       |     1.59723e-05 |                  4 |                            32 | 21.4019  |             |\n",
            "| _inner_2182b94e | TERMINATED |       |     9.39003e-05 |                  1 |                            32 | 36.0425  |    0.788077 |\n",
            "| _inner_21872f74 | TERMINATED |       |     1.91335e-06 |                  3 |                            32 |  8.09127 |    0.740807 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=946)\u001b[0m {'eval_loss': 0.5122585296630859, 'eval_accuracy': 0.735, 'eval_f1': 0.7408069877883311, 'eval_precision': 0.7513574845827476, 'eval_recall': 0.735, 'eval_runtime': 7.9894, 'eval_samples_per_second': 250.33, 'epoch': 1.28}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1088)\u001b[0m 2021-03-17 11:59:24.317169: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1088)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=1088)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1088)\u001b[0m {'loss': 0.411, 'learning_rate': 1.0863856497913225e-05, 'epoch': 1.28}\n",
            "Result for _inner_9be36dc8:\n",
            "  date: 2021-03-17_12-03-42\n",
            "  done: false\n",
            "  epoch: 1.28\n",
            "  eval_accuracy: 0.7825\n",
            "  eval_f1: 0.7860975386574355\n",
            "  eval_loss: 0.5072636008262634\n",
            "  eval_precision: 0.7923291273396258\n",
            "  eval_recall: 0.7825\n",
            "  eval_runtime: 7.9748\n",
            "  eval_samples_per_second: 250.79\n",
            "  experiment_id: aef9e5a2f60e44c2839e1bcc7f15b365\n",
            "  hostname: 30101e747662\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7860975386574355\n",
            "  pid: 1088\n",
            "  time_since_restore: 259.27694749832153\n",
            "  time_this_iter_s: 259.27694749832153\n",
            "  time_total_s: 259.27694749832153\n",
            "  timestamp: 1615982622\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 9be36dc8\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 6.1/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7870871869329248\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.42 GiB heap, 0.0/2.54 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-17_11-51-38\n",
            "Number of trials: 4/10 (1 PENDING, 1 RUNNING, 2 TERMINATED)\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc             |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_9be36dc8 | RUNNING    | 172.28.0.2:1088 |     1.59723e-05 |                  4 |                            32 | 21.4019  |    0.786098 |\n",
            "| _inner_3646bbfe | PENDING    |                 |     1.79784e-06 |                  2 |                             4 | 13.0258  |             |\n",
            "| _inner_2182b94e | TERMINATED |                 |     9.39003e-05 |                  1 |                            32 | 36.0425  |    0.788077 |\n",
            "| _inner_21872f74 | TERMINATED |                 |     1.91335e-06 |                  3 |                            32 |  8.09127 |    0.740807 |\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1088)\u001b[0m {'eval_loss': 0.5072636008262634, 'eval_accuracy': 0.7825, 'eval_f1': 0.7860975386574355, 'eval_precision': 0.7923291273396258, 'eval_recall': 0.7825, 'eval_runtime': 7.9748, 'eval_samples_per_second': 250.79, 'epoch': 1.28}\n",
            "\u001b[2m\u001b[36m(pid=1088)\u001b[0m {'loss': 0.2172, 'learning_rate': 5.755460025540235e-06, 'epoch': 2.56}\n",
            "Result for _inner_9be36dc8:\n",
            "  date: 2021-03-17_12-07-49\n",
            "  done: false\n",
            "  epoch: 2.56\n",
            "  eval_accuracy: 0.7915\n",
            "  eval_f1: 0.7866731191045062\n",
            "  eval_loss: 0.6679012179374695\n",
            "  eval_precision: 0.7850783520651942\n",
            "  eval_recall: 0.7915\n",
            "  eval_runtime: 7.9824\n",
            "  eval_samples_per_second: 250.551\n",
            "  experiment_id: aef9e5a2f60e44c2839e1bcc7f15b365\n",
            "  hostname: 30101e747662\n",
            "  iterations_since_restore: 2\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7866731191045062\n",
            "  pid: 1088\n",
            "  time_since_restore: 505.7992169857025\n",
            "  time_this_iter_s: 246.52226948738098\n",
            "  time_total_s: 505.7992169857025\n",
            "  timestamp: 1615982869\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 2\n",
            "  trial_id: 9be36dc8\n",
            "  \n",
            "\u001b[2m\u001b[36m(pid=1088)\u001b[0m {'eval_loss': 0.6679012179374695, 'eval_accuracy': 0.7915, 'eval_f1': 0.7866731191045062, 'eval_precision': 0.7850783520651942, 'eval_recall': 0.7915, 'eval_runtime': 7.9824, 'eval_samples_per_second': 250.551, 'epoch': 2.56}\n",
            "== Status ==\n",
            "Memory usage on this node: 6.1/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7870871869329248\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.42 GiB heap, 0.0/2.54 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-17_11-51-38\n",
            "Number of trials: 4/10 (1 PENDING, 1 RUNNING, 2 TERMINATED)\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc             |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_9be36dc8 | RUNNING    | 172.28.0.2:1088 |     1.59723e-05 |                  4 |                            32 | 21.4019  |    0.786673 |\n",
            "| _inner_3646bbfe | PENDING    |                 |     1.79784e-06 |                  2 |                             4 | 13.0258  |             |\n",
            "| _inner_2182b94e | TERMINATED |                 |     9.39003e-05 |                  1 |                            32 | 36.0425  |    0.788077 |\n",
            "| _inner_21872f74 | TERMINATED |                 |     1.91335e-06 |                  3 |                            32 |  8.09127 |    0.740807 |\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1088)\u001b[0m {'loss': 0.0947, 'learning_rate': 6.470635531672453e-07, 'epoch': 3.84}\n",
            "Result for _inner_9be36dc8:\n",
            "  date: 2021-03-17_12-11-55\n",
            "  done: false\n",
            "  epoch: 3.84\n",
            "  eval_accuracy: 0.787\n",
            "  eval_f1: 0.7821351465850096\n",
            "  eval_loss: 1.0020341873168945\n",
            "  eval_precision: 0.7803934499810705\n",
            "  eval_recall: 0.787\n",
            "  eval_runtime: 8.0091\n",
            "  eval_samples_per_second: 249.717\n",
            "  experiment_id: aef9e5a2f60e44c2839e1bcc7f15b365\n",
            "  hostname: 30101e747662\n",
            "  iterations_since_restore: 3\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7821351465850096\n",
            "  pid: 1088\n",
            "  time_since_restore: 751.690482378006\n",
            "  time_this_iter_s: 245.89126539230347\n",
            "  time_total_s: 751.690482378006\n",
            "  timestamp: 1615983115\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 3\n",
            "  trial_id: 9be36dc8\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 6.1/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7870871869329248\n",
            "Resources requested: 1/2 CPUs, 1/1 GPUs, 0.0/7.42 GiB heap, 0.0/2.54 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-17_11-51-38\n",
            "Number of trials: 4/10 (1 PENDING, 1 RUNNING, 2 TERMINATED)\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc             |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_9be36dc8 | RUNNING    | 172.28.0.2:1088 |     1.59723e-05 |                  4 |                            32 | 21.4019  |    0.782135 |\n",
            "| _inner_3646bbfe | PENDING    |                 |     1.79784e-06 |                  2 |                             4 | 13.0258  |             |\n",
            "| _inner_2182b94e | TERMINATED |                 |     9.39003e-05 |                  1 |                            32 | 36.0425  |    0.788077 |\n",
            "| _inner_21872f74 | TERMINATED |                 |     1.91335e-06 |                  3 |                            32 |  8.09127 |    0.740807 |\n",
            "+-----------------+------------+-----------------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1088)\u001b[0m {'eval_loss': 1.0020341873168945, 'eval_accuracy': 0.787, 'eval_f1': 0.7821351465850096, 'eval_precision': 0.7803934499810705, 'eval_recall': 0.787, 'eval_runtime': 8.0091, 'eval_samples_per_second': 249.717, 'epoch': 3.84}\n",
            "\u001b[2m\u001b[36m(pid=1088)\u001b[0m {'train_runtime': 768.6755, 'train_samples_per_second': 2.441, 'epoch': 4.0}\n",
            "Result for _inner_9be36dc8:\n",
            "  date: 2021-03-17_12-11-55\n",
            "  done: true\n",
            "  epoch: 3.84\n",
            "  eval_accuracy: 0.787\n",
            "  eval_f1: 0.7821351465850096\n",
            "  eval_loss: 1.0020341873168945\n",
            "  eval_precision: 0.7803934499810705\n",
            "  eval_recall: 0.787\n",
            "  eval_runtime: 8.0091\n",
            "  eval_samples_per_second: 249.717\n",
            "  experiment_id: aef9e5a2f60e44c2839e1bcc7f15b365\n",
            "  experiment_tag: 3_learning_rate=1.5972e-05,num_train_epochs=4,per_device_train_batch_size=32,seed=21.402\n",
            "  hostname: 30101e747662\n",
            "  iterations_since_restore: 3\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7821351465850096\n",
            "  pid: 1088\n",
            "  time_since_restore: 751.690482378006\n",
            "  time_this_iter_s: 245.89126539230347\n",
            "  time_total_s: 751.690482378006\n",
            "  timestamp: 1615983115\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 3\n",
            "  trial_id: 9be36dc8\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 5.9/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=1\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7870871869329248\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.42 GiB heap, 0.0/2.54 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-17_11-51-38\n",
            "Number of trials: 4/10 (1 PENDING, 3 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_3646bbfe | PENDING    |       |     1.79784e-06 |                  2 |                             4 | 13.0258  |             |\n",
            "| _inner_2182b94e | TERMINATED |       |     9.39003e-05 |                  1 |                            32 | 36.0425  |    0.788077 |\n",
            "| _inner_21872f74 | TERMINATED |       |     1.91335e-06 |                  3 |                            32 |  8.09127 |    0.740807 |\n",
            "| _inner_9be36dc8 | TERMINATED |       |     1.59723e-05 |                  4 |                            32 | 21.4019  |    0.782135 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1234)\u001b[0m 2021-03-17 12:12:26.820016: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1234)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=1234)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1234)\u001b[0m {'loss': 0.6101, 'learning_rate': 1.6540149148583185e-06, 'epoch': 0.16}\n",
            "Result for _inner_3646bbfe:\n",
            "  date: 2021-03-17_12-13-57\n",
            "  done: true\n",
            "  epoch: 0.16\n",
            "  eval_accuracy: 0.6985\n",
            "  eval_f1: 0.5745095672652341\n",
            "  eval_loss: 0.5952737331390381\n",
            "  eval_precision: 0.48790225\n",
            "  eval_recall: 0.6985\n",
            "  eval_runtime: 7.8651\n",
            "  eval_samples_per_second: 254.289\n",
            "  experiment_id: 4c41b34615da4d00bec30143ab5108a6\n",
            "  hostname: 30101e747662\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.5745095672652341\n",
            "  pid: 1234\n",
            "  time_since_restore: 90.87262320518494\n",
            "  time_this_iter_s: 90.87262320518494\n",
            "  time_total_s: 90.87262320518494\n",
            "  timestamp: 1615983237\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 3646bbfe\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 6.1/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=2\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7865923627951802\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.42 GiB heap, 0.0/2.54 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-17_11-51-38\n",
            "Number of trials: 5/10 (1 PENDING, 4 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_08f0c710 | PENDING    |       |     4.07212e-05 |                  3 |                             4 | 28.0017  |             |\n",
            "| _inner_2182b94e | TERMINATED |       |     9.39003e-05 |                  1 |                            32 | 36.0425  |    0.788077 |\n",
            "| _inner_21872f74 | TERMINATED |       |     1.91335e-06 |                  3 |                            32 |  8.09127 |    0.740807 |\n",
            "| _inner_9be36dc8 | TERMINATED |       |     1.59723e-05 |                  4 |                            32 | 21.4019  |    0.782135 |\n",
            "| _inner_3646bbfe | TERMINATED |       |     1.79784e-06 |                  2 |                             4 | 13.0258  |    0.57451  |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1234)\u001b[0m {'eval_loss': 0.5952737331390381, 'eval_accuracy': 0.6985, 'eval_f1': 0.5745095672652341, 'eval_precision': 0.48790225, 'eval_recall': 0.6985, 'eval_runtime': 7.8651, 'eval_samples_per_second': 254.289, 'epoch': 0.16}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1234)\u001b[0m /usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "\u001b[2m\u001b[36m(pid=1234)\u001b[0m   _warn_prf(average, modifier, msg_start, len(result))\n",
            "\u001b[2m\u001b[36m(pid=1263)\u001b[0m 2021-03-17 12:13:58.869195: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1263)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "\u001b[2m\u001b[36m(pid=1263)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1263)\u001b[0m {'loss': 0.5699, 'learning_rate': 3.8549377943276704e-05, 'epoch': 0.16}\n",
            "Result for _inner_08f0c710:\n",
            "  date: 2021-03-17_12-15-29\n",
            "  done: true\n",
            "  epoch: 0.16\n",
            "  eval_accuracy: 0.7205\n",
            "  eval_f1: 0.6548769313662963\n",
            "  eval_loss: 0.8656408786773682\n",
            "  eval_precision: 0.7032909998374488\n",
            "  eval_recall: 0.7205\n",
            "  eval_runtime: 7.7369\n",
            "  eval_samples_per_second: 258.502\n",
            "  experiment_id: 415d4b0662bb4e859a943c1b143d1f6f\n",
            "  hostname: 30101e747662\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.6548769313662963\n",
            "  pid: 1263\n",
            "  time_since_restore: 90.98707389831543\n",
            "  time_this_iter_s: 90.98707389831543\n",
            "  time_total_s: 90.98707389831543\n",
            "  timestamp: 1615983329\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 08f0c710\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 6.1/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=3\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7860975386574355\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.42 GiB heap, 0.0/2.54 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-17_11-51-38\n",
            "Number of trials: 6/10 (1 PENDING, 5 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_3f985d6e | PENDING    |       |     8.02598e-06 |                  4 |                            32 |  2.6107  |             |\n",
            "| _inner_2182b94e | TERMINATED |       |     9.39003e-05 |                  1 |                            32 | 36.0425  |    0.788077 |\n",
            "| _inner_21872f74 | TERMINATED |       |     1.91335e-06 |                  3 |                            32 |  8.09127 |    0.740807 |\n",
            "| _inner_9be36dc8 | TERMINATED |       |     1.59723e-05 |                  4 |                            32 | 21.4019  |    0.782135 |\n",
            "| _inner_3646bbfe | TERMINATED |       |     1.79784e-06 |                  2 |                             4 | 13.0258  |    0.57451  |\n",
            "| _inner_08f0c710 | TERMINATED |       |     4.07212e-05 |                  3 |                             4 | 28.0017  |    0.654877 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1263)\u001b[0m {'eval_loss': 0.8656408786773682, 'eval_accuracy': 0.7205, 'eval_f1': 0.6548769313662963, 'eval_precision': 0.7032909998374488, 'eval_recall': 0.7205, 'eval_runtime': 7.7369, 'eval_samples_per_second': 258.502, 'epoch': 0.16}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1292)\u001b[0m 2021-03-17 12:15:30.887924: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1292)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=1292)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1292)\u001b[0m {'loss': 0.4317, 'learning_rate': 5.459037897329642e-06, 'epoch': 1.28}\n",
            "Result for _inner_3f985d6e:\n",
            "  date: 2021-03-17_12-19-48\n",
            "  done: true\n",
            "  epoch: 1.28\n",
            "  eval_accuracy: 0.7725\n",
            "  eval_f1: 0.7748205457248436\n",
            "  eval_loss: 0.48232269287109375\n",
            "  eval_precision: 0.7780235042735043\n",
            "  eval_recall: 0.7725\n",
            "  eval_runtime: 7.9783\n",
            "  eval_samples_per_second: 250.68\n",
            "  experiment_id: 945fca4fe9214e8d9c5d73f7d639e845\n",
            "  hostname: 30101e747662\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7748205457248436\n",
            "  pid: 1292\n",
            "  time_since_restore: 258.387149810791\n",
            "  time_this_iter_s: 258.387149810791\n",
            "  time_total_s: 258.387149810791\n",
            "  timestamp: 1615983588\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 3f985d6e\n",
            "  \n",
            "\u001b[2m\u001b[36m(pid=1292)\u001b[0m {'eval_loss': 0.48232269287109375, 'eval_accuracy': 0.7725, 'eval_f1': 0.7748205457248436, 'eval_precision': 0.7780235042735043, 'eval_recall': 0.7725, 'eval_runtime': 7.9783, 'eval_samples_per_second': 250.68, 'epoch': 1.28}\n",
            "== Status ==\n",
            "Memory usage on this node: 6.1/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=4\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7832782904242875\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.42 GiB heap, 0.0/2.54 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-17_11-51-38\n",
            "Number of trials: 7/10 (1 PENDING, 6 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_7675b2dc | PENDING    |       |     1.16336e-06 |                  3 |                            64 | 12.5797  |             |\n",
            "| _inner_2182b94e | TERMINATED |       |     9.39003e-05 |                  1 |                            32 | 36.0425  |    0.788077 |\n",
            "| _inner_21872f74 | TERMINATED |       |     1.91335e-06 |                  3 |                            32 |  8.09127 |    0.740807 |\n",
            "| _inner_9be36dc8 | TERMINATED |       |     1.59723e-05 |                  4 |                            32 | 21.4019  |    0.782135 |\n",
            "| _inner_3646bbfe | TERMINATED |       |     1.79784e-06 |                  2 |                             4 | 13.0258  |    0.57451  |\n",
            "| _inner_08f0c710 | TERMINATED |       |     4.07212e-05 |                  3 |                             4 | 28.0017  |    0.654877 |\n",
            "| _inner_3f985d6e | TERMINATED |       |     8.02598e-06 |                  4 |                            32 |  2.6107  |    0.774821 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1339)\u001b[0m 2021-03-17 12:19:50.286935: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1339)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "\u001b[2m\u001b[36m(pid=1339)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1339)\u001b[0m {'loss': 0.5608, 'learning_rate': 1.7326635912012824e-07, 'epoch': 2.55}\n",
            "Result for _inner_7675b2dc:\n",
            "  date: 2021-03-17_12-27-16\n",
            "  done: true\n",
            "  epoch: 2.55\n",
            "  eval_accuracy: 0.7275\n",
            "  eval_f1: 0.6841092471578667\n",
            "  eval_loss: 0.5370897054672241\n",
            "  eval_precision: 0.7056142394528754\n",
            "  eval_recall: 0.7275\n",
            "  eval_runtime: 8.0335\n",
            "  eval_samples_per_second: 248.957\n",
            "  experiment_id: e0c6a3113e714786bb395cb5998febaf\n",
            "  hostname: 30101e747662\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.6841092471578667\n",
            "  pid: 1339\n",
            "  time_since_restore: 446.70611929893494\n",
            "  time_this_iter_s: 446.70611929893494\n",
            "  time_total_s: 446.70611929893494\n",
            "  timestamp: 1615984036\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 7675b2dc\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 6.1/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=5\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7804590421911395\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.42 GiB heap, 0.0/2.54 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-17_11-51-38\n",
            "Number of trials: 8/10 (1 PENDING, 7 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_1119b43c | PENDING    |       |     6.4134e-05  |                  3 |                            16 | 31.506   |             |\n",
            "| _inner_2182b94e | TERMINATED |       |     9.39003e-05 |                  1 |                            32 | 36.0425  |    0.788077 |\n",
            "| _inner_21872f74 | TERMINATED |       |     1.91335e-06 |                  3 |                            32 |  8.09127 |    0.740807 |\n",
            "| _inner_9be36dc8 | TERMINATED |       |     1.59723e-05 |                  4 |                            32 | 21.4019  |    0.782135 |\n",
            "| _inner_3646bbfe | TERMINATED |       |     1.79784e-06 |                  2 |                             4 | 13.0258  |    0.57451  |\n",
            "| _inner_08f0c710 | TERMINATED |       |     4.07212e-05 |                  3 |                             4 | 28.0017  |    0.654877 |\n",
            "| _inner_3f985d6e | TERMINATED |       |     8.02598e-06 |                  4 |                            32 |  2.6107  |    0.774821 |\n",
            "| _inner_7675b2dc | TERMINATED |       |     1.16336e-06 |                  3 |                            64 | 12.5797  |    0.684109 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1339)\u001b[0m {'eval_loss': 0.5370897054672241, 'eval_accuracy': 0.7275, 'eval_f1': 0.6841092471578667, 'eval_precision': 0.7056142394528754, 'eval_recall': 0.7275, 'eval_runtime': 8.0335, 'eval_samples_per_second': 248.957, 'epoch': 2.55}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1384)\u001b[0m 2021-03-17 12:27:18.193219: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1384)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=1384)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1384)\u001b[0m {'loss': 0.4937, 'learning_rate': 5.0459356169508037e-05, 'epoch': 0.64}\n",
            "Result for _inner_1119b43c:\n",
            "  date: 2021-03-17_12-29-59\n",
            "  done: true\n",
            "  epoch: 0.64\n",
            "  eval_accuracy: 0.7415\n",
            "  eval_f1: 0.7469664776146879\n",
            "  eval_loss: 0.5140169858932495\n",
            "  eval_precision: 0.7568265798380905\n",
            "  eval_recall: 0.7415\n",
            "  eval_runtime: 7.9892\n",
            "  eval_samples_per_second: 250.339\n",
            "  experiment_id: 5f9e7ce20f5b4b7fab0746c63b806d21\n",
            "  hostname: 30101e747662\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7469664776146879\n",
            "  pid: 1384\n",
            "  time_since_restore: 162.6242184638977\n",
            "  time_this_iter_s: 162.6242184638977\n",
            "  time_total_s: 162.6242184638977\n",
            "  timestamp: 1615984199\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 1119b43c\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 6.1/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=6\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7776397939579915\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.42 GiB heap, 0.0/2.54 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-17_11-51-38\n",
            "Number of trials: 9/10 (1 PENDING, 8 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_1bf85a56 | PENDING    |       |     2.41069e-05 |                  3 |                             8 |  7.10508 |             |\n",
            "| _inner_2182b94e | TERMINATED |       |     9.39003e-05 |                  1 |                            32 | 36.0425  |    0.788077 |\n",
            "| _inner_21872f74 | TERMINATED |       |     1.91335e-06 |                  3 |                            32 |  8.09127 |    0.740807 |\n",
            "| _inner_9be36dc8 | TERMINATED |       |     1.59723e-05 |                  4 |                            32 | 21.4019  |    0.782135 |\n",
            "| _inner_3646bbfe | TERMINATED |       |     1.79784e-06 |                  2 |                             4 | 13.0258  |    0.57451  |\n",
            "| _inner_08f0c710 | TERMINATED |       |     4.07212e-05 |                  3 |                             4 | 28.0017  |    0.654877 |\n",
            "| _inner_3f985d6e | TERMINATED |       |     8.02598e-06 |                  4 |                            32 |  2.6107  |    0.774821 |\n",
            "| _inner_7675b2dc | TERMINATED |       |     1.16336e-06 |                  3 |                            64 | 12.5797  |    0.684109 |\n",
            "| _inner_1119b43c | TERMINATED |       |     6.4134e-05  |                  3 |                            16 | 31.506   |    0.746966 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1384)\u001b[0m {'eval_loss': 0.5140169858932495, 'eval_accuracy': 0.7415, 'eval_f1': 0.7469664776146879, 'eval_precision': 0.7568265798380905, 'eval_recall': 0.7415, 'eval_runtime': 7.9892, 'eval_samples_per_second': 250.339, 'epoch': 0.64}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1419)\u001b[0m 2021-03-17 12:30:01.743876: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1419)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "\u001b[2m\u001b[36m(pid=1419)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1419)\u001b[0m {'loss': 0.4944, 'learning_rate': 2.1535492040104403e-05, 'epoch': 0.32}\n",
            "Result for _inner_1bf85a56:\n",
            "  date: 2021-03-17_12-31-53\n",
            "  done: true\n",
            "  epoch: 0.32\n",
            "  eval_accuracy: 0.742\n",
            "  eval_f1: 0.7503533246414602\n",
            "  eval_loss: 0.4928911626338959\n",
            "  eval_precision: 0.7729951551386455\n",
            "  eval_recall: 0.742\n",
            "  eval_runtime: 7.9808\n",
            "  eval_samples_per_second: 250.6\n",
            "  experiment_id: 71126dabd6c9496a883dbf8f18902818\n",
            "  hostname: 30101e747662\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7503533246414602\n",
            "  pid: 1419\n",
            "  time_since_restore: 112.51147770881653\n",
            "  time_this_iter_s: 112.51147770881653\n",
            "  time_total_s: 112.51147770881653\n",
            "  timestamp: 1615984313\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 1bf85a56\n",
            "  \n",
            "== Status ==\n",
            "Memory usage on this node: 6.1/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=7\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7748205457248436\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.42 GiB heap, 0.0/2.54 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-17_11-51-38\n",
            "Number of trials: 10/10 (1 PENDING, 9 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_7d8d3bb0 | PENDING    |       |     1.72005e-05 |                  4 |                             4 | 22.2483  |             |\n",
            "| _inner_2182b94e | TERMINATED |       |     9.39003e-05 |                  1 |                            32 | 36.0425  |    0.788077 |\n",
            "| _inner_21872f74 | TERMINATED |       |     1.91335e-06 |                  3 |                            32 |  8.09127 |    0.740807 |\n",
            "| _inner_9be36dc8 | TERMINATED |       |     1.59723e-05 |                  4 |                            32 | 21.4019  |    0.782135 |\n",
            "| _inner_3646bbfe | TERMINATED |       |     1.79784e-06 |                  2 |                             4 | 13.0258  |    0.57451  |\n",
            "| _inner_08f0c710 | TERMINATED |       |     4.07212e-05 |                  3 |                             4 | 28.0017  |    0.654877 |\n",
            "| _inner_3f985d6e | TERMINATED |       |     8.02598e-06 |                  4 |                            32 |  2.6107  |    0.774821 |\n",
            "| _inner_7675b2dc | TERMINATED |       |     1.16336e-06 |                  3 |                            64 | 12.5797  |    0.684109 |\n",
            "| _inner_1119b43c | TERMINATED |       |     6.4134e-05  |                  3 |                            16 | 31.506   |    0.746966 |\n",
            "| _inner_1bf85a56 | TERMINATED |       |     2.41069e-05 |                  3 |                             8 |  7.10508 |    0.750353 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "\u001b[2m\u001b[36m(pid=1419)\u001b[0m {'eval_loss': 0.4928911626338959, 'eval_accuracy': 0.742, 'eval_f1': 0.7503533246414602, 'eval_precision': 0.7729951551386455, 'eval_recall': 0.742, 'eval_runtime': 7.9808, 'eval_samples_per_second': 250.6, 'epoch': 0.32}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1452)\u001b[0m 2021-03-17 12:31:55.351798: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[2m\u001b[36m(pid=1452)\u001b[0m Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased-conversational and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "\u001b[2m\u001b[36m(pid=1452)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[2m\u001b[36m(pid=1452)\u001b[0m {'loss': 0.5537, 'learning_rate': 1.651245813229014e-05, 'epoch': 0.16}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2021-03-17 12:33:25,783\tINFO tune.py:450 -- Total run time: 2507.68 seconds (2507.66 seconds for the tuning loop).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Result for _inner_7d8d3bb0:\n",
            "  date: 2021-03-17_12-33-25\n",
            "  done: true\n",
            "  epoch: 0.16\n",
            "  eval_accuracy: 0.757\n",
            "  eval_f1: 0.7366643352449901\n",
            "  eval_loss: 0.6576597094535828\n",
            "  eval_precision: 0.7429200033513469\n",
            "  eval_recall: 0.757\n",
            "  eval_runtime: 7.9077\n",
            "  eval_samples_per_second: 252.917\n",
            "  experiment_id: bd494ddec0ed4499a77e781a75dba360\n",
            "  hostname: 30101e747662\n",
            "  iterations_since_restore: 1\n",
            "  node_ip: 172.28.0.2\n",
            "  objective: 0.7366643352449901\n",
            "  pid: 1452\n",
            "  time_since_restore: 91.10976338386536\n",
            "  time_this_iter_s: 91.10976338386536\n",
            "  time_total_s: 91.10976338386536\n",
            "  timestamp: 1615984405\n",
            "  timesteps_since_restore: 0\n",
            "  training_iteration: 1\n",
            "  trial_id: 7d8d3bb0\n",
            "  \n",
            "\u001b[2m\u001b[36m(pid=1452)\u001b[0m {'eval_loss': 0.6576597094535828, 'eval_accuracy': 0.757, 'eval_f1': 0.7366643352449901, 'eval_precision': 0.7429200033513469, 'eval_recall': 0.757, 'eval_runtime': 7.9077, 'eval_samples_per_second': 252.917, 'epoch': 0.16}\n",
            "== Status ==\n",
            "Memory usage on this node: 6.1/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=8\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7687037404539978\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.42 GiB heap, 0.0/2.54 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-17_11-51-38\n",
            "Number of trials: 10/10 (10 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_2182b94e | TERMINATED |       |     9.39003e-05 |                  1 |                            32 | 36.0425  |    0.788077 |\n",
            "| _inner_21872f74 | TERMINATED |       |     1.91335e-06 |                  3 |                            32 |  8.09127 |    0.740807 |\n",
            "| _inner_9be36dc8 | TERMINATED |       |     1.59723e-05 |                  4 |                            32 | 21.4019  |    0.782135 |\n",
            "| _inner_3646bbfe | TERMINATED |       |     1.79784e-06 |                  2 |                             4 | 13.0258  |    0.57451  |\n",
            "| _inner_08f0c710 | TERMINATED |       |     4.07212e-05 |                  3 |                             4 | 28.0017  |    0.654877 |\n",
            "| _inner_3f985d6e | TERMINATED |       |     8.02598e-06 |                  4 |                            32 |  2.6107  |    0.774821 |\n",
            "| _inner_7675b2dc | TERMINATED |       |     1.16336e-06 |                  3 |                            64 | 12.5797  |    0.684109 |\n",
            "| _inner_1119b43c | TERMINATED |       |     6.4134e-05  |                  3 |                            16 | 31.506   |    0.746966 |\n",
            "| _inner_1bf85a56 | TERMINATED |       |     2.41069e-05 |                  3 |                             8 |  7.10508 |    0.750353 |\n",
            "| _inner_7d8d3bb0 | TERMINATED |       |     1.72005e-05 |                  4 |                             4 | 22.2483  |    0.736664 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n",
            "== Status ==\n",
            "Memory usage on this node: 6.0/12.7 GiB\n",
            "Using AsyncHyperBand: num_stopped=8\n",
            "Bracket: Iter 64.000: None | Iter 16.000: None | Iter 4.000: None | Iter 1.000: 0.7687037404539978\n",
            "Resources requested: 0/2 CPUs, 0/1 GPUs, 0.0/7.42 GiB heap, 0.0/2.54 GiB objects (0/1.0 accelerator_type:T4)\n",
            "Result logdir: /root/ray_results/_inner_2021-03-17_11-51-38\n",
            "Number of trials: 10/10 (10 TERMINATED)\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "| Trial name      | status     | loc   |   learning_rate |   num_train_epochs |   per_device_train_batch_size |     seed |   objective |\n",
            "|-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------|\n",
            "| _inner_2182b94e | TERMINATED |       |     9.39003e-05 |                  1 |                            32 | 36.0425  |    0.788077 |\n",
            "| _inner_21872f74 | TERMINATED |       |     1.91335e-06 |                  3 |                            32 |  8.09127 |    0.740807 |\n",
            "| _inner_9be36dc8 | TERMINATED |       |     1.59723e-05 |                  4 |                            32 | 21.4019  |    0.782135 |\n",
            "| _inner_3646bbfe | TERMINATED |       |     1.79784e-06 |                  2 |                             4 | 13.0258  |    0.57451  |\n",
            "| _inner_08f0c710 | TERMINATED |       |     4.07212e-05 |                  3 |                             4 | 28.0017  |    0.654877 |\n",
            "| _inner_3f985d6e | TERMINATED |       |     8.02598e-06 |                  4 |                            32 |  2.6107  |    0.774821 |\n",
            "| _inner_7675b2dc | TERMINATED |       |     1.16336e-06 |                  3 |                            64 | 12.5797  |    0.684109 |\n",
            "| _inner_1119b43c | TERMINATED |       |     6.4134e-05  |                  3 |                            16 | 31.506   |    0.746966 |\n",
            "| _inner_1bf85a56 | TERMINATED |       |     2.41069e-05 |                  3 |                             8 |  7.10508 |    0.750353 |\n",
            "| _inner_7d8d3bb0 | TERMINATED |       |     1.72005e-05 |                  4 |                             4 | 22.2483  |    0.736664 |\n",
            "+-----------------+------------+-------+-----------------+--------------------+-------------------------------+----------+-------------+\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jZV5aryhLqBi",
        "outputId": "2c2afbed-ad7e-4271-9ff7-109494071074"
      },
      "source": [
        "best_run"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BestRun(run_id='2182b94e', objective=0.7880768352084142, hyperparameters={'learning_rate': 9.390028242887286e-05, 'num_train_epochs': 1, 'seed': 36.04249333663025, 'per_device_train_batch_size': 32})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "xUH1gIAowYRl"
      },
      "source": [
        "#!rm -r ./bert1"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}