modelTraining.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "17c8de5f3dd64f67a6f43013e2496272": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6b1086093c6e41c6917764e78140ce3e",
              "IPY_MODEL_527bb2a801ef47ac85aae4bba833a682",
              "IPY_MODEL_eb84bec3c9b84f3794a2938b05b8cfa0"
            ],
            "layout": "IPY_MODEL_29f4a0f59f994be59c691d18722ffbde"
          }
        },
        "6b1086093c6e41c6917764e78140ce3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e07bf3046f9e44d39458ce00731d6376",
            "placeholder": "​",
            "style": "IPY_MODEL_8c0176362702429590dff8bcf4a64876",
            "value": "Downloading (…)olve/main/vocab.json: 100%"
          }
        },
        "527bb2a801ef47ac85aae4bba833a682": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8ea8e4d592cb4c45b4385d9bb912f08a",
            "max": 1042301,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_369c3d998af048ddbbcadc0a43001af3",
            "value": 1042301
          }
        },
        "eb84bec3c9b84f3794a2938b05b8cfa0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_05dd5c53aab8448f891dfaac4fe03e20",
            "placeholder": "​",
            "style": "IPY_MODEL_ac600ecf514c4f19aafc9ce90c2b4324",
            "value": " 1.04M/1.04M [00:00&lt;00:00, 4.18MB/s]"
          }
        },
        "29f4a0f59f994be59c691d18722ffbde": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e07bf3046f9e44d39458ce00731d6376": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8c0176362702429590dff8bcf4a64876": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8ea8e4d592cb4c45b4385d9bb912f08a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "369c3d998af048ddbbcadc0a43001af3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "05dd5c53aab8448f891dfaac4fe03e20": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ac600ecf514c4f19aafc9ce90c2b4324": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0a63fbe7ad4246359a2b98d8b34eadf9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_921a86b1bcec42299575353cdd3aa7ed",
              "IPY_MODEL_d1c5e55eaaba4cb9899508dd249ff66b",
              "IPY_MODEL_b6035c0785ff422ba7d18116c8937ec2"
            ],
            "layout": "IPY_MODEL_0a01ece99181447bbbc7f769d670f6b9"
          }
        },
        "921a86b1bcec42299575353cdd3aa7ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a7ed7d90fcc34a78ba390789a14dfda0",
            "placeholder": "​",
            "style": "IPY_MODEL_e0e4c25eda1b4577a7b5963c4c1b34e3",
            "value": "Downloading (…)olve/main/merges.txt: 100%"
          }
        },
        "d1c5e55eaaba4cb9899508dd249ff66b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_db2651278bd24043a23b1d3ab3d671ad",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d9755f38509141dda6e07b3b17fde939",
            "value": 456318
          }
        },
        "b6035c0785ff422ba7d18116c8937ec2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_87e09ceda8f541d79d7f6840edfbf76a",
            "placeholder": "​",
            "style": "IPY_MODEL_1b5916517bce4eee964e33e1bd5d2f88",
            "value": " 456k/456k [00:00&lt;00:00, 2.46MB/s]"
          }
        },
        "0a01ece99181447bbbc7f769d670f6b9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a7ed7d90fcc34a78ba390789a14dfda0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e0e4c25eda1b4577a7b5963c4c1b34e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "db2651278bd24043a23b1d3ab3d671ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d9755f38509141dda6e07b3b17fde939": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "87e09ceda8f541d79d7f6840edfbf76a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1b5916517bce4eee964e33e1bd5d2f88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f46bf63b024042e2a2de4c419827a078": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_297a77faf2b6411a83f9bd98383feaab",
              "IPY_MODEL_d6de6b95d2874a558808b56d33d23951",
              "IPY_MODEL_dc03c6d966b048c09eab6bcd2e269c11"
            ],
            "layout": "IPY_MODEL_0f47e1c5aad5412ab7f1c486df2de2f9"
          }
        },
        "297a77faf2b6411a83f9bd98383feaab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3bb5acdbe4ae4de3a2c8dce71a7f4afc",
            "placeholder": "​",
            "style": "IPY_MODEL_2594994d314645c8b013d768e8fd0276",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "d6de6b95d2874a558808b56d33d23951": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f4929203d45f4d75aafc76a97be94b40",
            "max": 665,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_661ab198e0df4340bdd590ef52e8a3eb",
            "value": 665
          }
        },
        "dc03c6d966b048c09eab6bcd2e269c11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5d6d7515e0714718848f52e5adf9c9a6",
            "placeholder": "​",
            "style": "IPY_MODEL_1cd21ea7e0b54dd692d97211216cc386",
            "value": " 665/665 [00:00&lt;00:00, 55.0kB/s]"
          }
        },
        "0f47e1c5aad5412ab7f1c486df2de2f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3bb5acdbe4ae4de3a2c8dce71a7f4afc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2594994d314645c8b013d768e8fd0276": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f4929203d45f4d75aafc76a97be94b40": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "661ab198e0df4340bdd590ef52e8a3eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5d6d7515e0714718848f52e5adf9c9a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1cd21ea7e0b54dd692d97211216cc386": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6cff2dc301724f4794c1db077d7aa754": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9c5a9947c1c1478eab9b49bb27cc1063",
              "IPY_MODEL_a9d9dba7bcab4ff7a15f795477793fa0",
              "IPY_MODEL_5f546334e74d463385499d6d2515fc97"
            ],
            "layout": "IPY_MODEL_bd294811a7ff4dbf946bd226114e52eb"
          }
        },
        "9c5a9947c1c1478eab9b49bb27cc1063": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8fab96357e0e4099873ff79de3684a50",
            "placeholder": "​",
            "style": "IPY_MODEL_060e38e5d8f940448f280e866a5947b8",
            "value": "Downloading pytorch_model.bin: 100%"
          }
        },
        "a9d9dba7bcab4ff7a15f795477793fa0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2b9e45f11d804abaaa6090b5bc973378",
            "max": 548118077,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c3cb945c53104c76bf43ed437a9e70e7",
            "value": 548118077
          }
        },
        "5f546334e74d463385499d6d2515fc97": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8e25881f1a9f49a397fe221fb650a733",
            "placeholder": "​",
            "style": "IPY_MODEL_521e39f6bc924d2599027b069891a52d",
            "value": " 548M/548M [00:02&lt;00:00, 252MB/s]"
          }
        },
        "bd294811a7ff4dbf946bd226114e52eb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8fab96357e0e4099873ff79de3684a50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "060e38e5d8f940448f280e866a5947b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2b9e45f11d804abaaa6090b5bc973378": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c3cb945c53104c76bf43ed437a9e70e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8e25881f1a9f49a397fe221fb650a733": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "521e39f6bc924d2599027b069891a52d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "976078fb7bfc47e18b27ded3cd79be60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_bd34095727f147ada67f90e2c345c405",
              "IPY_MODEL_83770b8879774fc89d3b66e361b093e9",
              "IPY_MODEL_36f4d9b28a2940f58be650d338dd4fb1"
            ],
            "layout": "IPY_MODEL_5fbe979b207e4c25a9b33eddc125a479"
          }
        },
        "bd34095727f147ada67f90e2c345c405": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c9cb64a9910945b494b88119f67f41c7",
            "placeholder": "​",
            "style": "IPY_MODEL_e48f29179d374e46aa987e47f0eecf32",
            "value": "Downloading (…)neration_config.json: 100%"
          }
        },
        "83770b8879774fc89d3b66e361b093e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8f3fcf0552b24366af075ef7d02d2390",
            "max": 124,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ac6b05d1363c4837a01ad206be68ab3c",
            "value": 124
          }
        },
        "36f4d9b28a2940f58be650d338dd4fb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0d5807e8a6f7495bb4114efdf18fe9b5",
            "placeholder": "​",
            "style": "IPY_MODEL_67874a52698f43349eabb3f15c40824a",
            "value": " 124/124 [00:00&lt;00:00, 5.74kB/s]"
          }
        },
        "5fbe979b207e4c25a9b33eddc125a479": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c9cb64a9910945b494b88119f67f41c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e48f29179d374e46aa987e47f0eecf32": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8f3fcf0552b24366af075ef7d02d2390": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ac6b05d1363c4837a01ad206be68ab3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0d5807e8a6f7495bb4114efdf18fe9b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "67874a52698f43349eabb3f15c40824a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Poem Generator<br>\n",
        "Dartmouth COSC 72 Final Project<br>\n",
        "Authors: Zhoucai Ni and Alex Kruger<br>\n",
        "Emails:\n",
        "zhoucai.ni.24@dartmouth.edu | alexander.j.kruger.23@dartmouth.edu<br>\n",
        "Description: Uses Pytorch's GPT2 and a dataset of poems from The Poetry Foundation to train a neural network that generates new poetry based on a prompt"
      ],
      "metadata": {
        "id": "nYGp4YxAu6wB"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Install necessary packages"
      ],
      "metadata": {
        "id": "tVRCNT0wvtBO"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "The following code draws inspiration from the week 6 homework on GPT-2 Training and from  https://scottmduda.medium.com/generating-an-edgar-allen-poe-styled-poem-using-gpt-2-289801ded82c"
      ],
      "metadata": {
        "id": "6v4PNr-ZsGYR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install tokenizer\n",
        "!pip install transformers"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5E62JN_nDk55",
        "outputId": "62c4108b-55b3-49d5-f7a2-a9e59e56ec70"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting tokenizer\n",
            "  Downloading tokenizer-3.4.2-py2.py3-none-any.whl (79 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.1/79.1 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: tokenizer\n",
            "Successfully installed tokenizer-3.4.2\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.29.2-py3-none-any.whl (7.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.1/7.1 MB\u001b[0m \u001b[31m73.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.0)\n",
            "Collecting huggingface-hub<1.0,>=0.14.1 (from transformers)\n",
            "  Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m236.8/236.8 kB\u001b[0m \u001b[31m31.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m78.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.15.1 tokenizers-0.13.3 transformers-4.29.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Import important packages and tools"
      ],
      "metadata": {
        "id": "j9JJmPCXvzRG"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "qeQZJ9XIC8Ua"
      },
      "outputs": [],
      "source": [
        "# General modules\n",
        "import numpy as np\n",
        "import pandas as pd \n",
        "import random\n",
        "import os\n",
        "\n",
        "# time related modules\n",
        "import time\n",
        "import datetime\n",
        "\n",
        "\n",
        "# PyTorch is an open source machine learning library based on the Torch library, used for applications such as computer vision and natural language processing, primarily developed by Facebook's AI Research lab.\n",
        "import torch\n",
        "\n",
        "# Transformers provides state-of-the-art general-purpose architectures for Natural Language Understanding (NLU) and Natural Language Generation (NLG).\n",
        "# GPT2Tokenizer is used to tokenize the text data for GPT-2 model.\n",
        "# GPT2LMHeadModel represents the GPT-2 model with a language modeling head.\n",
        "# GPT2Config represents the configuration of the GPT-2 model.\n",
        "# AdamW is a class representing the Adam optimizer with weight decay.\n",
        "# get_linear_schedule_with_warmup creates a schedule with a learning rate that decreases linearly after linearly increasing during a warmup period.\n",
        "from transformers import GPT2Tokenizer, GPT2LMHeadModel, GPT2Config, AdamW, get_linear_schedule_with_warmup\n",
        "\n",
        "# Dataset represents a Python iterable over a dataset.\n",
        "# random_split is a function that splits the dataset into non-overlapping new datasets of given lengths.\n",
        "# DataLoader combines a dataset and a sampler and provides an iterable over the given dataset.\n",
        "# RandomSampler samples elements randomly.\n",
        "# SequentialSampler samples elements sequentially.\n",
        "from torch.utils.data import Dataset, random_split, DataLoader, RandomSampler, SequentialSampler\n",
        "import plotly.express as px\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Set global variables"
      ],
      "metadata": {
        "id": "TahCmYXgv5wf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "RAND_SEED = 73\n",
        "BATCH_SIZE = 2\n",
        "EPOCHS = 1\n",
        "MAX_LEN = 1024"
      ],
      "metadata": {
        "id": "Jb4qN9pTEdT9"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Import data from Poetry Foundation as a pandas dataframe<br>\n",
        "data can be downloaded here: https://www.kaggle.com/datasets/tgdivy/poetry-foundation-poems"
      ],
      "metadata": {
        "id": "bRz5Tn-hv-pZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "poem_stanza_df = pd.read_csv('PoetryFoundationData.csv')\n",
        "poem_stanza_df = poem_stanza_df.fillna('')\n",
        "\n",
        "# Uncomment to train a smaller subset for specifc Poets\n",
        "# Bob_frost_df = poem_stanza_df[poem_stanza_df['Poet'] == 'Robert Frost']\n",
        "\n",
        "# Bob_frost_df['Poem']"
      ],
      "metadata": {
        "id": "1KmZR8jrE8_A"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Prepare tokenizer"
      ],
      "metadata": {
        "id": "wJ6GRnz4wdN2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
        "special_tokens_dict = {\n",
        "    'bos_token': '<BOS>', \n",
        "    'eos_token': '<EOS>', \n",
        "    'pad_token': '<PAD>'}\n",
        "num_added_tokens = tokenizer.add_special_tokens(special_tokens_dict)"
      ],
      "metadata": {
        "id": "s1cpstdsDC3C",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "17c8de5f3dd64f67a6f43013e2496272",
            "6b1086093c6e41c6917764e78140ce3e",
            "527bb2a801ef47ac85aae4bba833a682",
            "eb84bec3c9b84f3794a2938b05b8cfa0",
            "29f4a0f59f994be59c691d18722ffbde",
            "e07bf3046f9e44d39458ce00731d6376",
            "8c0176362702429590dff8bcf4a64876",
            "8ea8e4d592cb4c45b4385d9bb912f08a",
            "369c3d998af048ddbbcadc0a43001af3",
            "05dd5c53aab8448f891dfaac4fe03e20",
            "ac600ecf514c4f19aafc9ce90c2b4324",
            "0a63fbe7ad4246359a2b98d8b34eadf9",
            "921a86b1bcec42299575353cdd3aa7ed",
            "d1c5e55eaaba4cb9899508dd249ff66b",
            "b6035c0785ff422ba7d18116c8937ec2",
            "0a01ece99181447bbbc7f769d670f6b9",
            "a7ed7d90fcc34a78ba390789a14dfda0",
            "e0e4c25eda1b4577a7b5963c4c1b34e3",
            "db2651278bd24043a23b1d3ab3d671ad",
            "d9755f38509141dda6e07b3b17fde939",
            "87e09ceda8f541d79d7f6840edfbf76a",
            "1b5916517bce4eee964e33e1bd5d2f88",
            "f46bf63b024042e2a2de4c419827a078",
            "297a77faf2b6411a83f9bd98383feaab",
            "d6de6b95d2874a558808b56d33d23951",
            "dc03c6d966b048c09eab6bcd2e269c11",
            "0f47e1c5aad5412ab7f1c486df2de2f9",
            "3bb5acdbe4ae4de3a2c8dce71a7f4afc",
            "2594994d314645c8b013d768e8fd0276",
            "f4929203d45f4d75aafc76a97be94b40",
            "661ab198e0df4340bdd590ef52e8a3eb",
            "5d6d7515e0714718848f52e5adf9c9a6",
            "1cd21ea7e0b54dd692d97211216cc386"
          ]
        },
        "outputId": "2d88fc04-a776-475c-e800-56fbedac2428"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "17c8de5f3dd64f67a6f43013e2496272"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0a63fbe7ad4246359a2b98d8b34eadf9"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/665 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f46bf63b024042e2a2de4c419827a078"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Use a class to better represent data as an object"
      ],
      "metadata": {
        "id": "3RjWHqeVxD8b"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class PoemDataset(Dataset):\n",
        "    \"\"\"\n",
        "    Custom Dataset subclass.\n",
        "    The dataset reads a list of strings (data), tokenizes them using a pre-specified tokenizer, and returns\n",
        "    their corresponding input_ids and attention_masks as tensors.\n",
        "    \n",
        "    :param data: List of strings to tokenize.\n",
        "    :param tokenizer: Tokenizer object to be used to tokenize data.\n",
        "    :param gpt2_type: (Optional) Type of GPT-2 used.\n",
        "    :param max_length: (Optional) Maximum length of the sequences.\n",
        "    \"\"\"\n",
        "    def __init__(self, data, tokenizer, gpt2_type='gpt2', max_length=MAX_LEN):\n",
        "        self.tokenizer = tokenizer\n",
        "        self.input_ids = []\n",
        "        self.attn_masks = []\n",
        "        \n",
        "        # Iterate over data, tokenize each sequence and append its input_id and attention_mask to respective lists\n",
        "        for i in data:\n",
        "            encodings_dict = tokenizer('<BOS>' + i + '<EOS>',\n",
        "                                     truncation=True,\n",
        "                                     max_length=max_length,\n",
        "                                     padding='max_length')\n",
        "\n",
        "            self.input_ids.append(torch.tensor(encodings_dict['input_ids']))\n",
        "            self.attn_masks.append(torch.tensor(encodings_dict['attention_mask']))\n",
        "\n",
        "    def __len__(self):\n",
        "        \"\"\"\n",
        "        Returns the number of sequences in data.\n",
        "        \n",
        "        :return: number of sequences in data\n",
        "        \"\"\"\n",
        "        return len(self.input_ids)\n",
        "    \n",
        "    def __getitem__(self, idx):\n",
        "        \"\"\"\n",
        "        Returns the input_id and attention_mask tensors of the sequence at the provided index.\n",
        "        \n",
        "        :param idx: index to access\n",
        "        :return: tensors of input_id and attention_mask of the sequence at the provided index.\n",
        "        \"\"\"\n",
        "        return self.input_ids[idx], self.attn_masks[idx]\n",
        "\n",
        "    \n"
      ],
      "metadata": {
        "id": "3C8UDHVQESvW"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "poem_stanza_dataset = PoemDataset(poem_stanza_df['Poem'].values, tokenizer, max_length=MAX_LEN)"
      ],
      "metadata": {
        "id": "HnyYis0zFZGR"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Split data into training and validation sets"
      ],
      "metadata": {
        "id": "6en1vs5ExSIe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def train_val_split(split, dataset):\n",
        "    \"\"\"\n",
        "    Calculates the size of the training and validation datasets.\n",
        "    \n",
        "    :param split: Float representing the proportion of data to be used for training. \n",
        "    Should be between 0 and 1, where 0 means no data for training, and 1 means all data for training.\n",
        "    :param dataset: The dataset to be split into training and validation sets.\n",
        "    :return: The sizes of the training and validation datasets.\n",
        "    \"\"\"\n",
        "     # calculate the size of the training dataset\n",
        "    train_size = int(split * len(dataset)) \n",
        "    # the remaining data will be used for validation\n",
        "    val_size = len(dataset) - train_size    \n",
        "    return train_size, val_size\n",
        "\n",
        "\n",
        "# Use the function defined above to split the PoemDataset into training and validation datasets\n",
        "poem_stanza_train_size, poem_stanza_val_size = train_val_split(0.8, poem_stanza_dataset)\n",
        "\n",
        "# Use PyTorch's random_split function to randomly split the PoemDataset into training and validation datasets\n",
        "poem_stanza_train_dataset, poem_stanza_val_dataset = random_split(poem_stanza_dataset, \n",
        "                                                                  [poem_stanza_train_size, poem_stanza_val_size])\n"
      ],
      "metadata": {
        "id": "1_AmXLsTGrOF"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Use our random seed global variable to initialize the randomizer and PyTorch"
      ],
      "metadata": {
        "id": "i5gKWI7SxcSj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "torch.cuda.manual_seed_all(RAND_SEED)\n",
        "random.seed(RAND_SEED)\n",
        "np.random.seed(RAND_SEED)\n",
        "torch.manual_seed(RAND_SEED)"
      ],
      "metadata": {
        "id": "GyNY66sTHqcW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "47b25712-d1a2-45e7-b2d7-959e97aadaea"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch._C.Generator at 0x7fcf648fde30>"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Apply the data loader to the training and validation sets"
      ],
      "metadata": {
        "id": "YXyMHRcrx9lZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "poem_stanza_train_dataloader = DataLoader(poem_stanza_train_dataset,\n",
        "                              sampler=RandomSampler(poem_stanza_train_dataset),\n",
        "                              batch_size=BATCH_SIZE)\n",
        "\n",
        "poem_stanza_val_dataloader = DataLoader(poem_stanza_val_dataset,\n",
        "                            sampler=SequentialSampler(poem_stanza_val_dataset),\n",
        "                            batch_size=BATCH_SIZE)"
      ],
      "metadata": {
        "id": "geDmYFTvIXjC"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Log time and initialize hyperparameters"
      ],
      "metadata": {
        "id": "yBQhJI03ylDE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# helper function for logging time\n",
        "def format_time(elapsed):\n",
        "    return str(datetime.timedelta(seconds=int(round((elapsed)))))\n",
        "\n",
        "# hyperparameters\n",
        "learning_rate = 1e-3\n",
        "eps = 1e-8\n",
        "warmup_steps = 50\n",
        "device = torch.device('cuda')"
      ],
      "metadata": {
        "id": "OgmD1yuiI087"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Configure and optimize the model"
      ],
      "metadata": {
        "id": "n7UK3F8Syzs5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a configuration object for a GPT2 model.\n",
        "# The configuration specifies the model architecture including the vocabulary size and the maximum length of position embeddings.\n",
        "# 'from_pretrained' method is used to initialize the configuration with the pretrained 'gpt2' model configuration.\n",
        "# output_hidden_states is set to True, which means that the model will return all hidden states.\n",
        "configuration = GPT2Config(vocab_size=len(tokenizer), n_positions=MAX_LEN).from_pretrained('gpt2', output_hidden_states=True)\n",
        "\n",
        "# Initialize a GPT2LMHeadModel with the above configuration.\n",
        "# GPT2LMHeadModel is the GPT2 model with a language modeling head, i.e., a linear layer on top of the hidden states output.\n",
        "# 'from_pretrained' method is used to initialize the model with the pretrained 'gpt2' weights.\n",
        "poem_stanza_model = GPT2LMHeadModel.from_pretrained('gpt2', config=configuration)\n",
        "\n",
        "# Resize the token embeddings of the model in case the current size doesn't match with the provided tokenizer's vocabulary size.\n",
        "# This is necessary when you've added some special tokens or used a different tokenizer than the one originally used to train 'gpt2'.\n",
        "poem_stanza_model.resize_token_embeddings(len(tokenizer))\n",
        "\n",
        "# Move the model to GPU for faster computations.\n",
        "poem_stanza_model.cuda()\n",
        "\n",
        "# Initialize the AdamW optimizer, which is an Adam optimizer with weight decay.\n",
        "# The parameters of the model will be updated by this optimizer during the training.\n",
        "optimizer = AdamW(poem_stanza_model.parameters(), lr=learning_rate, eps=eps)\n",
        "\n",
        "# Compute the total number of training steps.\n",
        "# This is used by the learning rate scheduler.\n",
        "total_steps = len(poem_stanza_train_dataloader) * EPOCHS\n",
        "\n",
        "# Initialize the learning rate scheduler.\n",
        "# We use a scheduler that linearly decreases the learning rate from the maximum value to 0, after a warmup period during which it linearly increases.\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer,\n",
        "                                            num_warmup_steps=warmup_steps,\n",
        "                                            num_training_steps=total_steps)\n",
        "\n",
        "\n",
        "# Move the model to the specific device (GPU/CPU).\n",
        "poem_stanza_model = poem_stanza_model.to(device)\n"
      ],
      "metadata": {
        "id": "ezhlhCflMarv",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 137,
          "referenced_widgets": [
            "6cff2dc301724f4794c1db077d7aa754",
            "9c5a9947c1c1478eab9b49bb27cc1063",
            "a9d9dba7bcab4ff7a15f795477793fa0",
            "5f546334e74d463385499d6d2515fc97",
            "bd294811a7ff4dbf946bd226114e52eb",
            "8fab96357e0e4099873ff79de3684a50",
            "060e38e5d8f940448f280e866a5947b8",
            "2b9e45f11d804abaaa6090b5bc973378",
            "c3cb945c53104c76bf43ed437a9e70e7",
            "8e25881f1a9f49a397fe221fb650a733",
            "521e39f6bc924d2599027b069891a52d",
            "976078fb7bfc47e18b27ded3cd79be60",
            "bd34095727f147ada67f90e2c345c405",
            "83770b8879774fc89d3b66e361b093e9",
            "36f4d9b28a2940f58be650d338dd4fb1",
            "5fbe979b207e4c25a9b33eddc125a479",
            "c9cb64a9910945b494b88119f67f41c7",
            "e48f29179d374e46aa987e47f0eecf32",
            "8f3fcf0552b24366af075ef7d02d2390",
            "ac6b05d1363c4837a01ad206be68ab3c",
            "0d5807e8a6f7495bb4114efdf18fe9b5",
            "67874a52698f43349eabb3f15c40824a"
          ]
        },
        "outputId": "8fb39f80-1472-4060-d160-687c0a4987c2"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading pytorch_model.bin:   0%|          | 0.00/548M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6cff2dc301724f4794c1db077d7aa754"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)neration_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "976078fb7bfc47e18b27ded3cd79be60"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:407: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Train the Model"
      ],
      "metadata": {
        "id": "cUIGmE4jzOP1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Initialize an empty list to hold the losses during training\n",
        "losses = []\n",
        "valid_losses =[]\n",
        "start_time = time.time()\n",
        "\n",
        "for epoch_i in range(0, EPOCHS):\n",
        "\n",
        "    print(f'Epoch {epoch_i + 1} of {EPOCHS}')\n",
        "\n",
        "    t0 = time.time()\n",
        "    \n",
        "    # Reset the total training loss for this epoch\n",
        "    total_train_loss = 0\n",
        "\n",
        "    poem_stanza_model.train()\n",
        "\n",
        "    # Loop over each batch from the training data loader\n",
        "    for step, batch in enumerate(poem_stanza_train_dataloader):\n",
        "\n",
        "        # Move the input ids, labels and masks to the GPU\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_labels = batch[0].to(device)\n",
        "        b_masks = batch[1].to(device)\n",
        "\n",
        "        # Clear out the gradients from the previous training step\n",
        "        poem_stanza_model.zero_grad()        \n",
        "\n",
        "        # Forward pass: compute the outputs of the model by passing in the input\n",
        "        outputs = poem_stanza_model(b_input_ids, labels=b_labels, attention_mask=b_masks, token_type_ids=None)\n",
        "\n",
        "        # Extract the loss from the outputs\n",
        "        loss = outputs[0]  \n",
        "\n",
        "        # Extract and accumulate the total loss\n",
        "        batch_loss = loss.item()\n",
        "        total_train_loss += batch_loss\n",
        "\n",
        "        # Perform a backward pass to calculate gradients\n",
        "        loss.backward()\n",
        "\n",
        "        # Update parameters\n",
        "        optimizer.step()\n",
        "\n",
        "        # Update the learning rate\n",
        "        scheduler.step()\n",
        "\n",
        "        losses.append(loss.item())\n",
        "        \n",
        "\n",
        "        if step % 50 == 0:\n",
        "            print(f\"Step: {step}, Loss: {loss.item():.4f}\")\n",
        "\n",
        "        # Break the loop after 1000 steps.\n",
        "        if step > 1000:\n",
        "            break\n",
        "\n",
        "    # Calculate the average training loss for this epoch\n",
        "    avg_train_loss = total_train_loss / len(poem_stanza_train_dataloader)       \n",
        "\n",
        "    # Calculate the time spent on this epoch\n",
        "    training_time = format_time(time.time() - t0)\n",
        "\n",
        "    # Print the average training loss and time spent on this epoch\n",
        "    print(f'Average Training Loss: {avg_train_loss}. Epoch Training Time: {training_time}')\n",
        "\n",
        "    # Set the model to 'eval' mode. This is important when the model has layers like dropout, batchnorm etc. which behave differently during training and evaluation.\n",
        "    poem_stanza_model.eval()\n",
        "\n",
        "    # Reset the total validation loss\n",
        "    total_eval_loss = 0\n",
        "    nb_eval_steps = 0\n",
        "\n",
        "    # Loop over each batch from the validation data loader\n",
        "    for batch in poem_stanza_val_dataloader:\n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_labels = batch[0].to(device)\n",
        "        b_masks = batch[1].to(device)\n",
        "\n",
        "        # We don't need to track gradients for validation, so wrap in no_grad to save memory\n",
        "        with torch.no_grad():        \n",
        "\n",
        "            # Forward pass\n",
        "            outputs  = poem_stanza_model(b_input_ids, attention_mask=b_masks, labels=b_labels)\n",
        "\n",
        "            loss = outputs[0]  \n",
        "\n",
        "        # Accumulate the validation loss\n",
        "        batch_loss = loss.item()\n",
        "        total_eval_loss += batch_loss     \n",
        "        valid_losses.append(batch_loss)   \n",
        "\n",
        "    # Calculate the average validation loss for this epoch\n",
        "    avg_val_loss = total_eval_loss / len(poem_stanza_val_dataloader)\n",
        "\n",
        "    print(f'Average Validation Loss: {avg_val_loss}')\n",
        "\n",
        "print(f'Total Training Time: {format_time(time.time()-start_time)}')\n"
      ],
      "metadata": {
        "id": "T01EICCxJDbg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9a49479a-896c-4797-b2ef-957496e64580"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1 of 1\n",
            "Step: 0, Loss: 1.9971\n",
            "Step: 50, Loss: 0.8168\n",
            "Step: 100, Loss: 2.1951\n",
            "Step: 150, Loss: 1.9830\n",
            "Step: 200, Loss: 0.8496\n",
            "Step: 250, Loss: 1.0005\n",
            "Step: 300, Loss: 1.1123\n",
            "Step: 350, Loss: 2.7396\n",
            "Step: 400, Loss: 2.1484\n",
            "Step: 450, Loss: 1.1395\n",
            "Step: 500, Loss: 0.4966\n",
            "Step: 550, Loss: 1.8984\n",
            "Step: 600, Loss: 1.3414\n",
            "Step: 650, Loss: 1.2759\n",
            "Step: 700, Loss: 0.7070\n",
            "Step: 750, Loss: 2.0921\n",
            "Step: 800, Loss: 1.5292\n",
            "Step: 850, Loss: 0.6120\n",
            "Step: 900, Loss: 0.5833\n",
            "Step: 950, Loss: 1.7738\n",
            "Step: 1000, Loss: 3.5659\n",
            "Average Training Loss: 0.23474663062587794. Epoch Training Time: 0:11:47\n",
            "Average Validation Loss: 1.2531718199731297\n",
            "Total Training Time: 0:17:17\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Loss Curve plot"
      ],
      "metadata": {
        "id": "hLojUAYloH0c"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create an array for the x-axis\n",
        "x = np.arange(len(losses)) * (MAX_LEN * BATCH_SIZE)\n",
        "\n",
        "# Plot the line chart\n",
        "px.line(y=losses, x=np.arange(len(losses))*(MAX_LEN * BATCH_SIZE), labels={\"y\":\"Loss\", \"x\":\"Tokens\"}, title=\"Training curve for my tiny demo model!\")\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "ZuNRSGkvTSre",
        "outputId": "26c81090-e549-472b-df92-2f8faf5964d8"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html>\n",
              "<head><meta charset=\"utf-8\" /></head>\n",
              "<body>\n",
              "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
              "        <script src=\"https://cdn.plot.ly/plotly-2.18.2.min.js\"></script>                <div id=\"3cee2fb8-c8e2-43e0-8813-d85f17733b61\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"3cee2fb8-c8e2-43e0-8813-d85f17733b61\")) {                    Plotly.newPlot(                        \"3cee2fb8-c8e2-43e0-8813-d85f17733b61\",                        [{\"hovertemplate\":\"Tokens=%{x}<br>Loss=%{y}<extra></extra>\",\"legendgroup\":\"\",\"line\":{\"color\":\"#636efa\",\"dash\":\"solid\"},\"marker\":{\"symbol\":\"circle\"},\"mode\":\"lines\",\"name\":\"\",\"showlegend\":false,\"x\":[0,2048,4096,6144,8192,10240,12288,14336,16384,18432,20480,22528,24576,26624,28672,30720,32768,34816,36864,38912,40960,43008,45056,47104,49152,51200,53248,55296,57344,59392,61440,63488,65536,67584,69632,71680,73728,75776,77824,79872,81920,83968,86016,88064,90112,92160,94208,96256,98304,100352,102400,104448,106496,108544,110592,112640,114688,116736,118784,120832,122880,124928,126976,129024,131072,133120,135168,137216,139264,141312,143360,145408,147456,149504,151552,153600,155648,157696,159744,161792,163840,165888,167936,169984,172032,174080,176128,178176,180224,182272,184320,186368,188416,190464,192512,194560,196608,198656,200704,202752,204800,206848,208896,210944,212992,215040,217088,219136,221184,223232,225280,227328,229376,231424,233472,235520,237568,239616,241664,243712,245760,247808,249856,251904,253952,256000,258048,260096,262144,264192,266240,268288,270336,272384,274432,276480,278528,280576,282624,284672,286720,288768,290816,292864,294912,296960,299008,301056,303104,305152,307200,309248,311296,313344,315392,317440,319488,321536,323584,325632,327680,329728,331776,333824,335872,337920,339968,342016,344064,346112,348160,350208,352256,354304,356352,358400,360448,362496,364544,366592,368640,370688,372736,374784,376832,378880,380928,382976,385024,387072,389120,391168,393216,395264,397312,399360,401408,403456,405504,407552,409600,411648,413696,415744,417792,419840,421888,423936,425984,428032,430080,432128,434176,436224,438272,440320,442368,444416,446464,448512,450560,452608,454656,456704,458752,460800,462848,464896,466944,468992,471040,473088,475136,477184,479232,481280,483328,485376,487424,489472,491520,493568,495616,497664,499712,501760,503808,505856,507904,509952,512000,514048,516096,518144,520192,522240,524288,526336,528384,530432,532480,534528,536576,538624,540672,542720,544768,546816,548864,550912,552960,555008,557056,559104,561152,563200,565248,567296,569344,571392,573440,575488,577536,579584,581632,583680,585728,587776,589824,591872,593920,595968,598016,600064,602112,604160,606208,608256,610304,612352,614400,616448,618496,620544,622592,624640,626688,628736,630784,632832,634880,636928,638976,641024,643072,645120,647168,649216,651264,653312,655360,657408,659456,661504,663552,665600,667648,669696,671744,673792,675840,677888,679936,681984,684032,686080,688128,690176,692224,694272,696320,698368,700416,702464,704512,706560,708608,710656,712704,714752,716800,718848,720896,722944,724992,727040,729088,731136,733184,735232,737280,739328,741376,743424,745472,747520,749568,751616,753664,755712,757760,759808,761856,763904,765952,768000,770048,772096,774144,776192,778240,780288,782336,784384,786432,788480,790528,792576,794624,796672,798720,800768,802816,804864,806912,808960,811008,813056,815104,817152,819200,821248,823296,825344,827392,829440,831488,833536,835584,837632,839680,841728,843776,845824,847872,849920,851968,854016,856064,858112,860160,862208,864256,866304,868352,870400,872448,874496,876544,878592,880640,882688,884736,886784,888832,890880,892928,894976,897024,899072,901120,903168,905216,907264,909312,911360,913408,915456,917504,919552,921600,923648,925696,927744,929792,931840,933888,935936,937984,940032,942080,944128,946176,948224,950272,952320,954368,956416,958464,960512,962560,964608,966656,968704,970752,972800,974848,976896,978944,980992,983040,985088,987136,989184,991232,993280,995328,997376,999424,1001472,1003520,1005568,1007616,1009664,1011712,1013760,1015808,1017856,1019904,1021952,1024000,1026048,1028096,1030144,1032192,1034240,1036288,1038336,1040384,1042432,1044480,1046528,1048576,1050624,1052672,1054720,1056768,1058816,1060864,1062912,1064960,1067008,1069056,1071104,1073152,1075200,1077248,1079296,1081344,1083392,1085440,1087488,1089536,1091584,1093632,1095680,1097728,1099776,1101824,1103872,1105920,1107968,1110016,1112064,1114112,1116160,1118208,1120256,1122304,1124352,1126400,1128448,1130496,1132544,1134592,1136640,1138688,1140736,1142784,1144832,1146880,1148928,1150976,1153024,1155072,1157120,1159168,1161216,1163264,1165312,1167360,1169408,1171456,1173504,1175552,1177600,1179648,1181696,1183744,1185792,1187840,1189888,1191936,1193984,1196032,1198080,1200128,1202176,1204224,1206272,1208320,1210368,1212416,1214464,1216512,1218560,1220608,1222656,1224704,1226752,1228800,1230848,1232896,1234944,1236992,1239040,1241088,1243136,1245184,1247232,1249280,1251328,1253376,1255424,1257472,1259520,1261568,1263616,1265664,1267712,1269760,1271808,1273856,1275904,1277952,1280000,1282048,1284096,1286144,1288192,1290240,1292288,1294336,1296384,1298432,1300480,1302528,1304576,1306624,1308672,1310720,1312768,1314816,1316864,1318912,1320960,1323008,1325056,1327104,1329152,1331200,1333248,1335296,1337344,1339392,1341440,1343488,1345536,1347584,1349632,1351680,1353728,1355776,1357824,1359872,1361920,1363968,1366016,1368064,1370112,1372160,1374208,1376256,1378304,1380352,1382400,1384448,1386496,1388544,1390592,1392640,1394688,1396736,1398784,1400832,1402880,1404928,1406976,1409024,1411072,1413120,1415168,1417216,1419264,1421312,1423360,1425408,1427456,1429504,1431552,1433600,1435648,1437696,1439744,1441792,1443840,1445888,1447936,1449984,1452032,1454080,1456128,1458176,1460224,1462272,1464320,1466368,1468416,1470464,1472512,1474560,1476608,1478656,1480704,1482752,1484800,1486848,1488896,1490944,1492992,1495040,1497088,1499136,1501184,1503232,1505280,1507328,1509376,1511424,1513472,1515520,1517568,1519616,1521664,1523712,1525760,1527808,1529856,1531904,1533952,1536000,1538048,1540096,1542144,1544192,1546240,1548288,1550336,1552384,1554432,1556480,1558528,1560576,1562624,1564672,1566720,1568768,1570816,1572864,1574912,1576960,1579008,1581056,1583104,1585152,1587200,1589248,1591296,1593344,1595392,1597440,1599488,1601536,1603584,1605632,1607680,1609728,1611776,1613824,1615872,1617920,1619968,1622016,1624064,1626112,1628160,1630208,1632256,1634304,1636352,1638400,1640448,1642496,1644544,1646592,1648640,1650688,1652736,1654784,1656832,1658880,1660928,1662976,1665024,1667072,1669120,1671168,1673216,1675264,1677312,1679360,1681408,1683456,1685504,1687552,1689600,1691648,1693696,1695744,1697792,1699840,1701888,1703936,1705984,1708032,1710080,1712128,1714176,1716224,1718272,1720320,1722368,1724416,1726464,1728512,1730560,1732608,1734656,1736704,1738752,1740800,1742848,1744896,1746944,1748992,1751040,1753088,1755136,1757184,1759232,1761280,1763328,1765376,1767424,1769472,1771520,1773568,1775616,1777664,1779712,1781760,1783808,1785856,1787904,1789952,1792000,1794048,1796096,1798144,1800192,1802240,1804288,1806336,1808384,1810432,1812480,1814528,1816576,1818624,1820672,1822720,1824768,1826816,1828864,1830912,1832960,1835008,1837056,1839104,1841152,1843200,1845248,1847296,1849344,1851392,1853440,1855488,1857536,1859584,1861632,1863680,1865728,1867776,1869824,1871872,1873920,1875968,1878016,1880064,1882112,1884160,1886208,1888256,1890304,1892352,1894400,1896448,1898496,1900544,1902592,1904640,1906688,1908736,1910784,1912832,1914880,1916928,1918976,1921024,1923072,1925120,1927168,1929216,1931264,1933312,1935360,1937408,1939456,1941504,1943552,1945600,1947648,1949696,1951744,1953792,1955840,1957888,1959936,1961984,1964032,1966080,1968128,1970176,1972224,1974272,1976320,1978368,1980416,1982464,1984512,1986560,1988608,1990656,1992704,1994752,1996800,1998848,2000896,2002944,2004992,2007040,2009088,2011136,2013184,2015232,2017280,2019328,2021376,2023424,2025472,2027520,2029568,2031616,2033664,2035712,2037760,2039808,2041856,2043904,2045952,2048000,2050048],\"xaxis\":\"x\",\"y\":[1.997060775756836,0.30489879846572876,0.814293622970581,1.0755366086959839,1.6727956533432007,1.4161840677261353,2.5181329250335693,0.7909080386161804,1.936932921409607,1.1617156267166138,0.9649264216423035,0.5842123031616211,1.9726717472076416,1.737540602684021,0.8702037334442139,2.062380790710449,2.618494987487793,1.6590880155563354,1.1882944107055664,1.47587251663208,0.8553219437599182,0.7901582717895508,0.9776909947395325,1.6241707801818848,0.46784693002700806,1.5942260026931763,1.3934203386306763,0.604438304901123,2.071012258529663,0.8175094723701477,2.2221574783325195,0.560272753238678,0.696368932723999,2.4975905418395996,1.4287649393081665,1.1668317317962646,1.2204638719558716,1.181130290031433,1.0689821243286133,1.8903928995132446,0.5508816838264465,1.557246446609497,0.7729376554489136,1.8670111894607544,1.3333741426467896,0.9044713973999023,0.44645410776138306,1.1434791088104248,2.024200439453125,1.0814006328582764,0.8167973160743713,1.4933009147644043,0.6359585523605347,0.46854954957962036,1.7229979038238525,2.1184134483337402,0.9176836013793945,0.690834105014801,0.8422278761863708,1.161910057067871,0.7248976230621338,1.0742244720458984,1.9531655311584473,1.3015555143356323,1.3634772300720215,0.597946286201477,1.8192996978759766,0.743319571018219,0.7877832055091858,1.635366678237915,1.4521547555923462,1.3449825048446655,1.4913283586502075,0.8059201836585999,1.2950655221939087,1.1673418283462524,2.5647683143615723,1.3877969980239868,1.32840895652771,2.8117306232452393,0.48273375630378723,0.6533123254776001,2.0976016521453857,1.2638417482376099,1.2387765645980835,0.9186906814575195,2.012216091156006,1.2834030389785767,0.30808037519454956,2.3782103061676025,0.8458660244941711,0.9155548214912415,0.8045139908790588,0.9080974459648132,2.5953590869903564,0.6479021310806274,1.970546841621399,0.7273420095443726,1.954619288444519,2.158764362335205,2.1950924396514893,1.2435358762741089,1.5536532402038574,0.5977331399917603,0.4091842472553253,2.418247938156128,3.3551299571990967,2.706409215927124,0.5872892141342163,1.741676926612854,2.0569868087768555,1.7201834917068481,0.9207724928855896,1.69214928150177,0.5223420262336731,0.9159242510795593,0.9918116331100464,0.8423826694488525,0.6637036800384521,1.3143231868743896,1.0698102712631226,2.2176802158355713,0.5614786744117737,1.3246471881866455,1.4789563417434692,0.5341739654541016,1.715252161026001,0.9872356653213501,0.7993932962417603,1.6277724504470825,1.610695242881775,3.500375747680664,0.614687442779541,0.9310238361358643,1.4072515964508057,0.4265945255756378,1.228525996208191,1.37113618850708,1.1093807220458984,1.637498378753662,2.5130438804626465,1.3501057624816895,1.9810940027236938,1.31003999710083,1.0974805355072021,1.360657811164856,1.4718266725540161,3.2802889347076416,0.610390841960907,0.9604440331459045,1.983028769493103,0.7987799048423767,1.2258930206298828,1.148828148841858,2.9217448234558105,1.8215217590332031,0.5290243029594421,0.8586680889129639,1.4112871885299683,1.0165380239486694,0.839450478553772,0.2229963093996048,3.261361598968506,0.7918078899383545,1.5769532918930054,0.7720330357551575,0.9783960580825806,1.5811949968338013,1.5204565525054932,0.36521705985069275,0.8937124609947205,0.5929877758026123,0.8009927868843079,0.7290682792663574,1.638208270072937,0.6646957993507385,1.036325216293335,0.4450869560241699,1.2960832118988037,1.6570091247558594,1.855984091758728,0.8079912662506104,1.2355185747146606,0.7987216711044312,2.815115451812744,3.029717445373535,2.4414925575256348,1.8243296146392822,2.5073680877685547,2.729835271835327,1.2828515768051147,0.9725059270858765,2.0892508029937744,2.6257317066192627,0.7570323348045349,0.6569043397903442,0.34387266635894775,0.8088745474815369,0.782250165939331,3.75589919090271,0.8495943546295166,3.0758402347564697,1.3346089124679565,0.6991298794746399,2.34499454498291,0.6169869303703308,1.765181541442871,0.6465134024620056,1.7835540771484375,1.6035467386245728,1.216443419456482,0.8642158508300781,2.9982404708862305,1.0532090663909912,1.0273653268814087,0.5645462274551392,1.4033093452453613,1.3083703517913818,0.3666808307170868,1.6579676866531372,1.056946039199829,0.2744588255882263,2.017221212387085,1.0338283777236938,2.061352491378784,0.7474743723869324,1.0984153747558594,1.0260028839111328,0.5990296602249146,0.9708870649337769,0.8685092329978943,0.9574472308158875,2.4495391845703125,0.7003239989280701,1.6875040531158447,0.7518501281738281,1.4947646856307983,1.4892669916152954,0.9425471425056458,1.3938565254211426,1.0039331912994385,2.585073709487915,1.2433278560638428,0.39407774806022644,0.8498850464820862,2.3493993282318115,2.2185051441192627,0.814907968044281,0.7560352087020874,2.627349615097046,1.0005037784576416,2.0222713947296143,2.710874557495117,1.0403475761413574,3.2727463245391846,1.156200647354126,0.7365062236785889,0.5870441794395447,1.437570571899414,0.628309428691864,0.5828944444656372,1.0745781660079956,1.1584326028823853,1.490416407585144,1.9156157970428467,1.9648200273513794,0.9871125817298889,1.2818982601165771,0.8928743004798889,1.794287085533142,0.9923539757728577,0.6347742676734924,0.3794265687465668,1.528179407119751,0.6681536436080933,1.9764103889465332,2.922189950942993,1.2804282903671265,1.2107023000717163,1.257483720779419,0.6500887274742126,2.068305015563965,0.6040964126586914,0.5780866742134094,1.2614998817443848,0.45364686846733093,1.2340009212493896,0.5949193239212036,4.66733980178833,0.8302758932113647,1.0235031843185425,0.45812368392944336,0.6958041787147522,0.6264567971229553,0.2287871539592743,1.164117455482483,0.40836507081985474,0.7964909672737122,0.7006539106369019,2.6279611587524414,1.1123279333114624,2.2720658779144287,0.8518418073654175,1.1671708822250366,0.3121504485607147,1.2576309442520142,0.5577669143676758,1.0748772621154785,1.7509114742279053,0.3059578239917755,1.8317975997924805,1.7117871046066284,1.044222354888916,0.9261459112167358,2.0617148876190186,2.1315886974334717,1.3127175569534302,0.7488798499107361,1.0906965732574463,1.512481689453125,0.8986984491348267,1.0036563873291016,0.1699025183916092,0.7722153663635254,1.864924430847168,1.0090452432632446,0.4097418487071991,0.7122732996940613,2.1879875659942627,2.0479683876037598,1.0354866981506348,2.5837602615356445,2.448244571685791,2.1495585441589355,0.90152508020401,0.74637371301651,1.654549479484558,1.0797464847564697,0.7631102204322815,0.4053167998790741,0.9977030158042908,3.5003650188446045,0.666761577129364,2.3462536334991455,0.8910974264144897,0.8010228872299194,1.8660310506820679,0.7703144550323486,1.0651037693023682,0.5592830181121826,2.7395546436309814,2.0736238956451416,0.9328778982162476,0.5980265736579895,0.3136129081249237,1.9998351335525513,0.2949208617210388,0.7485240697860718,1.3597569465637207,0.9471712112426758,0.5094336867332458,1.7061214447021484,1.3792614936828613,0.705000638961792,3.2274978160858154,0.6493197679519653,0.28512391448020935,0.6241005063056946,0.9977510571479797,0.7156495451927185,1.2187771797180176,0.48497453331947327,2.6061453819274902,0.5839860439300537,0.6700525283813477,1.697165846824646,2.21856427192688,2.7327585220336914,1.9235326051712036,1.0187569856643677,2.879737377166748,0.856952965259552,1.1376597881317139,1.7919509410858154,1.4213510751724243,2.692152738571167,2.1989521980285645,1.0167794227600098,0.48711493611335754,0.5159053206443787,0.8543157577514648,1.8638715744018555,0.6736876964569092,1.9362494945526123,0.9113432765007019,0.6215596199035645,1.4373385906219482,1.6387304067611694,2.0061895847320557,0.9655689597129822,2.1483914852142334,0.7645822763442993,0.8628864288330078,1.0679491758346558,0.8204216957092285,0.7603329420089722,0.5801158547401428,0.6919612288475037,1.3274027109146118,1.769465684890747,2.2741951942443848,1.0174363851547241,1.3670755624771118,2.9681556224823,0.46789613366127014,0.8161191940307617,1.5750906467437744,0.7190377116203308,0.7633876204490662,0.7024725675582886,0.7634323239326477,2.310635566711426,0.32323187589645386,2.329396963119507,1.9811077117919922,1.9885189533233643,1.541635513305664,0.943959653377533,0.46247899532318115,2.4440219402313232,2.049596071243286,1.1101287603378296,1.381645679473877,1.2263882160186768,1.400820016860962,0.7851165533065796,0.5175577998161316,0.8499264121055603,2.371324062347412,0.5129075646400452,0.7069039344787598,2.3423831462860107,0.9415637850761414,1.0663551092147827,0.7246556282043457,1.7963114976882935,0.7955463528633118,1.2742332220077515,0.9549276828765869,1.608335256576538,1.1395200490951538,1.727756142616272,0.9381040930747986,1.7209882736206055,0.9073282480239868,0.8419561386108398,0.6593315601348877,0.4437716603279114,1.5351755619049072,2.2832679748535156,1.5276463031768799,1.1802464723587036,1.304903268814087,1.5224429368972778,1.2280758619308472,1.853248953819275,1.0025272369384766,0.6632242798805237,2.0299718379974365,2.647946357727051,1.3768949508666992,1.511731505393982,1.3738114833831787,0.6981530785560608,0.14622265100479126,0.2821771502494812,0.7081199288368225,0.8805497288703918,1.2065173387527466,0.8097022771835327,2.3274760246276855,0.8417018055915833,1.5426181554794312,0.41168367862701416,1.1043963432312012,0.8348071575164795,0.9515860080718994,0.7677971124649048,1.5368216037750244,0.8370701670646667,0.9442368745803833,0.7837615609169006,2.2420051097869873,1.6921279430389404,1.596649408340454,0.6074963212013245,2.6128861904144287,1.1058000326156616,1.5721254348754883,0.8996959328651428,0.4965628981590271,0.514869749546051,0.6708092093467712,3.0806617736816406,0.608477771282196,0.8108340501785278,0.44568178057670593,0.5366017818450928,0.5883892178535461,1.1366909742355347,1.1055333614349365,1.0870145559310913,0.9609546661376953,1.097531795501709,0.4880022704601288,1.4515118598937988,0.5352641344070435,1.3968676328659058,0.7534735798835754,1.1174156665802002,2.1042323112487793,0.5311662554740906,0.7434312105178833,1.3927570581436157,0.9181739091873169,0.8632845878601074,0.8199548721313477,2.44502329826355,1.448525071144104,0.9624210596084595,1.3471132516860962,1.2140337228775024,0.38048067688941956,1.5364915132522583,2.511970281600952,1.8352282047271729,1.2341805696487427,2.4557876586914062,0.644602358341217,0.7093859910964966,0.7689847946166992,0.7511143088340759,0.3621440827846527,1.4780449867248535,0.9671766757965088,1.5094059705734253,1.00095796585083,0.387485146522522,0.9151818156242371,1.908896565437317,1.8983739614486694,0.9483403563499451,1.7719612121582031,1.6497139930725098,0.3370186686515808,1.4742766618728638,1.3233720064163208,1.6212692260742188,1.8872119188308716,1.8101798295974731,0.5788443684577942,2.0627732276916504,0.9720818996429443,1.0037816762924194,0.9124632477760315,0.7227904200553894,1.0466020107269287,0.289059042930603,2.478827476501465,1.4801950454711914,0.7224797010421753,0.9185209274291992,1.4427622556686401,1.2483266592025757,0.7726854085922241,2.642254590988159,1.263996958732605,1.0637619495391846,1.6969013214111328,0.8180574774742126,1.6877275705337524,1.2124918699264526,0.7928659915924072,2.5526018142700195,0.9038196206092834,1.6692023277282715,1.0690803527832031,1.9520968198776245,0.6731137037277222,1.9505695104599,1.3436118364334106,1.1586658954620361,2.699962854385376,1.6037311553955078,2.387619733810425,1.1597013473510742,0.6676142811775208,0.9323081970214844,1.1007126569747925,1.4303553104400635,1.3414311408996582,3.2560625076293945,1.9235119819641113,1.2730658054351807,1.4045212268829346,0.7206857800483704,1.1725188493728638,0.7516778111457825,1.4172205924987793,0.9669919610023499,2.0241622924804688,0.6182990074157715,1.7622075080871582,0.8705030083656311,1.1471401453018188,1.0240784883499146,1.069831371307373,2.6718528270721436,1.712843894958496,0.9898900389671326,0.8458768129348755,0.6896800398826599,0.46022695302963257,0.7048009634017944,0.7930951118469238,0.3317154347896576,0.4787439703941345,1.3520355224609375,1.0641676187515259,1.0687097311019897,0.5084617137908936,2.1966121196746826,4.725432872772217,2.3292505741119385,0.672471821308136,0.8191860914230347,1.8687951564788818,1.8064740896224976,0.9445061087608337,1.5188051462173462,0.554171621799469,2.3303024768829346,2.2747623920440674,1.2457908391952515,0.8793988823890686,1.0630208253860474,2.1687872409820557,1.0289859771728516,2.3417277336120605,0.7738555669784546,1.2759485244750977,1.01371431350708,2.6011905670166016,0.6448595523834229,3.12046217918396,0.9690418243408203,0.7105635404586792,0.9717460870742798,0.54448002576828,0.46818360686302185,1.1455694437026978,1.9290335178375244,1.5332673788070679,0.9728388786315918,0.448694109916687,2.1025524139404297,0.6895871162414551,2.0608513355255127,0.5812345743179321,1.2420239448547363,1.8582710027694702,1.5201408863067627,1.7315375804901123,1.6607693433761597,1.8305498361587524,0.842856228351593,1.9662410020828247,1.2516274452209473,0.5656949877738953,1.4791386127471924,1.332603931427002,1.4912807941436768,0.674728512763977,1.0120563507080078,0.7531189918518066,0.9948670268058777,0.8528217077255249,1.0846799612045288,0.6969079375267029,1.3438245058059692,1.9840710163116455,0.7433018684387207,2.0007429122924805,2.22092342376709,1.904895305633545,1.637893795967102,1.6812589168548584,1.55640709400177,3.1074085235595703,0.9565231204032898,0.7070452570915222,0.7782841920852661,0.7687800526618958,4.488184928894043,0.5422074794769287,1.071115493774414,2.4234731197357178,1.185918927192688,2.0126359462738037,0.26646292209625244,1.2702347040176392,1.0697565078735352,1.5335172414779663,1.0781227350234985,3.2530901432037354,1.1494406461715698,2.490463972091675,1.1246434450149536,2.4900097846984863,1.1771626472473145,2.430677890777588,2.912057876586914,1.4099940061569214,1.2452120780944824,0.6589565873146057,1.5831196308135986,1.674686074256897,0.7559651136398315,1.3869870901107788,0.8477351665496826,1.7204116582870483,1.2557404041290283,1.7163244485855103,1.1870896816253662,0.7823072075843811,0.5778138637542725,0.8078227639198303,1.199321985244751,0.8507382273674011,0.6996636986732483,2.6000823974609375,0.5347152352333069,2.7499616146087646,0.5003594160079956,0.749530017375946,1.2038618326187134,2.549302101135254,1.3438000679016113,1.579298734664917,2.0926859378814697,2.092114210128784,0.6556931138038635,2.830970287322998,1.1024442911148071,0.9119367003440857,0.9368142485618591,0.6695147752761841,1.4550455808639526,0.754612147808075,0.47219470143318176,1.9536901712417603,0.9937589168548584,0.6863038539886475,1.225596308708191,0.9109605550765991,1.040280818939209,4.250742435455322,1.9940848350524902,0.7852674126625061,0.7956708669662476,1.4710525274276733,0.5049103498458862,0.7229517102241516,1.6219041347503662,1.054525375366211,0.3410918414592743,2.6100757122039795,0.8831173181533813,1.131639003753662,0.9367502927780151,0.8551190495491028,1.5196481943130493,1.1542303562164307,0.5354156494140625,0.6012673377990723,0.7793903350830078,1.5290850400924683,1.9147075414657593,1.2405866384506226,0.6804568767547607,0.9624978303909302,1.2222098112106323,0.4610448479652405,0.3424419164657593,0.6523134112358093,0.855358898639679,0.8599307537078857,3.7422776222229004,0.7525885105133057,1.8864039182662964,1.5292202234268188,0.6788471937179565,1.4059991836547852,1.1785649061203003,1.2538255453109741,1.9420894384384155,1.2250460386276245,0.8355389833450317,1.1389864683151245,0.7747636437416077,2.188190460205078,1.0982494354248047,1.4787533283233643,2.5346596240997314,0.8540828824043274,0.7963488698005676,1.3587281703948975,1.5620983839035034,2.0005340576171875,2.549919366836548,2.1896159648895264,0.6676105260848999,0.9025399684906006,1.767545461654663,2.262526273727417,1.540973424911499,1.3680421113967896,1.0231596231460571,1.112177848815918,0.7149287462234497,0.42548519372940063,0.922217607498169,2.419633626937866,0.9462481737136841,0.9349005818367004,0.6124197840690613,1.6818194389343262,0.6936467885971069,2.0701255798339844,1.1210887432098389,1.1153383255004883,1.0107409954071045,0.7076888680458069,0.4748564064502716,0.6172307729721069,1.9694736003875732,1.0908198356628418,1.7484418153762817,0.9282336831092834,0.4869159162044525,0.6120293140411377,0.8580365777015686,1.2329152822494507,1.823320984840393,0.7366858124732971,0.6173496842384338,1.7504395246505737,0.729824960231781,2.249812602996826,1.108884572982788,0.7163743376731873,1.3216079473495483,2.4006764888763428,1.216407299041748,1.0551517009735107,1.0361454486846924,3.3387484550476074,1.2128956317901611,1.3569082021713257,0.9036285877227783,0.563946008682251,0.9539235830307007,2.2340431213378906,2.333940267562866,1.1224123239517212,0.8769200444221497,2.968982458114624,1.1429731845855713,1.4709421396255493,1.2162419557571411,1.3190912008285522,0.6445204615592957,0.4639657735824585,1.2967503070831299,1.039918065071106,0.6835126876831055,0.9343479871749878,2.196462631225586,0.868971049785614,0.6608940958976746,2.404296875,1.8290555477142334,1.344272494316101,2.0372791290283203,0.7735486626625061,1.1035603284835815,2.4412636756896973,0.5760026574134827,1.7028467655181885,0.7480068206787109,0.5832735896110535,0.5807682871818542,2.1703803539276123,1.0312342643737793,1.4510234594345093,1.3709063529968262,0.9302095174789429,0.9726131558418274,0.2934117913246155,1.1834604740142822,0.5975750088691711,0.5264666080474854,0.7372849583625793,0.5365695953369141,1.9529764652252197,2.461261034011841,0.2459467649459839,1.161828637123108,0.7825146317481995,0.8546186089515686,1.1896944046020508,1.1557635068893433,1.6033927202224731,1.0043838024139404,1.2960214614868164,1.558221697807312,1.68796706199646,1.2055989503860474,2.079719066619873,0.9051024317741394,0.27125146985054016,0.9741644859313965,1.3987120389938354,1.5879870653152466,0.8387030959129333,1.7491724491119385,0.5849536061286926,2.003624677658081,0.5913205146789551,1.8746494054794312,1.008560299873352,0.7730854749679565,1.2596218585968018,0.8546820282936096,1.833661437034607,1.5245311260223389,1.3611806631088257,1.4665530920028687,1.1902360916137695,0.8605767488479614,1.7737635374069214,0.6554341316223145,0.5271220803260803,1.1136926412582397,1.140495777130127,2.550586462020874,0.7572441697120667,3.2036380767822266,1.5467755794525146,0.3477250933647156,0.9212915301322937,1.4227558374404907,0.786197304725647,1.5238624811172485,1.437134861946106,1.5684611797332764,2.2324352264404297,0.5863487124443054,0.4052814841270447,0.9028687477111816,1.779496192932129,1.8821814060211182,0.9349769353866577,1.4279119968414307,0.5386962890625,3.9684793949127197,1.1480413675308228,0.6369624733924866,2.963693618774414,1.4162888526916504,0.4473138153553009,1.7724547386169434,1.640331745147705,1.2343708276748657,1.087699294090271,2.1004087924957275,0.6105945110321045,1.272087812423706,0.9413323998451233,0.987244725227356,0.9154917001724243,0.9179965853691101,1.3362246751785278,1.871665358543396,0.9541001319885254,2.086542844772339,0.5156075954437256,0.9629085659980774,0.4740106761455536,0.8861987590789795,3.5658750534057617,0.8356255888938904],\"yaxis\":\"y\",\"type\":\"scattergl\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Tokens\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Loss\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Training curve for my tiny demo model!\"}},                        {\"responsive\": true}                    ).then(function(){\n",
              "                            \n",
              "var gd = document.getElementById('3cee2fb8-c8e2-43e0-8813-d85f17733b61');\n",
              "var x = new MutationObserver(function (mutations, observer) {{\n",
              "        var display = window.getComputedStyle(gd).display;\n",
              "        if (!display || display === 'none') {{\n",
              "            console.log([gd, 'removed!']);\n",
              "            Plotly.purge(gd);\n",
              "            observer.disconnect();\n",
              "        }}\n",
              "}});\n",
              "\n",
              "// Listen for the removal of the full notebook cells\n",
              "var notebookContainer = gd.closest('#notebook-container');\n",
              "if (notebookContainer) {{\n",
              "    x.observe(notebookContainer, {childList: true});\n",
              "}}\n",
              "\n",
              "// Listen for the clearing of the current output cell\n",
              "var outputEl = gd.closest('.output');\n",
              "if (outputEl) {{\n",
              "    x.observe(outputEl, {childList: true});\n",
              "}}\n",
              "\n",
              "                        })                };                            </script>        </div>\n",
              "</body>\n",
              "</html>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Save the trained model"
      ],
      "metadata": {
        "id": "SyUqUW1tzl-P"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "torch.save(poem_stanza_model.state_dict(),  'poem_stanza_model.pth')"
      ],
      "metadata": {
        "id": "jXANq62qZL5G"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Use the newly trained model to generate poems"
      ],
      "metadata": {
        "id": "S33y94JDztlJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# text = \"I love my dog\"\n",
        "# input_ids = tokenizer.encode(text, return_tensors='tf')\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "poem_stanza_model = poem_stanza_model.to(device)\n",
        "\n",
        "# create text generation seed promp\n",
        "prompt = \"<BOS> I miss home\"\n",
        "generated = torch.tensor(tokenizer.encode(prompt)).unsqueeze(0)\n",
        "generated = generated.to(device)\n",
        "\n",
        "sample_outputs = poem_stanza_model.generate(\n",
        "                                generated, \n",
        "                                do_sample=True,   \n",
        "                                top_k=50, \n",
        "                                max_length=MAX_LEN,\n",
        "                                top_p=0.95, \n",
        "                                num_return_sequences=3\n",
        "                                )\n",
        "\n",
        "# print(tokenizer.decode(sample_outputs[0], skip_special_tokens=True))\n",
        "\n",
        "for i, sample_output in enumerate(sample_outputs):\n",
        "    print(\"{}: {}\\n\\n\".format(i, tokenizer.decode(sample_output, skip_special_tokens=True)))"
      ],
      "metadata": {
        "id": "Ydgs8W6sKJYQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d062e5cf-df71-4781-e56d-bf73d46a0855"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0: I miss home,\r\r\n",
            "but I miss you too,\r\r\n",
            "like the last week\r\r\n",
            "until you start a new life\r\r\n",
            "from my blood—\r\r\n",
            "I miss you too,\r\r\n",
            "like the last day you left\r\r\n",
            "until I go home to my bed.\r\r\n",
            "I don’t love you,\r\r\n",
            "but I love you even now.\r\r\n",
            "I don’t love you with a smile\r\r\n",
            "as much as I do that day.\r\r\n",
            "I love you with the smell of your milk\r\r\n",
            "and its scent\r\r\n",
            "on your lips, in all your smiles,\r\r\n",
            "as if I were your lover.\r\r\n",
            "I love you with the sounds you make\r\r\n",
            "on your cheeks, laughing in them as though there were no words.\r\r\n",
            "I love you with the sound of your music,\r\r\n",
            "talking to you like an animal, saying,\r\r\n",
            "and it has just got to stop. — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — — —\n",
            "\n",
            "\n",
            "1: I miss home\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "The day was long ago.\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "My parents and I had one child\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "and the way I saw it,\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "I knew\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "and everything\r\r\n",
            "\r\r\n",
            "\r\r\n",
            " I wanted\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "I thought back to it\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\r\r\n",
            "\n",
            "\n",
            "2: I miss home when I walk down to my garden.\r\r\n",
            "I remember a young man\r\r\n",
            "who lived in a small village\r\r\n",
            "where the sun never fell.\r\r\n",
            "He kept to himself\r\r\n",
            "a little girl with big dreams.\r\r\n",
            "Sometimes, though, he asked\r\r\n",
            "how I should dress if he could imagine\r\r\n",
            "how he'd come to live in a house\r\r\n",
            "where the wind would beat his nose\r\r\n",
            "and take the hair\r\r\n",
            "from his face,\r\r\n",
            "the sun would bite into the dirt\r\r\n",
            "and pour through\r\r\n",
            "the streets.\r\r\n",
            "The old man kept to himself\r\r\n",
            "the little girl\r\r\n",
            "who looked as if she were\r\r\n",
            "wondering\r\r\n",
            "how I should dress if he could imagine\r\r\n",
            "how I'd come to live in a house\r\r\n",
            "where the wind would beat his nose.\r\r\n",
            "The old man kept\r\r\n",
            "his head down\r\r\n",
            "and kept to himself\r\r\n",
            "the little girl\r\r\n",
            "who looked as if she were\r\r\n",
            "wondering\r\r\n",
            "how I should dress if he could imagine\r\r\n",
            "how I'd come to live in a house where\r\r\n",
            "the wind would beat his nose\r\r\n",
            "and pour through\r\r\n",
            "the streets. – MELIA—’s father. Gently\r\r\n",
            "wants to sing and dance and sing.\r\r\n",
            "He makes fun of his daughters.\r\r\n",
            "There was a time when he wanted to marry the same girl he loved.\r\r\n",
            "He wanted to be the one to choose a new girlfriend.\r\r\n",
            "He wants to become a woman, as he said,\r\r\n",
            "and to give up her, as he told the child,\r\r\n",
            "to be married.\r\r\n",
            "He wants to marry the same girl he loved.\r\r\n",
            "There was a time when he wanted to be the one to choose a new girlfriend.\r\r\n",
            "He wants to become a woman, as he said,\r\r\n",
            "and to give up her, as he told the child,\r\r\n",
            "to be married.\r\r\n",
            "And he wants to be married.\n",
            "\n",
            "\n"
          ]
        }
      ]
    }
  ]
}