2_inside_transformers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "628978270c664edeb5568981ce8f7656": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_445fe77c6f5d4dc392d1a2aebf7b7cb7",
              "IPY_MODEL_09515ecd15974497a6dcbb8609abc6b3",
              "IPY_MODEL_714aa79898964658b48187bbf9a5d778"
            ],
            "layout": "IPY_MODEL_b5bcdfdd111c405686570ccb37778f7c"
          }
        },
        "445fe77c6f5d4dc392d1a2aebf7b7cb7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_75e5b3578e65430f8ae006d26de2e3fe",
            "placeholder": "​",
            "style": "IPY_MODEL_af1bd955ee1e44b2a745b488295dbe96",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "09515ecd15974497a6dcbb8609abc6b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_664642914e15416493471cddde224eae",
            "max": 48,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a0071b707009486097fd4de5dcea9a4e",
            "value": 48
          }
        },
        "714aa79898964658b48187bbf9a5d778": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b2331dcc520644fe890ded18881107d6",
            "placeholder": "​",
            "style": "IPY_MODEL_31d1dffd7e1a454b8233d86b7bae2aae",
            "value": " 48.0/48.0 [00:00&lt;00:00, 266B/s]"
          }
        },
        "b5bcdfdd111c405686570ccb37778f7c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "75e5b3578e65430f8ae006d26de2e3fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "af1bd955ee1e44b2a745b488295dbe96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "664642914e15416493471cddde224eae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a0071b707009486097fd4de5dcea9a4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b2331dcc520644fe890ded18881107d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "31d1dffd7e1a454b8233d86b7bae2aae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a571bec8ed2f4f1da463abd8d186b9f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_93190f571e244af29319f52919e1227c",
              "IPY_MODEL_c1c1f31bf2a548feb1deba0b16f9a2e3",
              "IPY_MODEL_ff44498d5be44d2691c6d95bf96047ad"
            ],
            "layout": "IPY_MODEL_2d5662cc171940e28737b4f29045335e"
          }
        },
        "93190f571e244af29319f52919e1227c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_06546d4e21f541d7b7fa63f5cc2d281b",
            "placeholder": "​",
            "style": "IPY_MODEL_521da74e072c48e788011a031a2382cd",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "c1c1f31bf2a548feb1deba0b16f9a2e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c7611f88a43f41158bb20a7ba5450b3a",
            "max": 629,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a58839694f2547e3b9c2ead5ececeb87",
            "value": 629
          }
        },
        "ff44498d5be44d2691c6d95bf96047ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c385dbf2c41c4a44ab3e59bfdd8864db",
            "placeholder": "​",
            "style": "IPY_MODEL_0a20cca2c78742b4944f1beb25befc77",
            "value": " 629/629 [00:00&lt;00:00, 6.78kB/s]"
          }
        },
        "2d5662cc171940e28737b4f29045335e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "06546d4e21f541d7b7fa63f5cc2d281b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "521da74e072c48e788011a031a2382cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c7611f88a43f41158bb20a7ba5450b3a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a58839694f2547e3b9c2ead5ececeb87": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c385dbf2c41c4a44ab3e59bfdd8864db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0a20cca2c78742b4944f1beb25befc77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fed769e6b2e84dc7bb23b929c2da9147": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a83d033e2f5a4d628969f2fc8886a3cb",
              "IPY_MODEL_fda796f1745e43c5a7a93637db485ef6",
              "IPY_MODEL_40df0b29612744ca8c9d8423d7919ed2"
            ],
            "layout": "IPY_MODEL_c992253d497a48d5830984036a366770"
          }
        },
        "a83d033e2f5a4d628969f2fc8886a3cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d963037e3ed14001bce8f09cd05bde0d",
            "placeholder": "​",
            "style": "IPY_MODEL_18ca582b9c404f46acb64c316561f2d5",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "fda796f1745e43c5a7a93637db485ef6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ca2eec18eb75491c914de37f8576818e",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6ad61d3a95334de296d8dc5f36e60129",
            "value": 231508
          }
        },
        "40df0b29612744ca8c9d8423d7919ed2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_12b3675b0333425f9e8f5c09a5e7ec48",
            "placeholder": "​",
            "style": "IPY_MODEL_33c912f28e664881a07eb35c183adfc9",
            "value": " 232k/232k [00:00&lt;00:00, 4.13MB/s]"
          }
        },
        "c992253d497a48d5830984036a366770": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d963037e3ed14001bce8f09cd05bde0d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "18ca582b9c404f46acb64c316561f2d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ca2eec18eb75491c914de37f8576818e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6ad61d3a95334de296d8dc5f36e60129": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "12b3675b0333425f9e8f5c09a5e7ec48": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "33c912f28e664881a07eb35c183adfc9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bbed07462383421ebb27686ccdd1af62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_84d35a6c1f414b0e8b176fb481ec23cf",
              "IPY_MODEL_d36ed3c17ad042c5a26959f625281f3b",
              "IPY_MODEL_8ae60ea29e5443a9b613d3e96f9bf1e4"
            ],
            "layout": "IPY_MODEL_64c8e63a1f2f4a3f85998f2b6d992f5c"
          }
        },
        "84d35a6c1f414b0e8b176fb481ec23cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_60f53ba294844ca28e8b7d2a2d3b82e9",
            "placeholder": "​",
            "style": "IPY_MODEL_5a624c3f1669455090d46763a80d2169",
            "value": "Downloading model.safetensors: 100%"
          }
        },
        "d36ed3c17ad042c5a26959f625281f3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8fd9292adecb4a829acd8d9f79bea3ae",
            "max": 267832558,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f4b5f177ae6846f2943058f415182743",
            "value": 267832558
          }
        },
        "8ae60ea29e5443a9b613d3e96f9bf1e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ead2510311834997a4fb2d53ae12651c",
            "placeholder": "​",
            "style": "IPY_MODEL_e9b88722d12e447caeac4de2a93d9494",
            "value": " 268M/268M [00:02&lt;00:00, 141MB/s]"
          }
        },
        "64c8e63a1f2f4a3f85998f2b6d992f5c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "60f53ba294844ca28e8b7d2a2d3b82e9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5a624c3f1669455090d46763a80d2169": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8fd9292adecb4a829acd8d9f79bea3ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f4b5f177ae6846f2943058f415182743": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ead2510311834997a4fb2d53ae12651c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e9b88722d12e447caeac4de2a93d9494": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "539c9288cdc74a32a709a3e1b98e71d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5ca3841351a642019b08f0c0f0ae7e08",
              "IPY_MODEL_b721bc8b242048998731ebe3daaf6259",
              "IPY_MODEL_81e7bada0f49408cbd75289e466166e1"
            ],
            "layout": "IPY_MODEL_8d7e6fb1ab044247ad9c7e21aebbba43"
          }
        },
        "5ca3841351a642019b08f0c0f0ae7e08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_35496e7f769a4b7685a4108bc9178a12",
            "placeholder": "​",
            "style": "IPY_MODEL_97e6f6c19113422496f724151e557735",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "b721bc8b242048998731ebe3daaf6259": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4fbe013bac44442798c8bc722fb0d7e3",
            "max": 665,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_236a29e8005544b9ac4ca2862e8713d3",
            "value": 665
          }
        },
        "81e7bada0f49408cbd75289e466166e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0b0dd87d3013401f9e045fefc1629ba6",
            "placeholder": "​",
            "style": "IPY_MODEL_79821bf48a6d4af6967ca4ea7cdbc365",
            "value": " 665/665 [00:00&lt;00:00, 9.54kB/s]"
          }
        },
        "8d7e6fb1ab044247ad9c7e21aebbba43": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "35496e7f769a4b7685a4108bc9178a12": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "97e6f6c19113422496f724151e557735": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4fbe013bac44442798c8bc722fb0d7e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "236a29e8005544b9ac4ca2862e8713d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0b0dd87d3013401f9e045fefc1629ba6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "79821bf48a6d4af6967ca4ea7cdbc365": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c33c874b10a1490a89a683faa6245a89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0a623502032a4da1bd5194828523f159",
              "IPY_MODEL_433eb54725eb45bf82785342aa2d2286",
              "IPY_MODEL_85410095beb442b5bf8feefbc6250aa0"
            ],
            "layout": "IPY_MODEL_f340f97f33864547a070132e40862fcd"
          }
        },
        "0a623502032a4da1bd5194828523f159": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f23cd1d36b3e49f5ac6d19eb0857d106",
            "placeholder": "​",
            "style": "IPY_MODEL_3f392df4d4304cf5913ebc58f3cd089e",
            "value": "Downloading (…)olve/main/vocab.json: 100%"
          }
        },
        "433eb54725eb45bf82785342aa2d2286": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cb573e349e2d412195fc85b26d2407dd",
            "max": 1042301,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_31fba667ebcc4cb98ee4ac3abc8338d4",
            "value": 1042301
          }
        },
        "85410095beb442b5bf8feefbc6250aa0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_48c653fb92be4334a27bad267ef1c6ef",
            "placeholder": "​",
            "style": "IPY_MODEL_51d4f6a5ec564ef79c6ac9fd2d5ecb4e",
            "value": " 1.04M/1.04M [00:00&lt;00:00, 5.29MB/s]"
          }
        },
        "f340f97f33864547a070132e40862fcd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f23cd1d36b3e49f5ac6d19eb0857d106": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3f392df4d4304cf5913ebc58f3cd089e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cb573e349e2d412195fc85b26d2407dd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "31fba667ebcc4cb98ee4ac3abc8338d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "48c653fb92be4334a27bad267ef1c6ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "51d4f6a5ec564ef79c6ac9fd2d5ecb4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5adee91cae594fd59be190663a701bbb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5788ef8ed7f54f95afb6ad0c797d59ba",
              "IPY_MODEL_bc935840d5f040709002c31a97ced449",
              "IPY_MODEL_275d9f1b12de4848bf7a1c06211f9044"
            ],
            "layout": "IPY_MODEL_f5f2ef2a3e4740ce99aa5300035c9159"
          }
        },
        "5788ef8ed7f54f95afb6ad0c797d59ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c9e92140b1d7485cb37be646087debed",
            "placeholder": "​",
            "style": "IPY_MODEL_16ec22ff5c78411d8d39fb56735169fb",
            "value": "Downloading (…)olve/main/merges.txt: 100%"
          }
        },
        "bc935840d5f040709002c31a97ced449": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1fd7fbb548734aa9ad39700e8bded736",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c56b39b0634449c0a6f2fce550f57236",
            "value": 456318
          }
        },
        "275d9f1b12de4848bf7a1c06211f9044": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5637f751391d4e24834622af93b9e34b",
            "placeholder": "​",
            "style": "IPY_MODEL_04f9d53cc9054e0d9241344d50fea144",
            "value": " 456k/456k [00:00&lt;00:00, 9.42MB/s]"
          }
        },
        "f5f2ef2a3e4740ce99aa5300035c9159": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c9e92140b1d7485cb37be646087debed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "16ec22ff5c78411d8d39fb56735169fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1fd7fbb548734aa9ad39700e8bded736": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c56b39b0634449c0a6f2fce550f57236": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5637f751391d4e24834622af93b9e34b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "04f9d53cc9054e0d9241344d50fea144": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c979ed1fe5c04d6cbb807adc67cf7483": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2a1133e914444f7194c4e4a91202bb4f",
              "IPY_MODEL_b78d5056028744c8a3cb6536e37e9d04",
              "IPY_MODEL_9df3af0baf1544aba8dfc1ac5ce01e66"
            ],
            "layout": "IPY_MODEL_4d2d4cb2e24a4dc9817a479f3b3e2b5f"
          }
        },
        "2a1133e914444f7194c4e4a91202bb4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_59adb35ad416480795c050a6955136e2",
            "placeholder": "​",
            "style": "IPY_MODEL_a3cee561433e46b6ad2a14cfcc3ec978",
            "value": "Downloading (…)/main/tokenizer.json: 100%"
          }
        },
        "b78d5056028744c8a3cb6536e37e9d04": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_47172b43baae4a65b005cd7e12ce8801",
            "max": 1355256,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_74218bef40a94080863681fa027a6b4f",
            "value": 1355256
          }
        },
        "9df3af0baf1544aba8dfc1ac5ce01e66": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e263cd5278c44bf8a56421a8087a926f",
            "placeholder": "​",
            "style": "IPY_MODEL_06e69aba754d475fa91e8146d4fd826d",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 21.7MB/s]"
          }
        },
        "4d2d4cb2e24a4dc9817a479f3b3e2b5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "59adb35ad416480795c050a6955136e2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a3cee561433e46b6ad2a14cfcc3ec978": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "47172b43baae4a65b005cd7e12ce8801": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "74218bef40a94080863681fa027a6b4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e263cd5278c44bf8a56421a8087a926f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "06e69aba754d475fa91e8146d4fd826d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3650d9274cf847f68a9cf344acc80820": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b26445e01cb34e319127e7cd89b4a798",
              "IPY_MODEL_5a454fecbc6e4a73bd56d00b039e4711",
              "IPY_MODEL_6b5a8680a024497b9f5803864167b6e0"
            ],
            "layout": "IPY_MODEL_e2da65cf24a148f6b309a040e7ae1673"
          }
        },
        "b26445e01cb34e319127e7cd89b4a798": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_484ae5b1fcfc405192ba5e2492713b90",
            "placeholder": "​",
            "style": "IPY_MODEL_22951b6eb14c46e9ab25f0ddba282478",
            "value": "Downloading model.safetensors: 100%"
          }
        },
        "5a454fecbc6e4a73bd56d00b039e4711": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d5e7f00445b94674b63de4120facefc3",
            "max": 548105171,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a56b48313c9840b492163cd6ad1bcf43",
            "value": 548105171
          }
        },
        "6b5a8680a024497b9f5803864167b6e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_29106f775ba6429e9f5756641a15b620",
            "placeholder": "​",
            "style": "IPY_MODEL_3fa0fc0c525e4eef8ffbb02bdd9c4495",
            "value": " 548M/548M [00:03&lt;00:00, 192MB/s]"
          }
        },
        "e2da65cf24a148f6b309a040e7ae1673": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "484ae5b1fcfc405192ba5e2492713b90": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "22951b6eb14c46e9ab25f0ddba282478": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d5e7f00445b94674b63de4120facefc3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a56b48313c9840b492163cd6ad1bcf43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "29106f775ba6429e9f5756641a15b620": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3fa0fc0c525e4eef8ffbb02bdd9c4495": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ac6baa9a483b49bcb4b65327c1e06dd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a40ae2f32e204a52b96b045e1ac06f33",
              "IPY_MODEL_23a669ce27a9444482fd326538224175",
              "IPY_MODEL_594e748d6b214233b75ef7ee6c269d80"
            ],
            "layout": "IPY_MODEL_191f73baddf7435b91e19b694d577b6b"
          }
        },
        "a40ae2f32e204a52b96b045e1ac06f33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1c7c170fd1144a4cbcd2ec857603c8da",
            "placeholder": "​",
            "style": "IPY_MODEL_c31bc671361b420e8f68e31c6a945f21",
            "value": "Downloading (…)neration_config.json: 100%"
          }
        },
        "23a669ce27a9444482fd326538224175": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a67c58c245c74ed6a5e1997fbd5f1a16",
            "max": 124,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_df75eba4d3de471991c9d5046d858110",
            "value": 124
          }
        },
        "594e748d6b214233b75ef7ee6c269d80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e6d94aca4e484f0fb17253793f53d220",
            "placeholder": "​",
            "style": "IPY_MODEL_96177d66deea41a989f5fb947bc7cc2d",
            "value": " 124/124 [00:00&lt;00:00, 5.15kB/s]"
          }
        },
        "191f73baddf7435b91e19b694d577b6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1c7c170fd1144a4cbcd2ec857603c8da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c31bc671361b420e8f68e31c6a945f21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a67c58c245c74ed6a5e1997fbd5f1a16": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "df75eba4d3de471991c9d5046d858110": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e6d94aca4e484f0fb17253793f53d220": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "96177d66deea41a989f5fb947bc7cc2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AYLjU9bVQiI9"
      },
      "source": [
        "#  🤖 A pratical look inside of Transformers 🤖\n",
        "\n",
        "📅 _Data Science Summer School 2023, 22.08.2023_\n",
        "\n",
        "👨‍🏫 By [Moritz Laurer](https://www.linkedin.com/in/moritz-laurer/).\n",
        "For questions, reach out to: m.laurer@vu.nl\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a target=\"_blank\" href=\"https://colab.research.google.com/github/MoritzLaurer/summer-school-transformers-2023/blob/main/2_inside_transformers.ipynb\">\n",
        "  <img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/>\n",
        "</a>"
      ],
      "metadata": {
        "id": "gOzoalCdFoQc"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Transformers' main components"
      ],
      "metadata": {
        "id": "0-JnK7Zkhmj6"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Hugging Face Transformers has two main components:**\n"
      ],
      "metadata": {
        "id": "u2v9wkhQhuf4"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fHkp89twf4Zq"
      },
      "source": [
        "\n",
        "1. The **tokenizer** prepares the text in a clean format, which the model understands.\n",
        "    - A token is a word or a sub-word unit. In BERT's vocabulary, the word \"good\" is one token and the word \"darwinism\" is two tokens  (\"darwin\" and \"ism\")\n",
        "    - The tokenizer transforms words into token-ids. With these token-ids, BERT can link words to any token it has already learned during pre-training.\n",
        "\n",
        "2. The **model** processes the tokenizer's ouput and returns a prediction, e.g. which class an input text belongs to.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Independently of the type of model (classification, summarisation, translation, etc.), these two components are almost the same."
      ],
      "metadata": {
        "id": "A4tmhG0piJ0P"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers~=4.31.0  # The Transformers library from Hugging Face"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TNjndqEeVP5h",
        "outputId": "afc89ee5-bf50-4c0a-f379-d4d9812f8a56"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers~=4.31.0\n",
            "  Downloading transformers-4.31.0-py3-none-any.whl (7.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.4/7.4 MB\u001b[0m \u001b[31m23.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers~=4.31.0) (3.12.2)\n",
            "Collecting huggingface-hub<1.0,>=0.14.1 (from transformers~=4.31.0)\n",
            "  Downloading huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m21.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers~=4.31.0) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers~=4.31.0) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers~=4.31.0) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers~=4.31.0) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers~=4.31.0) (2.31.0)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers~=4.31.0)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m38.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers~=4.31.0)\n",
            "  Downloading safetensors-0.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m50.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers~=4.31.0) (4.66.1)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers~=4.31.0) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers~=4.31.0) (4.7.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers~=4.31.0) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers~=4.31.0) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers~=4.31.0) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers~=4.31.0) (2023.7.22)\n",
            "Installing collected packages: tokenizers, safetensors, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.16.4 safetensors-0.3.2 tokenizers-0.13.3 transformers-4.31.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Models like BERT (encoders)"
      ],
      "metadata": {
        "id": "j0axlrhOhTGu"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hykl2GkBhq5F"
      },
      "source": [
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
        "import torch"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HneEleBYh_tY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 257,
          "referenced_widgets": [
            "628978270c664edeb5568981ce8f7656",
            "445fe77c6f5d4dc392d1a2aebf7b7cb7",
            "09515ecd15974497a6dcbb8609abc6b3",
            "714aa79898964658b48187bbf9a5d778",
            "b5bcdfdd111c405686570ccb37778f7c",
            "75e5b3578e65430f8ae006d26de2e3fe",
            "af1bd955ee1e44b2a745b488295dbe96",
            "664642914e15416493471cddde224eae",
            "a0071b707009486097fd4de5dcea9a4e",
            "b2331dcc520644fe890ded18881107d6",
            "31d1dffd7e1a454b8233d86b7bae2aae",
            "a571bec8ed2f4f1da463abd8d186b9f9",
            "93190f571e244af29319f52919e1227c",
            "c1c1f31bf2a548feb1deba0b16f9a2e3",
            "ff44498d5be44d2691c6d95bf96047ad",
            "2d5662cc171940e28737b4f29045335e",
            "06546d4e21f541d7b7fa63f5cc2d281b",
            "521da74e072c48e788011a031a2382cd",
            "c7611f88a43f41158bb20a7ba5450b3a",
            "a58839694f2547e3b9c2ead5ececeb87",
            "c385dbf2c41c4a44ab3e59bfdd8864db",
            "0a20cca2c78742b4944f1beb25befc77",
            "fed769e6b2e84dc7bb23b929c2da9147",
            "a83d033e2f5a4d628969f2fc8886a3cb",
            "fda796f1745e43c5a7a93637db485ef6",
            "40df0b29612744ca8c9d8423d7919ed2",
            "c992253d497a48d5830984036a366770",
            "d963037e3ed14001bce8f09cd05bde0d",
            "18ca582b9c404f46acb64c316561f2d5",
            "ca2eec18eb75491c914de37f8576818e",
            "6ad61d3a95334de296d8dc5f36e60129",
            "12b3675b0333425f9e8f5c09a5e7ec48",
            "33c912f28e664881a07eb35c183adfc9",
            "bbed07462383421ebb27686ccdd1af62",
            "84d35a6c1f414b0e8b176fb481ec23cf",
            "d36ed3c17ad042c5a26959f625281f3b",
            "8ae60ea29e5443a9b613d3e96f9bf1e4",
            "64c8e63a1f2f4a3f85998f2b6d992f5c",
            "60f53ba294844ca28e8b7d2a2d3b82e9",
            "5a624c3f1669455090d46763a80d2169",
            "8fd9292adecb4a829acd8d9f79bea3ae",
            "f4b5f177ae6846f2943058f415182743",
            "ead2510311834997a4fb2d53ae12651c",
            "e9b88722d12e447caeac4de2a93d9494"
          ]
        },
        "outputId": "bb1314f9-bc1b-4013-a483-63eaffd48f77"
      },
      "source": [
        "# load any classification model from the HuggingFace model hub\n",
        "# See here: https://huggingface.co/models?pipeline_tag=text-classification\n",
        "\n",
        "model_name = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "\n",
        "# instantiate the tokenizer\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "# instantiate the model\n",
        "model = AutoModelForSequenceClassification.from_pretrained(model_name)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/48.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "628978270c664edeb5568981ce8f7656"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/629 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a571bec8ed2f4f1da463abd8d186b9f9"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fed769e6b2e84dc7bb23b929c2da9147"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading model.safetensors:   0%|          | 0.00/268M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bbed07462383421ebb27686ccdd1af62"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zluWT2EpHd64"
      },
      "source": [
        "### Tokenization\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vLKD6HULf4hX",
        "outputId": "b1e1c651-b56a-47ec-b0d7-d5e19ce9201a"
      },
      "source": [
        "### 1. Tokenization\n",
        "# Tokenizer documentation: https://huggingface.co/transformers/main_classes/tokenizer.html\n",
        "\n",
        "text = 'I believe that the EU is trustworthy.'\n",
        "print(f\"Input text: '{text}'\\n\")\n",
        "\n",
        "input_ids = tokenizer(text, truncation=True, return_tensors=\"pt\")[\"input_ids\"]\n",
        "print(f\"\"\"The tokenizer splits the text string into separate tokens. A token is either an entire word,\n",
        "or a 'sub-word unit' in case of rare words (or punctuation).\n",
        "The word 'trustworthy', for example is split into two tokens: {tokenizer.tokenize(\"Trustworthy\")}.\n",
        "The main advantage of these sub-word units is that rare words cannot be out-of-vocabulary (an issue of other text-as-data approaches).\n",
        "Transformer models typically have a vocabulary of around 30.000 - 250.000 tokens, learned from the training data.\n",
        "Here is e.g. the vocabulary of DistilBERT: https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt\\n\"\"\")\n",
        "\n",
        "print(f\"The input text is split into the following tokens:\\n{tokenizer.tokenize(text)}.\")\n",
        "print(\"The tokenizer then maps each token to the corresponding token ID in the model's vocabulary:\")\n",
        "print(input_ids[0].tolist()[1:-1])\n",
        "print(\"Transformer models only understand these token IDs.\\n\")\n",
        "\n",
        "print(\"\"\"In addition, the tokenizer adds two special tokens:\n",
        " First, the [CLS] (classification) token is always added at the beginning.\n",
        "        While individual tokens represent individual (sub)words, the [CLS] token represents the entire text.\n",
        "        The [CLS] token \"is  used  as  the  aggregate sequence representation for classification tasks\" (Devlin et al. 2019: 4). Details: https://arxiv.org/pdf/1810.04805.pdf\n",
        " Second, the [SEP] token separates two texts. It is useful for tasks which require two text inputs, for example Questions & Answer tasks.\n",
        "        (It is not relevant in our case)\n",
        "\\n\"\"\")\n",
        "\n",
        "print(\"\"\"The final input for a BERT transformer model therefore looks like this:\"\"\")\n",
        "token_strings = tokenizer.convert_ids_to_tokens(ids=input_ids[0])\n",
        "#token_strings = tokenizer.tokenize(text)\n",
        "for token_id, token_string in zip(input_ids[0].tolist(), token_strings):\n",
        "  print(token_id, \" == \", token_string)\n",
        "\n",
        "\n",
        "# entire vocabulary: tokenizer.pretrained_vocab_files_map[\"vocab_file\"][\"distilbert-base-uncased\"]\n",
        "# https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Input text: 'I believe that the EU is trustworthy.'\n",
            "\n",
            "The tokenizer splits the text string into separate tokens. A token is either an entire word,\n",
            "or a 'sub-word unit' in case of rare words (or punctuation).\n",
            "The word 'trustworthy', for example is split into two tokens: ['trust', '##worthy'].\n",
            "The main advantage of these sub-word units is that rare words cannot be out-of-vocabulary (an issue of other text-as-data approaches).\n",
            "Transformer models typically have a vocabulary of around 30.000 - 250.000 tokens, learned from the training data.\n",
            "Here is e.g. the vocabulary of DistilBERT: https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt\n",
            "\n",
            "The input text is split into the following tokens:\n",
            "['i', 'believe', 'that', 'the', 'eu', 'is', 'trust', '##worthy', '.'].\n",
            "The tokenizer then maps each token to the corresponding token ID in the model's vocabulary:\n",
            "[1045, 2903, 2008, 1996, 7327, 2003, 3404, 13966, 1012]\n",
            "Transformer models only understand these token IDs.\n",
            "\n",
            "In addition, the tokenizer adds two special tokens:\n",
            " First, the [CLS] (classification) token is always added at the beginning.\n",
            "        While individual tokens represent individual (sub)words, the [CLS] token represents the entire text.\n",
            "        The [CLS] token \"is  used  as  the  aggregate sequence representation for classification tasks\" (Devlin et al. 2019: 4). Details: https://arxiv.org/pdf/1810.04805.pdf\n",
            " Second, the [SEP] token separates two texts. It is useful for tasks which require two text inputs, for example Questions & Answer tasks.\n",
            "        (It is not relevant in our case)\n",
            "\n",
            "\n",
            "The final input for a BERT transformer model therefore looks like this:\n",
            "101  ==  [CLS]\n",
            "1045  ==  i\n",
            "2903  ==  believe\n",
            "2008  ==  that\n",
            "1996  ==  the\n",
            "7327  ==  eu\n",
            "2003  ==  is\n",
            "3404  ==  trust\n",
            "13966  ==  ##worthy\n",
            "1012  ==  .\n",
            "102  ==  [SEP]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Tokens (words) flowing through the neural network"
      ],
      "metadata": {
        "id": "rw7qQhjGcZ8d"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1z8vNYtliIaJ",
        "outputId": "38d6ea3a-33c0-4b2c-8932-b5fcde60b325"
      },
      "source": [
        "### Processing the input with the model\n",
        "# Model class documentation: https://huggingface.co/transformers/main_classes/model.html\n",
        "# Documentation for DistilBERT specifically: https://huggingface.co/transformers/model_doc/distilbert.html\n",
        "\n",
        "print(f\"\"\"\\nAfter the preprocessing by the tokenizer, the model then feeds the sequence of tokens through the neural network.\n",
        "Each token is represented by a vector of 768 numbers (a 768 dimensional tensor).\n",
        "The tensor for the token \"trust\" looks for example like this before being fed into the first neural network layer\n",
        "(only 100 numbers are displayed):\\n\"\"\")\n",
        "print(model.distilbert.embeddings.word_embeddings(input_ids[0][7])[:100], \"\\n\")\n",
        "\n",
        "print(f\"\"\"The tensors for each token are then fed through and transformed by between 6-24~ neural network layers.\\n\"\"\")\n",
        "\n",
        "output = model(input_ids, output_hidden_states=True, output_attentions=False, return_dict=True)\n",
        "print(\"Same word after the first layer:\\n\\n\", output.hidden_states[1][0][7][:100], \"\\n\")  # same word embedding after the first attention layer\n",
        "print(\"Same word after the second layer:\\n\\n\", output.hidden_states[2][0][7][:100], \"\\n\")  # same word embedding after the second attention layer\n",
        "#print(\"Same word after the third layer:\\n\", output.hidden_states[3][0][7][:100], \"\\n\")  # same word embedding after the third attention layer\n",
        "print(\"\\n ... etc ...\\n\")\n",
        "\n",
        "print(f'The final output is a a contextualised representation of the sequence: \"{text}\"')\n",
        "#output.hidden_states[6][0][0][:100]  # final CLS token"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "After the preprocessing by the tokenizer, the model then feeds the sequence of tokens through the neural network.\n",
            "Each token is represented by a vector of 768 numbers (a 768 dimensional tensor).\n",
            "The tensor for the token \"trust\" looks for example like this before being fed into the first neural network layer\n",
            "(only 100 numbers are displayed):\n",
            "\n",
            "tensor([-0.0263, -0.0292, -0.0186,  0.0289,  0.0225,  0.0005, -0.0649,  0.0440,\n",
            "         0.0201,  0.0052, -0.0857, -0.0903, -0.0182, -0.0214, -0.0514, -0.0074,\n",
            "        -0.0361, -0.0715,  0.0125, -0.0320, -0.0118, -0.0252, -0.0431, -0.0383,\n",
            "         0.0073,  0.0188,  0.0049, -0.0829, -0.0150, -0.0313, -0.0517,  0.0518,\n",
            "         0.0099,  0.0418, -0.0135, -0.0256, -0.0432, -0.0029, -0.0191,  0.0006,\n",
            "         0.0023,  0.0052, -0.0705, -0.0053, -0.0237, -0.0131,  0.0082, -0.0160,\n",
            "        -0.0512,  0.0171,  0.0104, -0.0164, -0.0536, -0.0759, -0.0407, -0.0006,\n",
            "        -0.0331, -0.0792,  0.0354, -0.0010, -0.0222, -0.0015, -0.0628, -0.0206,\n",
            "        -0.1149, -0.0215, -0.0275, -0.0074, -0.0037,  0.0314, -0.0694, -0.0056,\n",
            "        -0.0402, -0.0959, -0.1495, -0.0860, -0.0756, -0.0939, -0.0162,  0.0127,\n",
            "        -0.0333, -0.0174, -0.0606, -0.0488,  0.0328, -0.0653, -0.0148,  0.0429,\n",
            "        -0.0382,  0.0021,  0.0154, -0.0832, -0.0312,  0.0751, -0.0100, -0.0326,\n",
            "         0.0089, -0.0080, -0.0745, -0.0516], grad_fn=<SliceBackward0>) \n",
            "\n",
            "The tensors for each token are then fed through and transformed by between 6-24~ neural network layers.\n",
            "\n",
            "Same word after the first layer:\n",
            "\n",
            " tensor([ 1.0197, -0.5487,  0.3499,  0.4607,  0.8391, -0.1893, -1.2744,  0.4050,\n",
            "         0.5758,  0.5190, -1.3808, -0.4710,  0.1655, -0.9737, -0.9132,  0.0945,\n",
            "         0.1522, -0.7325,  0.4531, -0.1877, -0.0658,  0.5347, -0.2890,  0.2111,\n",
            "         0.0095,  1.8999,  0.3661, -0.1923, -0.1967,  0.0840, -0.9437,  0.6535,\n",
            "         0.8659,  1.1503,  0.5523, -0.0571, -0.5224,  0.5320,  0.4575,  0.0705,\n",
            "         0.5444,  0.6075, -0.5054,  0.6493, -0.7816, -0.1386,  0.9620,  0.1940,\n",
            "         0.2520,  0.2409,  0.3830, -0.0106, -0.9761, -0.4375, -0.1450,  0.3104,\n",
            "        -0.4289, -0.2048,  0.7036,  0.0452,  0.3296,  0.4499,  0.4429, -0.7283,\n",
            "        -2.3567,  0.3876, -0.9062,  0.0851, -0.0852,  0.0829, -0.8152, -0.3009,\n",
            "        -0.6328, -1.0584, -2.2323, -0.9247, -1.3073, -0.2668,  0.3371,  0.0062,\n",
            "         0.2274, -0.0837, -1.3364, -0.1008,  0.2650, -1.4553,  0.6272,  0.5761,\n",
            "        -0.1431, -0.0612, -0.0905, -0.8610, -0.4050,  1.5188,  0.6204, -0.2890,\n",
            "         0.3655,  0.1397, -1.0399, -0.5957], grad_fn=<SliceBackward0>) \n",
            "\n",
            "Same word after the second layer:\n",
            "\n",
            " tensor([ 8.6304e-01, -8.6820e-01,  7.8730e-01, -2.2138e-01,  1.2101e+00,\n",
            "         3.8670e-01, -1.0896e+00,  8.7422e-02,  4.7191e-01,  3.4434e-02,\n",
            "        -1.0020e+00,  6.2982e-01, -1.2089e-01, -5.2946e-01, -1.1913e+00,\n",
            "        -4.6408e-01,  1.9621e-02, -7.6431e-01, -4.3572e-01, -2.6467e-01,\n",
            "         1.8397e-01,  1.6879e-01,  3.4552e-02,  3.8005e-01, -1.0561e+00,\n",
            "         1.5122e+00,  8.2248e-01,  1.1769e-02,  6.4608e-02,  7.2765e-01,\n",
            "        -6.4231e-01, -1.5865e-02,  3.4215e-01,  1.7413e+00,  4.9454e-01,\n",
            "        -5.1311e-01, -3.9811e-01,  3.1635e-01,  9.2214e-01, -5.0008e-02,\n",
            "        -1.1514e-01,  6.3453e-01, -2.1228e-02,  5.0645e-01, -3.6516e-01,\n",
            "         9.9460e-02,  5.1015e-01, -2.4458e-01, -7.1837e-02, -1.2050e-01,\n",
            "        -1.2111e-01, -5.2317e-01, -9.3992e-01, -3.8500e-01,  1.9844e-01,\n",
            "        -7.1985e-01, -1.7236e+00, -2.9698e-01,  5.4476e-01,  6.2087e-01,\n",
            "         1.1353e+00,  4.7976e-02,  7.7644e-02, -1.1994e+00, -1.6474e+00,\n",
            "        -8.3197e-02, -7.1110e-01,  4.4613e-01, -1.4741e-01, -1.5836e-02,\n",
            "         9.5763e-04, -1.7963e-01, -2.0893e-01, -8.2586e-01, -1.6915e+00,\n",
            "        -3.0478e-01, -7.3935e-01, -3.4275e-01,  7.6856e-01, -1.3046e-01,\n",
            "         6.7788e-01, -1.4471e-01, -1.2505e+00, -2.1345e-01, -2.3939e-01,\n",
            "        -1.3216e+00,  1.0545e+00, -1.3779e-01, -2.7386e-01,  3.0029e-01,\n",
            "        -6.4045e-01, -9.0891e-01,  1.5595e-01,  1.4556e+00,  4.6438e-01,\n",
            "        -8.5181e-02,  5.4725e-01,  3.0987e-01, -6.2029e-01, -6.2813e-01],\n",
            "       grad_fn=<SliceBackward0>) \n",
            "\n",
            "\n",
            " ... etc ...\n",
            "\n",
            "The final output is a a contextualised representation of the sequence: \"I believe that the EU is trustworthy.\"\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ey7iZ13DsoKa",
        "outputId": "f44d4940-97ae-4ceb-e620-efa14d0b9db8"
      },
      "source": [
        "print(\"This is what the different model layers ('the architecture') look like:\\n\")\n",
        "print(model)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "This is what the different model layers ('the architecture') look like:\n",
            "\n",
            "DistilBertForSequenceClassification(\n",
            "  (distilbert): DistilBertModel(\n",
            "    (embeddings): Embeddings(\n",
            "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
            "      (position_embeddings): Embedding(512, 768)\n",
            "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "      (dropout): Dropout(p=0.1, inplace=False)\n",
            "    )\n",
            "    (transformer): Transformer(\n",
            "      (layer): ModuleList(\n",
            "        (0-5): 6 x TransformerBlock(\n",
            "          (attention): MultiHeadSelfAttention(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "          )\n",
            "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (ffn): FFN(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "            (activation): GELUActivation()\n",
            "          )\n",
            "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (pre_classifier): Linear(in_features=768, out_features=768, bias=True)\n",
            "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
            "  (dropout): Dropout(p=0.2, inplace=False)\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### The final output"
      ],
      "metadata": {
        "id": "SNThz6Mdc6Ei"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W1ecRDjcsh-Z",
        "outputId": "bd116717-60c6-4d26-ba19-65531b1eb588"
      },
      "source": [
        "print(f\"\"\"At the end, Transformer models always output so called 'logits',\\n one number for each class the model was trained to classify text into.\\n\n",
        "Our input text was: '{text}'\\n\n",
        "These logis represent the predicted probability for our binary sentiment classification task:\\n\\n{output[\"logits\"][0].tolist()}\\n\"\"\")\n",
        "\n",
        "print(\"Logits are not very interpretable, so they are then converted to percentages.\\nEach percentages represents the model's prediction, which class the input text belongs to.\\n\")\n",
        "probabilities = torch.softmax(output[\"logits\"][0], -1).tolist()\n",
        "label_names = model.config.id2label.values()\n",
        "prediction = {name: round(float(pred) * 100, 1) for pred, name in zip(probabilities, label_names)}\n",
        "print(prediction)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "At the end, Transformer models always output so called 'logits',\n",
            " one number for each class the model was trained to classify text into.\n",
            "\n",
            "Our input text was: 'I believe that the EU is trustworthy.'\n",
            "\n",
            "These logis represent the predicted probability for our binary sentiment classification task:\n",
            "\n",
            "[-3.50547456741333, 3.680955171585083]\n",
            "\n",
            "Logits are not very interpretable, so they are then converted to percentages.\n",
            "Each percentages represents the model's prediction, which class the input text belongs to.\n",
            "\n",
            "{'NEGATIVE': 0.1, 'POSITIVE': 99.9}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x_AJJsLQHq3l"
      },
      "source": [
        "### Everything put together\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PZK019KRAxHm",
        "outputId": "67c03fda-0e13-46a8-c141-03798a9c9ec2"
      },
      "source": [
        "## In short, the code looks like this:\n",
        "\n",
        "# load the relevant functions from HuggingFace and PyTorch\n",
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
        "import torch\n",
        "\n",
        "# Choose any classification model from the model hub\n",
        "model_name = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "\n",
        "# instantiate the tokenizer and the model\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "model = AutoModelForSequenceClassification.from_pretrained(model_name)\n",
        "\n",
        "# tokenization\n",
        "text = 'I believe that the EU is trustworthy.'\n",
        "input = tokenizer(text, truncation=True, return_tensors=\"pt\")[\"input_ids\"]\n",
        "\n",
        "# model prediction\n",
        "output = model(input, output_hidden_states=False, output_attentions=False, return_dict=True)\n",
        "probabilities = torch.softmax(output[\"logits\"][0], -1).tolist()\n",
        "label_names = model.config.id2label.values()\n",
        "prediction = {name: round(float(pred) * 100, 1) for pred, name in zip(probabilities, label_names)}\n",
        "print(prediction)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'NEGATIVE': 0.1, 'POSITIVE': 99.9}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rrJNiO8HBX8v",
        "outputId": "c0647e30-fec8-44c0-d4e0-f1a053674a29"
      },
      "source": [
        "## Or via the simplified pipeline:\n",
        "from transformers import pipeline\n",
        "pipe_classification = pipeline(\"text-classification\", model=\"distilbert-base-uncased-finetuned-sst-2-english\", top_k=2)\n",
        "text = 'I believe that the EU is trustworthy.'\n",
        "pipe_classification(text)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda'\n",
            "Xformers is not installed correctly. If you want to use memory_efficient_attention to accelerate training use the following command to install Xformers\n",
            "pip install xformers.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[{'label': 'POSITIVE', 'score': 0.9992438554763794},\n",
              "  {'label': 'NEGATIVE', 'score': 0.0007562137907370925}]]"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Generative models like GPT (decoders)"
      ],
      "metadata": {
        "id": "RCq3Nf6-z4LS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
        "\n",
        "# https://huggingface.co/gpt2\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"gpt2\")\n",
        "model = AutoModelForCausalLM.from_pretrained(\"gpt2\")\n",
        "\n",
        "prompt = \"Today I believe we can finally\"\n",
        "inputs = tokenizer(prompt, return_tensors=\"pt\")\n",
        "\n",
        "outputs = model.generate(**inputs, max_length=30)\n",
        "\n",
        "outputs_decoded = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n",
        "print(outputs_decoded)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 433,
          "referenced_widgets": [
            "539c9288cdc74a32a709a3e1b98e71d1",
            "5ca3841351a642019b08f0c0f0ae7e08",
            "b721bc8b242048998731ebe3daaf6259",
            "81e7bada0f49408cbd75289e466166e1",
            "8d7e6fb1ab044247ad9c7e21aebbba43",
            "35496e7f769a4b7685a4108bc9178a12",
            "97e6f6c19113422496f724151e557735",
            "4fbe013bac44442798c8bc722fb0d7e3",
            "236a29e8005544b9ac4ca2862e8713d3",
            "0b0dd87d3013401f9e045fefc1629ba6",
            "79821bf48a6d4af6967ca4ea7cdbc365",
            "c33c874b10a1490a89a683faa6245a89",
            "0a623502032a4da1bd5194828523f159",
            "433eb54725eb45bf82785342aa2d2286",
            "85410095beb442b5bf8feefbc6250aa0",
            "f340f97f33864547a070132e40862fcd",
            "f23cd1d36b3e49f5ac6d19eb0857d106",
            "3f392df4d4304cf5913ebc58f3cd089e",
            "cb573e349e2d412195fc85b26d2407dd",
            "31fba667ebcc4cb98ee4ac3abc8338d4",
            "48c653fb92be4334a27bad267ef1c6ef",
            "51d4f6a5ec564ef79c6ac9fd2d5ecb4e",
            "5adee91cae594fd59be190663a701bbb",
            "5788ef8ed7f54f95afb6ad0c797d59ba",
            "bc935840d5f040709002c31a97ced449",
            "275d9f1b12de4848bf7a1c06211f9044",
            "f5f2ef2a3e4740ce99aa5300035c9159",
            "c9e92140b1d7485cb37be646087debed",
            "16ec22ff5c78411d8d39fb56735169fb",
            "1fd7fbb548734aa9ad39700e8bded736",
            "c56b39b0634449c0a6f2fce550f57236",
            "5637f751391d4e24834622af93b9e34b",
            "04f9d53cc9054e0d9241344d50fea144",
            "c979ed1fe5c04d6cbb807adc67cf7483",
            "2a1133e914444f7194c4e4a91202bb4f",
            "b78d5056028744c8a3cb6536e37e9d04",
            "9df3af0baf1544aba8dfc1ac5ce01e66",
            "4d2d4cb2e24a4dc9817a479f3b3e2b5f",
            "59adb35ad416480795c050a6955136e2",
            "a3cee561433e46b6ad2a14cfcc3ec978",
            "47172b43baae4a65b005cd7e12ce8801",
            "74218bef40a94080863681fa027a6b4f",
            "e263cd5278c44bf8a56421a8087a926f",
            "06e69aba754d475fa91e8146d4fd826d",
            "3650d9274cf847f68a9cf344acc80820",
            "b26445e01cb34e319127e7cd89b4a798",
            "5a454fecbc6e4a73bd56d00b039e4711",
            "6b5a8680a024497b9f5803864167b6e0",
            "e2da65cf24a148f6b309a040e7ae1673",
            "484ae5b1fcfc405192ba5e2492713b90",
            "22951b6eb14c46e9ab25f0ddba282478",
            "d5e7f00445b94674b63de4120facefc3",
            "a56b48313c9840b492163cd6ad1bcf43",
            "29106f775ba6429e9f5756641a15b620",
            "3fa0fc0c525e4eef8ffbb02bdd9c4495",
            "ac6baa9a483b49bcb4b65327c1e06dd5",
            "a40ae2f32e204a52b96b045e1ac06f33",
            "23a669ce27a9444482fd326538224175",
            "594e748d6b214233b75ef7ee6c269d80",
            "191f73baddf7435b91e19b694d577b6b",
            "1c7c170fd1144a4cbcd2ec857603c8da",
            "c31bc671361b420e8f68e31c6a945f21",
            "a67c58c245c74ed6a5e1997fbd5f1a16",
            "df75eba4d3de471991c9d5046d858110",
            "e6d94aca4e484f0fb17253793f53d220",
            "96177d66deea41a989f5fb947bc7cc2d"
          ]
        },
        "id": "4R3RIWJj0DBf",
        "outputId": "05090d83-0bac-4da2-9c58-7aebece6efa3"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/665 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "539c9288cdc74a32a709a3e1b98e71d1"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c33c874b10a1490a89a683faa6245a89"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5adee91cae594fd59be190663a701bbb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c979ed1fe5c04d6cbb807adc67cf7483"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading model.safetensors:   0%|          | 0.00/548M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3650d9274cf847f68a9cf344acc80820"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)neration_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ac6baa9a483b49bcb4b65327c1e06dd5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['Today I believe we can finally get to the point where we can make a difference in the lives of the people of the United States of America.\\n']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# https://huggingface.co/gpt2\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"gpt2\")\n",
        "model = AutoModelForCausalLM.from_pretrained(\"gpt2\")\n",
        "\n",
        "prompt = \"Today I believe we can finally\"\n",
        "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
        "\n",
        "# gpt2's vocabulary: https://huggingface.co/gpt2/raw/main/vocab.json\n",
        "\n",
        "outputs = model.generate(\n",
        "    input_ids, max_length=30,\n",
        "    output_scores=True, return_dict_in_generate=True,\n",
        "    output_attentions=False, do_sample=False\n",
        ")\n",
        "\n",
        "print(\"\\nThe output looks quite messy:\\n\")\n",
        "print(outputs)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O_FR2ctZ0yvG",
        "outputId": "def55315-5ff3-4643-cc01-26f7e7c69756"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "The output looks quite messy:\n",
            "\n",
            "GreedySearchDecoderOnlyOutput(sequences=tensor([[8888,  314, 1975,  356,  460, 3443,  651,  284,  262,  966,  810,  356,\n",
            "          460,  787,  257, 3580,  287,  262, 3160,  286,  262,  661,  286,  262,\n",
            "         1578, 1829,  286, 2253,   13,  198]]), scores=(tensor([[-148.6821, -149.2908, -156.0585,  ..., -162.4584, -158.8699,\n",
            "         -150.9391]]), tensor([[-115.6684, -116.1133, -120.9430,  ..., -121.5678, -122.0460,\n",
            "         -116.7151]]), tensor([[-102.9193, -102.8433, -106.7675,  ..., -109.7448, -110.1562,\n",
            "         -104.3626]]), tensor([[-113.0016, -111.4651, -116.1575,  ..., -115.7575, -119.9194,\n",
            "         -112.3750]]), tensor([[-85.0494, -86.2461, -92.9495,  ..., -96.9331, -97.5099, -88.6309]]), tensor([[-101.4949, -101.3607, -106.5000,  ..., -105.3603, -108.2615,\n",
            "         -102.6192]]), tensor([[-144.4079, -143.1993, -147.8557,  ..., -153.8577, -149.4084,\n",
            "         -145.1063]]), tensor([[-142.9678, -142.7155, -149.5450,  ..., -154.4601, -153.3980,\n",
            "         -144.9618]]), tensor([[-107.0143, -106.5294, -112.6670,  ..., -110.6300, -114.1609,\n",
            "         -108.2180]]), tensor([[-136.4494, -134.5512, -138.7625,  ..., -143.6068, -142.1176,\n",
            "         -135.7248]]), tensor([[-109.8607, -109.8173, -119.3194,  ..., -124.9031, -125.0524,\n",
            "         -113.8864]]), tensor([[-106.7517, -105.5091, -110.3003,  ..., -111.1602, -112.8438,\n",
            "         -107.1499]]), tensor([[-130.9350, -128.7390, -133.6039,  ..., -132.8402, -135.4675,\n",
            "         -129.3462]]), tensor([[-39.1256, -36.0714, -45.9913,  ..., -49.6894, -46.9359, -41.5144]]), tensor([[-123.4334, -122.4138, -127.8127,  ..., -124.1029, -129.7923,\n",
            "         -123.4686]]), tensor([[-125.3868, -123.8290, -128.6104,  ..., -124.4080, -129.1839,\n",
            "         -124.1066]]), tensor([[-110.4702, -109.6479, -119.3324,  ..., -122.6706, -120.5586,\n",
            "         -114.1836]]), tensor([[ -97.1338,  -96.7021, -100.0708,  ..., -101.8569, -104.7260,\n",
            "          -97.4858]]), tensor([[-116.5378, -115.0052, -118.4467,  ..., -117.5225, -121.5477,\n",
            "         -115.1521]]), tensor([[-223.0999, -221.9220, -225.2948,  ..., -236.5789, -240.2266,\n",
            "         -223.4586]]), tensor([[-82.4273, -81.1039, -90.4330,  ..., -95.9789, -96.0215, -85.2337]]), tensor([[-178.7834, -180.3470, -182.6522,  ..., -186.7448, -193.9578,\n",
            "         -179.8751]]), tensor([[ -92.5517,  -92.2312, -100.1864,  ..., -108.7172, -106.8259,\n",
            "          -95.7613]]), tensor([[-160.1644, -159.1199, -159.8177,  ..., -175.6872, -177.5045,\n",
            "         -151.5896]])), attentions=None, hidden_states=None)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"GPT2's vocabulary is composed of 50257 tokens. Each has a 'word vector' composed of 768 numbers:\")\n",
        "print(model.transformer.wte)\n",
        "\n",
        "print(f\"\"\"\\nWe can look at GPT2's entire vocabulary here: https://huggingface.co/gpt2/raw/main/vocab.json\n",
        "\\nFor example, the token 'Love' is at position 18565.\n",
        "\\nWe can access it's word vector here (first 100 numbers):\\n\n",
        "{model.transformer.wte.weight[18565][:100]}\n",
        "\"\"\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SIcWIkzx3wmC",
        "outputId": "669cd196-1992-44ad-c34e-0ab15a1081ac"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "GPT2's vocabulary is composed of 50257 tokens. Each has a 'word vector' composed of 768 numbers:\n",
            "Embedding(50257, 768)\n",
            "\n",
            "We can look at GPT2's entire vocabulary here: https://huggingface.co/gpt2/raw/main/vocab.json\n",
            "\n",
            "For example, the token 'Love' is at position 18565.\n",
            "\n",
            "We can access it's word vector here (first 100 numbers):\n",
            "\n",
            "tensor([-0.0521,  0.0063,  0.0773,  0.1031, -0.0365, -0.0253, -0.2183,  0.0222,\n",
            "        -0.1285, -0.0917, -0.0771, -0.1728,  0.1625, -0.1056,  0.1838, -0.0049,\n",
            "         0.0246, -0.0203,  0.0717,  0.1154,  0.0384, -0.2783,  0.0206,  0.0678,\n",
            "        -0.1182, -0.0169,  0.0946, -0.1425,  0.1875, -0.0393,  0.1161, -0.4728,\n",
            "         0.1959,  0.0616, -0.1545,  0.0377, -0.3193,  0.1089,  0.0265, -0.0317,\n",
            "         0.1023, -0.0070,  0.0394,  0.0017,  0.1093,  0.1821,  0.1139, -0.0832,\n",
            "         0.0032, -0.0456, -0.0501, -0.0303, -0.0005, -0.2116, -0.0135, -0.2888,\n",
            "        -0.0223,  0.1179,  0.0222,  0.3011,  0.0113,  0.1022, -0.1399, -0.0165,\n",
            "         0.2658,  0.1221, -0.1152, -0.1597,  0.3961,  0.0925,  0.0124,  0.1865,\n",
            "        -0.1736,  0.1215,  0.0576, -0.0720,  0.1281,  0.0823, -0.0360,  0.0325,\n",
            "         0.2140,  0.1219, -0.0035,  0.2351,  0.0536, -0.0379, -0.1372,  0.2510,\n",
            "         0.0493,  0.1837, -0.1628,  0.0623,  0.0266, -0.0706,  0.2387,  0.0577,\n",
            "         0.0005, -0.1481, -0.1461, -0.0766], grad_fn=<SliceBackward0>)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "print(f\"\"\"\n",
        "While the outputs produce by classifiers like BERT are probabilities of classes,\n",
        "the outputs produced by generators like GPT2 are probabilities of tokens.\n",
        "\\nThese probabilities of tokens are in the 'outputs' object returned by model.generate()\n",
        "\\nThe IDs of the most probably tokens are:\n",
        "{outputs.sequences}\n",
        "\\nThese token IDs can be mapped to actuall words/tokens in the vocabulary:\n",
        "{tokenizer.batch_decode(outputs.sequences, skip_special_tokens=True)}\\n\\n\n",
        "\n",
        "Our original prompt was:\\n'{prompt}'\n",
        "GPT2 then tries to predict the most probable next token. One token after the other.\n",
        "\n",
        "To calculate the first token, it makes a prediction over ALL of the 50257 tokens it knows.\n",
        "Each of the 50257 tokens receives a probability.\n",
        "First the first token, the probability distribution over its ENTIRE vocabulary looks like this:\n",
        "{outputs.scores[0][0]}\n",
        "\n",
        "The ID of the most probable *first* token is {torch.argmax(outputs.scores[0][0], dim=0)}\n",
        "The corresponding token is: {tokenizer.decode(torch.argmax(outputs.scores[0][0], dim=0))}\n",
        "\n",
        "The ID of the most probable *second* token is {torch.argmax(outputs.scores[1][0], dim=0)}\n",
        "The corresponding token is: {tokenizer.decode(torch.argmax(outputs.scores[1][0], dim=0))}\n",
        "\n",
        "The ID of the most probable *third* token is {torch.argmax(outputs.scores[2][0], dim=0)}\n",
        "The corresponding token is: {tokenizer.decode(torch.argmax(outputs.scores[2][0], dim=0))}\n",
        "\n",
        "This is how GPT2 gradually generated the text:\n",
        "{tokenizer.batch_decode(outputs.sequences, skip_special_tokens=True)}\n",
        "\n",
        "The same principles apply to all generative LLMs like GPT4, Llama-2 etc.\n",
        "Only that they are bigger, with a better architecture and better fine-tuning.\n",
        "\"\"\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dREWnBIZ5q0d",
        "outputId": "fa87f721-9b32-4ac3-ee4e-4a6170c8bbe4"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "While the outputs produce by classifiers like BERT are probabilities of classes,\n",
            "the outputs produced by generators like GPT2 are probabilities of tokens.\n",
            "\n",
            "These probabilities of tokens are in the 'outputs' object returned by model.generate()\n",
            "\n",
            "The IDs of the most probably tokens are:\n",
            "tensor([[8888,  314, 1975,  356,  460, 3443,  651,  284,  262,  966,  810,  356,\n",
            "          460,  787,  257, 3580,  287,  262, 3160,  286,  262,  661,  286,  262,\n",
            "         1578, 1829,  286, 2253,   13,  198]])\n",
            "\n",
            "These token IDs can be mapped to actuall words/tokens in the vocabulary:\n",
            "['Today I believe we can finally get to the point where we can make a difference in the lives of the people of the United States of America.\\n']\n",
            "\n",
            "\n",
            "\n",
            "Our original prompt was:\n",
            "'Today I believe we can finally'\n",
            "GPT2 then tries to predict the most probable next token. One token after the other.\n",
            "\n",
            "To calculate the first token, it makes a prediction over ALL of the 50257 tokens it knows.\n",
            "Each of the 50257 tokens receives a probability.\n",
            "First the first token, the probability distribution over its ENTIRE vocabulary looks like this:\n",
            "tensor([-148.6821, -149.2908, -156.0585,  ..., -162.4584, -158.8699,\n",
            "        -150.9391])\n",
            "\n",
            "The ID of the most probable *first* token is 651\n",
            "The corresponding token is:  get\n",
            "\n",
            "The ID of the most probable *second* token is 284\n",
            "The corresponding token is:  to\n",
            "\n",
            "The ID of the most probable *third* token is 262\n",
            "The corresponding token is:  the\n",
            "\n",
            "This is how GPT2 gradually generated the text:\n",
            "['Today I believe we can finally get to the point where we can make a difference in the lives of the people of the United States of America.\\n']\n",
            "\n",
            "The same principles apply to all generative LLMs like GPT4, Llama-2 etc.\n",
            "Only that they are bigger, with a better architecture and better fine-tuning.\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n"
      ],
      "metadata": {
        "id": "wjTJh6DaMsSc"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---"
      ],
      "metadata": {
        "id": "qLBWZ7tJ6K7b"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Reflection  +  Q&A"
      ],
      "metadata": {
        "id": "ja80SRkiLP50"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "**Reading, thinking & asking:** (5 min)\n",
        "* Re-read the notebook and ask any questions you might have in the chat.\n",
        "\n",
        "* Write your answers to the following questions on a piece of paper / digital notebook. While think about these questions, also don't hesitate to ask any questions that come up in the chat.\n",
        "    * In your own words, write down the main differences between models like BERT and models like GPT with regard to their outputs.\n",
        "    * What could be disadvantages and advantages of these two different approaches (encoders vs. decoders)?\n",
        "\n",
        "* I'll collect your questions and answer them after a few minutes.\n",
        "\n"
      ],
      "metadata": {
        "id": "y8uhW4V_AU-m"
      }
    }
  ]
}