LoRA_gemma_7b_finetuning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "00060156431443e59e507168919d334a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5b66d257232945dcb124fb019d905478",
              "IPY_MODEL_2c4b6ae838084f9b936ebeafb76427dd",
              "IPY_MODEL_7cbbe5bace73488c926dc74b42b452f2",
              "IPY_MODEL_37643de38c2c43878899dff2b5d758e2"
            ],
            "layout": "IPY_MODEL_d80913c3b8e74852b8d342b658d20fea"
          }
        },
        "4e6987dbe8244c969c33cffdf5726a8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6d5e39248e0c4873ac16746abeae7963",
            "placeholder": "​",
            "style": "IPY_MODEL_3712c2ab4044434da2599016e998ec6b",
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
          }
        },
        "474e30a825b6408cbcb148a2f647c3ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "PasswordModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "PasswordModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "PasswordView",
            "continuous_update": true,
            "description": "Token:",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_e55ea956378a43be9624eb787fb1bf70",
            "placeholder": "​",
            "style": "IPY_MODEL_45bd851f79784639aafb7e11e1f25b97",
            "value": ""
          }
        },
        "56805da7e8c2422f8d95cb42b361ecff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "CheckboxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Add token as git credential?",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_cc949c97dfea46f0a3ae6485deb6721a",
            "style": "IPY_MODEL_b1670b27f0ab49f69a2c58aadb895328",
            "value": true
          }
        },
        "af6964c50f584131b802f90d2e9548d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ButtonView",
            "button_style": "",
            "description": "Login",
            "disabled": false,
            "icon": "",
            "layout": "IPY_MODEL_fa6e01b2df1747d1bad84ca377f185e3",
            "style": "IPY_MODEL_0b2aff92fed6434db05d22d8ee071831",
            "tooltip": ""
          }
        },
        "35e73ee6c77549fba214b2e5e058a0ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cb445d37990d4a51a3d1b500faec7584",
            "placeholder": "​",
            "style": "IPY_MODEL_142465da37de48a99511d2bbc22751d8",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
          }
        },
        "d80913c3b8e74852b8d342b658d20fea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": "center",
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "flex",
            "flex": null,
            "flex_flow": "column",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "6d5e39248e0c4873ac16746abeae7963": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3712c2ab4044434da2599016e998ec6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e55ea956378a43be9624eb787fb1bf70": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "45bd851f79784639aafb7e11e1f25b97": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cc949c97dfea46f0a3ae6485deb6721a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b1670b27f0ab49f69a2c58aadb895328": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fa6e01b2df1747d1bad84ca377f185e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0b2aff92fed6434db05d22d8ee071831": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "button_color": null,
            "font_weight": ""
          }
        },
        "cb445d37990d4a51a3d1b500faec7584": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "142465da37de48a99511d2bbc22751d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "60478f8b46444b2fb05466ec9b8000d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6113980237414bd589a3e53d3bb550d0",
            "placeholder": "​",
            "style": "IPY_MODEL_d9678c28edb948fe852dc5346548d9b9",
            "value": "Connecting..."
          }
        },
        "6113980237414bd589a3e53d3bb550d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d9678c28edb948fe852dc5346548d9b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5b66d257232945dcb124fb019d905478": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_be2c491128f04b94bfed2dec428495c4",
            "placeholder": "​",
            "style": "IPY_MODEL_f87b822e24d04444804495634dc40c26",
            "value": "Token is valid (permission: write)."
          }
        },
        "2c4b6ae838084f9b936ebeafb76427dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_11a8c0e82ba64334b811ed85e8d8b53b",
            "placeholder": "​",
            "style": "IPY_MODEL_e0be3c06743e49e7a04e3f3a13120f74",
            "value": "Your token has been saved in your configured git credential helpers (store)."
          }
        },
        "7cbbe5bace73488c926dc74b42b452f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_aca8138f45634091904512e42d265bd5",
            "placeholder": "​",
            "style": "IPY_MODEL_0e8ed5a13b5c4222bc721991aaebd01d",
            "value": "Your token has been saved to /root/.cache/huggingface/token"
          }
        },
        "37643de38c2c43878899dff2b5d758e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f038614bbe7f4c04aa1128d179bdb152",
            "placeholder": "​",
            "style": "IPY_MODEL_3e61f5e282ae4d10b9a6a9b6d1a43009",
            "value": "Login successful"
          }
        },
        "be2c491128f04b94bfed2dec428495c4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f87b822e24d04444804495634dc40c26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "11a8c0e82ba64334b811ed85e8d8b53b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e0be3c06743e49e7a04e3f3a13120f74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "aca8138f45634091904512e42d265bd5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0e8ed5a13b5c4222bc721991aaebd01d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f038614bbe7f4c04aa1128d179bdb152": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3e61f5e282ae4d10b9a6a9b6d1a43009": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "32422c5dc0574b6a9e3f0a9a6bb057d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_71cac9ec2c914790a3f24a9ce6298aa2",
              "IPY_MODEL_e198c6bd9fd0422d89c9ff2c6f7d18ec",
              "IPY_MODEL_485d9b49a0814aceae800ceeb9467bdd"
            ],
            "layout": "IPY_MODEL_93232175929547a099d59580136ed260"
          }
        },
        "71cac9ec2c914790a3f24a9ce6298aa2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_563071914fea4a88b7e042805fa2296d",
            "placeholder": "​",
            "style": "IPY_MODEL_d45336ac9dd54a18aac0b5871f012cca",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "e198c6bd9fd0422d89c9ff2c6f7d18ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5999d89f68b4431cbf32c2d75ad3cace",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_359486581f62496abfc9977b17c59c93",
            "value": 2
          }
        },
        "485d9b49a0814aceae800ceeb9467bdd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_31503e450cfc4ba4b6c8dc1b34c9533c",
            "placeholder": "​",
            "style": "IPY_MODEL_e68952a5bbfa4358af79c869b5da218a",
            "value": " 2/2 [00:00&lt;00:00, 135.38it/s]"
          }
        },
        "93232175929547a099d59580136ed260": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "563071914fea4a88b7e042805fa2296d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d45336ac9dd54a18aac0b5871f012cca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5999d89f68b4431cbf32c2d75ad3cace": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "359486581f62496abfc9977b17c59c93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "31503e450cfc4ba4b6c8dc1b34c9533c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e68952a5bbfa4358af79c869b5da218a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "from huggingface_hub import notebook_login\n",
        "\n",
        "notebook_login()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "00060156431443e59e507168919d334a",
            "4e6987dbe8244c969c33cffdf5726a8d",
            "474e30a825b6408cbcb148a2f647c3ce",
            "56805da7e8c2422f8d95cb42b361ecff",
            "af6964c50f584131b802f90d2e9548d7",
            "35e73ee6c77549fba214b2e5e058a0ce",
            "d80913c3b8e74852b8d342b658d20fea",
            "6d5e39248e0c4873ac16746abeae7963",
            "3712c2ab4044434da2599016e998ec6b",
            "e55ea956378a43be9624eb787fb1bf70",
            "45bd851f79784639aafb7e11e1f25b97",
            "cc949c97dfea46f0a3ae6485deb6721a",
            "b1670b27f0ab49f69a2c58aadb895328",
            "fa6e01b2df1747d1bad84ca377f185e3",
            "0b2aff92fed6434db05d22d8ee071831",
            "cb445d37990d4a51a3d1b500faec7584",
            "142465da37de48a99511d2bbc22751d8",
            "60478f8b46444b2fb05466ec9b8000d5",
            "6113980237414bd589a3e53d3bb550d0",
            "d9678c28edb948fe852dc5346548d9b9",
            "5b66d257232945dcb124fb019d905478",
            "2c4b6ae838084f9b936ebeafb76427dd",
            "7cbbe5bace73488c926dc74b42b452f2",
            "37643de38c2c43878899dff2b5d758e2",
            "be2c491128f04b94bfed2dec428495c4",
            "f87b822e24d04444804495634dc40c26",
            "11a8c0e82ba64334b811ed85e8d8b53b",
            "e0be3c06743e49e7a04e3f3a13120f74",
            "aca8138f45634091904512e42d265bd5",
            "0e8ed5a13b5c4222bc721991aaebd01d",
            "f038614bbe7f4c04aa1128d179bdb152",
            "3e61f5e282ae4d10b9a6a9b6d1a43009"
          ]
        },
        "id": "elFJLjCiAH6y",
        "outputId": "de280e96-6a97-4b30-9cd4-7ce0ec9e86f7"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "00060156431443e59e507168919d334a"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zcl2VRoB1C1-",
        "outputId": "47e74fa0-cf24-488f-8201-359669db7f94"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/Neilus03/GEMMA_ORCAMATH_FINETUNING.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i5m6N7eN-aqI",
        "outputId": "2298c42d-49d7-451f-a568-1fa796d7b1de"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'GEMMA_ORCAMATH_FINETUNING'...\n",
            "remote: Enumerating objects: 148, done.\u001b[K\n",
            "remote: Counting objects: 100% (26/26), done.\u001b[K\n",
            "remote: Compressing objects: 100% (20/20), done.\u001b[K\n",
            "remote: Total 148 (delta 11), reused 17 (delta 6), pack-reused 122\u001b[K\n",
            "Receiving objects: 100% (148/148), 60.55 MiB | 27.35 MiB/s, done.\n",
            "Resolving deltas: 100% (74/74), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd GEMMA_ORCAMATH_FINETUNING\n",
        "!pip install -r requirements.txt\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5w1WufSY-jOs",
        "outputId": "d60bf292-d8dc-4726-ae8b-01b6a5ae49ea"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/GEMMA_ORCAMATH_FINETUNING/GEMMA_ORCAMATH_FINETUNING\n",
            "Collecting git+https://github.com/huggingface/accelerate.git (from -r requirements.txt (line 1))\n",
            "  Cloning https://github.com/huggingface/accelerate.git to /tmp/pip-req-build-_7urvl47\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/huggingface/accelerate.git /tmp/pip-req-build-_7urvl47\n",
            "  Resolved https://github.com/huggingface/accelerate.git to commit 60bfdaa9349e3e1a373863a8c5f20baa89cfd000\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting git+https://github.com/huggingface/peft.git (from -r requirements.txt (line 9))\n",
            "  Cloning https://github.com/huggingface/peft.git to /tmp/pip-req-build-z57k6zmq\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/huggingface/peft.git /tmp/pip-req-build-z57k6zmq\n",
            "  Resolved https://github.com/huggingface/peft.git to commit 5471c9a1be61d798447a24660acada40892777ca\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting git+https://github.com/huggingface/transformers.git (from -r requirements.txt (line 10))\n",
            "  Cloning https://github.com/huggingface/transformers.git to /tmp/pip-req-build-b4pi_fxn\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers.git /tmp/pip-req-build-b4pi_fxn\n",
            "  Resolved https://github.com/huggingface/transformers.git to commit b382a09e28c7e59129246ccdf4b00f2cac4547a4\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: appdirs in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 2)) (1.4.4)\n",
            "Requirement already satisfied: loralib in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 3)) (0.1.2)\n",
            "Requirement already satisfied: bitsandbytes>=0.39.0 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 4)) (0.43.0)\n",
            "Requirement already satisfied: black in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 5)) (24.2.0)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 7)) (2.18.0)\n",
            "Requirement already satisfied: fire in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 8)) (0.6.0)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 11)) (0.1.99)\n",
            "Requirement already satisfied: gradio in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 12)) (4.21.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 13)) (1.11.4)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 14)) (1.25.2)\n",
            "Requirement already satisfied: rouge_score in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 15)) (0.1.2)\n",
            "Requirement already satisfied: tokenizers>=0.13.3 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 16)) (0.15.2)\n",
            "Requirement already satisfied: wandb in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 17)) (0.16.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate==0.28.0.dev0->-r requirements.txt (line 1)) (23.2)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate==0.28.0.dev0->-r requirements.txt (line 1)) (5.9.5)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate==0.28.0.dev0->-r requirements.txt (line 1)) (6.0.1)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate==0.28.0.dev0->-r requirements.txt (line 1)) (2.1.0+cu121)\n",
            "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from accelerate==0.28.0.dev0->-r requirements.txt (line 1)) (0.20.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from accelerate==0.28.0.dev0->-r requirements.txt (line 1)) (0.4.2)\n",
            "Requirement already satisfied: click>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from black->-r requirements.txt (line 5)) (8.1.7)\n",
            "Requirement already satisfied: mypy-extensions>=0.4.3 in /usr/local/lib/python3.10/dist-packages (from black->-r requirements.txt (line 5)) (1.0.0)\n",
            "Requirement already satisfied: pathspec>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from black->-r requirements.txt (line 5)) (0.12.1)\n",
            "Requirement already satisfied: platformdirs>=2 in /usr/local/lib/python3.10/dist-packages (from black->-r requirements.txt (line 5)) (4.2.0)\n",
            "Requirement already satisfied: tomli>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from black->-r requirements.txt (line 5)) (2.0.1)\n",
            "Requirement already satisfied: typing-extensions>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from black->-r requirements.txt (line 5)) (4.10.0)\n",
            "Requirement already satisfied: ipython>=7.8.0 in /usr/local/lib/python3.10/dist-packages (from black->-r requirements.txt (line 5)) (7.34.0)\n",
            "Requirement already satisfied: tokenize-rt>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from black->-r requirements.txt (line 5)) (5.2.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 7)) (3.13.1)\n",
            "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 7)) (14.0.2)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 7)) (0.6)\n",
            "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 7)) (0.3.8)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 7)) (1.5.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 7)) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 7)) (4.66.2)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 7)) (3.4.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 7)) (0.70.16)\n",
            "Requirement already satisfied: fsspec[http]<=2024.2.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 7)) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->-r requirements.txt (line 7)) (3.9.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from fire->-r requirements.txt (line 8)) (1.16.0)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from fire->-r requirements.txt (line 8)) (2.4.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.39.0.dev0->-r requirements.txt (line 10)) (2023.12.25)\n",
            "Requirement already satisfied: aiofiles<24.0,>=22.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (23.2.1)\n",
            "Requirement already satisfied: altair<6.0,>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (4.2.2)\n",
            "Requirement already satisfied: fastapi in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (0.110.0)\n",
            "Requirement already satisfied: ffmpy in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (0.3.2)\n",
            "Requirement already satisfied: gradio-client==0.12.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (0.12.0)\n",
            "Requirement already satisfied: httpx>=0.24.1 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (0.27.0)\n",
            "Requirement already satisfied: importlib-resources<7.0,>=1.3 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (6.1.3)\n",
            "Requirement already satisfied: jinja2<4.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (3.1.3)\n",
            "Requirement already satisfied: markupsafe~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (2.1.5)\n",
            "Requirement already satisfied: matplotlib~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (3.7.1)\n",
            "Requirement already satisfied: orjson~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (3.9.15)\n",
            "Requirement already satisfied: pillow<11.0,>=8.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (9.4.0)\n",
            "Requirement already satisfied: pydantic>=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (2.6.3)\n",
            "Requirement already satisfied: pydub in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (0.25.1)\n",
            "Requirement already satisfied: python-multipart>=0.0.9 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (0.0.9)\n",
            "Requirement already satisfied: ruff>=0.2.2 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (0.3.2)\n",
            "Requirement already satisfied: semantic-version~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (2.10.0)\n",
            "Requirement already satisfied: tomlkit==0.12.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (0.12.0)\n",
            "Requirement already satisfied: typer[all]<1.0,>=0.9 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (0.9.0)\n",
            "Requirement already satisfied: uvicorn>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from gradio->-r requirements.txt (line 12)) (0.28.0)\n",
            "Requirement already satisfied: websockets<12.0,>=10.0 in /usr/local/lib/python3.10/dist-packages (from gradio-client==0.12.0->gradio->-r requirements.txt (line 12)) (11.0.3)\n",
            "Requirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from rouge_score->-r requirements.txt (line 15)) (1.4.0)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from rouge_score->-r requirements.txt (line 15)) (3.8.1)\n",
            "Requirement already satisfied: GitPython!=3.1.29,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 17)) (3.1.42)\n",
            "Requirement already satisfied: sentry-sdk>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 17)) (1.41.0)\n",
            "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 17)) (0.4.0)\n",
            "Requirement already satisfied: setproctitle in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 17)) (1.3.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 17)) (67.7.2)\n",
            "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb->-r requirements.txt (line 17)) (3.20.3)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio->-r requirements.txt (line 12)) (0.4)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio->-r requirements.txt (line 12)) (4.19.2)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio->-r requirements.txt (line 12)) (0.12.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 7)) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 7)) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 7)) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 7)) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 7)) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->-r requirements.txt (line 7)) (4.0.3)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from GitPython!=3.1.29,>=1.0.0->wandb->-r requirements.txt (line 17)) (4.0.11)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio->-r requirements.txt (line 12)) (3.7.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio->-r requirements.txt (line 12)) (2024.2.2)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio->-r requirements.txt (line 12)) (1.0.4)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio->-r requirements.txt (line 12)) (3.6)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.24.1->gradio->-r requirements.txt (line 12)) (1.3.1)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx>=0.24.1->gradio->-r requirements.txt (line 12)) (0.14.0)\n",
            "Requirement already satisfied: jedi>=0.16 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (0.19.1)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (4.4.2)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (0.7.5)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (5.7.1)\n",
            "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (3.0.43)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (2.16.1)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (0.2.0)\n",
            "Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (0.1.6)\n",
            "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.10/dist-packages (from ipython>=7.8.0->black->-r requirements.txt (line 5)) (4.9.0)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 12)) (1.2.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 12)) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 12)) (4.49.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 12)) (1.4.5)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 12)) (3.1.2)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio->-r requirements.txt (line 12)) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->-r requirements.txt (line 7)) (2023.4)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0->gradio->-r requirements.txt (line 12)) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2.0->gradio->-r requirements.txt (line 12)) (2.16.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets->-r requirements.txt (line 7)) (3.3.2)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets->-r requirements.txt (line 7)) (2.0.7)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.28.0.dev0->-r requirements.txt (line 1)) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.28.0.dev0->-r requirements.txt (line 1)) (3.2.1)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate==0.28.0.dev0->-r requirements.txt (line 1)) (2.1.0)\n",
            "Requirement already satisfied: colorama<0.5.0,>=0.4.3 in /usr/local/lib/python3.10/dist-packages (from typer[all]<1.0,>=0.9->gradio->-r requirements.txt (line 12)) (0.4.6)\n",
            "Requirement already satisfied: shellingham<2.0.0,>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from typer[all]<1.0,>=0.9->gradio->-r requirements.txt (line 12)) (1.5.4)\n",
            "Requirement already satisfied: rich<14.0.0,>=10.11.0 in /usr/local/lib/python3.10/dist-packages (from typer[all]<1.0,>=0.9->gradio->-r requirements.txt (line 12)) (13.7.1)\n",
            "Requirement already satisfied: starlette<0.37.0,>=0.36.3 in /usr/local/lib/python3.10/dist-packages (from fastapi->gradio->-r requirements.txt (line 12)) (0.36.3)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->rouge_score->-r requirements.txt (line 15)) (1.3.2)\n",
            "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb->-r requirements.txt (line 17)) (5.0.1)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /usr/local/lib/python3.10/dist-packages (from jedi>=0.16->ipython>=7.8.0->black->-r requirements.txt (line 5)) (0.8.3)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio->-r requirements.txt (line 12)) (2023.12.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio->-r requirements.txt (line 12)) (0.33.0)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio->-r requirements.txt (line 12)) (0.18.0)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.10/dist-packages (from pexpect>4.3->ipython>=7.8.0->black->-r requirements.txt (line 5)) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=7.8.0->black->-r requirements.txt (line 5)) (0.2.13)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=10.11.0->typer[all]<1.0,>=0.9->gradio->-r requirements.txt (line 12)) (3.0.0)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx>=0.24.1->gradio->-r requirements.txt (line 12)) (1.2.0)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate==0.28.0.dev0->-r requirements.txt (line 1)) (1.3.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=10.11.0->typer[all]<1.0,>=0.9->gradio->-r requirements.txt (line 12)) (0.1.2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Download Orcamath from the following link: [OrcaMath Dataset](https://drive.google.com/uc?id=1JHDUPlG5igZ1QZ0McNYmUKzixV9pJXIZ) and add it to the folder."
      ],
      "metadata": {
        "id": "EU0WnMs9FtAp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#You can either decompress it manually and store it in the data subfolder or run the following lines\n",
        "\n",
        "!unzip \"/content/orcamath_data.zip\" -d \"/content/GEMMA_ORCAMATH_FINETUNING/data\"\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yJWooYqW-n98",
        "outputId": "af28f95d-2f90-43fb-a28d-23678d5be906"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archive:  /content/orcamath_data.zip\n",
            "  inflating: /content/GEMMA_ORCAMATH_FINETUNING/data/orcamath_data.json  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "To finetune the model on the OrcaMath dataset run the command below"
      ],
      "metadata": {
        "id": "mZoS2eHYJCTe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "!chmod +x run.sh\n",
        "!./run.sh"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6LoEu7pa_SmI",
        "outputId": "31a95f96-1d7f-4d36-c237-63da736bf4cc"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "dir: /content/GEMMA_ORCAMATH_FINETUNING/GEMMA_ORCAMATH_FINETUNING\n",
            "2024-03-12 10:09:40.164884: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-03-12 10:09:40.164944: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-03-12 10:09:40.167089: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-03-12 10:09:42.009705: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Namespace(load_in_4bit=True, load_in_8bit=False, use_lora=True, lora_r=8, lora_alpha=16, lora_target_modules=['q_proj', 'v_proj', 'k_proj', 'o_proj']) True\n",
            "model: google/gemma-2b\n",
            "config.json: 100% 627/627 [00:00<00:00, 3.38MB/s]\n",
            "`low_cpu_mem_usage` was None, now set to True since model is quantized.\n",
            "model.safetensors.index.json: 100% 13.5k/13.5k [00:00<00:00, 16.2MB/s]\n",
            "Downloading shards:   0% 0/2 [00:00<?, ?it/s]\n",
            "model-00001-of-00002.safetensors:   0% 0.00/4.95G [00:00<?, ?B/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   0% 10.5M/4.95G [00:00<01:57, 42.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   0% 21.0M/4.95G [00:00<01:17, 63.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   1% 31.5M/4.95G [00:00<01:03, 77.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   1% 52.4M/4.95G [00:00<00:52, 92.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   1% 73.4M/4.95G [00:00<00:49, 99.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   2% 83.9M/4.95G [00:00<00:48, 99.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   2% 94.4M/4.95G [00:01<00:48, 99.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   2% 105M/4.95G [00:01<00:48, 99.9MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:   2% 115M/4.95G [00:01<00:48, 100MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:   3% 126M/4.95G [00:01<00:48, 100MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   3% 136M/4.95G [00:01<00:47, 101MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   3% 147M/4.95G [00:01<00:47, 101MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   3% 157M/4.95G [00:01<00:47, 100MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   4% 178M/4.95G [00:01<00:46, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   4% 199M/4.95G [00:02<00:44, 105MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   4% 220M/4.95G [00:02<00:44, 107MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   5% 241M/4.95G [00:02<00:44, 107MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   5% 262M/4.95G [00:02<00:43, 107MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   6% 283M/4.95G [00:02<00:44, 105MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   6% 304M/4.95G [00:03<00:44, 104MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   6% 315M/4.95G [00:03<00:44, 104MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   7% 325M/4.95G [00:03<00:44, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   7% 336M/4.95G [00:03<00:44, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   7% 346M/4.95G [00:03<00:45, 101MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   7% 367M/4.95G [00:03<00:44, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   8% 377M/4.95G [00:03<00:44, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   8% 398M/4.95G [00:03<00:43, 105MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   8% 419M/4.95G [00:04<00:42, 106MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   9% 440M/4.95G [00:04<00:42, 107MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   9% 461M/4.95G [00:04<00:42, 106MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  10% 482M/4.95G [00:04<00:42, 105MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  10% 493M/4.95G [00:04<00:42, 104MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  10% 503M/4.95G [00:04<00:43, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  10% 514M/4.95G [00:05<00:43, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  11% 524M/4.95G [00:05<00:43, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  11% 545M/4.95G [00:05<00:42, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  11% 566M/4.95G [00:05<00:41, 105MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  12% 587M/4.95G [00:05<00:41, 105MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  12% 608M/4.95G [00:05<00:40, 106MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  13% 629M/4.95G [00:06<00:40, 107MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  13% 650M/4.95G [00:06<00:40, 106MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  14% 671M/4.95G [00:06<00:41, 104MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  14% 682M/4.95G [00:06<00:41, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  14% 692M/4.95G [00:06<00:41, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  14% 703M/4.95G [00:06<00:41, 101MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  14% 713M/4.95G [00:06<00:41, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  15% 724M/4.95G [00:07<00:41, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  15% 734M/4.95G [00:07<00:41, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  15% 744M/4.95G [00:07<00:41, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  15% 755M/4.95G [00:07<00:40, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  15% 765M/4.95G [00:07<00:43, 95.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  16% 776M/4.95G [00:08<02:19, 29.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  16% 797M/4.95G [00:08<01:32, 44.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  17% 818M/4.95G [00:08<01:12, 56.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  17% 828M/4.95G [00:08<01:05, 63.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  17% 839M/4.95G [00:09<00:58, 69.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  17% 849M/4.95G [00:09<00:53, 76.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  17% 860M/4.95G [00:09<00:50, 80.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  18% 870M/4.95G [00:09<00:47, 85.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  18% 881M/4.95G [00:09<00:46, 87.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  18% 891M/4.95G [00:09<00:46, 87.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  18% 902M/4.95G [00:09<00:45, 88.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  18% 912M/4.95G [00:09<00:45, 88.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  19% 923M/4.95G [00:09<00:43, 91.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  19% 933M/4.95G [00:10<00:43, 92.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  19% 944M/4.95G [00:10<00:41, 95.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  19% 954M/4.95G [00:10<00:40, 97.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  20% 965M/4.95G [00:10<00:40, 98.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  20% 975M/4.95G [00:10<00:39, 99.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  20% 986M/4.95G [00:10<00:39, 99.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  20% 996M/4.95G [00:10<00:39, 99.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  20% 1.01G/4.95G [00:10<00:39, 101MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  21% 1.02G/4.95G [00:10<00:38, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  21% 1.03G/4.95G [00:11<00:39, 98.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  21% 1.04G/4.95G [00:11<00:39, 98.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  21% 1.05G/4.95G [00:11<00:41, 94.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  21% 1.06G/4.95G [00:11<00:41, 93.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  22% 1.07G/4.95G [00:11<00:42, 92.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  22% 1.08G/4.95G [00:11<00:43, 89.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  22% 1.09G/4.95G [00:11<00:41, 92.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  22% 1.10G/4.95G [00:11<00:40, 95.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  23% 1.12G/4.95G [00:11<00:38, 100MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:  23% 1.14G/4.95G [00:12<00:36, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  24% 1.16G/4.95G [00:12<00:35, 106MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  24% 1.18G/4.95G [00:12<00:35, 106MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  24% 1.21G/4.95G [00:12<00:35, 106MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  25% 1.23G/4.95G [00:12<00:35, 105MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  25% 1.24G/4.95G [00:13<00:36, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  25% 1.25G/4.95G [00:13<00:39, 93.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  25% 1.26G/4.95G [00:13<00:40, 91.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  26% 1.27G/4.95G [00:13<00:41, 88.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  26% 1.28G/4.95G [00:13<00:40, 90.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  26% 1.29G/4.95G [00:13<00:39, 92.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  26% 1.30G/4.95G [00:13<00:38, 95.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  27% 1.31G/4.95G [00:13<00:44, 81.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  27% 1.32G/4.95G [00:14<01:07, 54.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  27% 1.33G/4.95G [00:14<00:57, 62.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  27% 1.35G/4.95G [00:14<01:04, 55.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  28% 1.36G/4.95G [00:14<00:57, 62.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  28% 1.37G/4.95G [00:15<00:51, 69.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  28% 1.38G/4.95G [00:15<00:47, 74.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  28% 1.39G/4.95G [00:15<00:46, 76.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  28% 1.41G/4.95G [00:15<00:43, 81.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  29% 1.42G/4.95G [00:15<00:43, 81.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  29% 1.44G/4.95G [00:15<00:38, 91.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  29% 1.45G/4.95G [00:15<00:37, 93.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  29% 1.46G/4.95G [00:15<00:36, 96.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  30% 1.47G/4.95G [00:16<00:35, 97.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  30% 1.48G/4.95G [00:16<00:36, 95.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  30% 1.49G/4.95G [00:16<00:35, 96.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  30% 1.50G/4.95G [00:16<00:36, 93.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  31% 1.51G/4.95G [00:16<00:35, 96.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  31% 1.53G/4.95G [00:16<00:33, 102MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:  31% 1.55G/4.95G [00:16<00:32, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  32% 1.57G/4.95G [00:17<00:32, 104MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  32% 1.59G/4.95G [00:17<00:31, 106MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  33% 1.61G/4.95G [00:17<00:31, 106MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  33% 1.64G/4.95G [00:17<00:31, 105MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  34% 1.66G/4.95G [00:17<00:31, 104MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  34% 1.67G/4.95G [00:17<00:31, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  34% 1.68G/4.95G [00:18<00:31, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  34% 1.69G/4.95G [00:18<00:31, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  34% 1.70G/4.95G [00:18<00:32, 101MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  35% 1.71G/4.95G [00:18<00:32, 101MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  35% 1.72G/4.95G [00:18<00:32, 101MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  35% 1.73G/4.95G [00:18<00:32, 100MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  35% 1.74G/4.95G [00:19<01:38, 32.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  35% 1.75G/4.95G [00:19<01:53, 28.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  36% 1.76G/4.95G [00:20<01:28, 35.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  36% 1.78G/4.95G [00:20<01:01, 51.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  36% 1.79G/4.95G [00:20<00:53, 58.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  37% 1.81G/4.95G [00:20<00:43, 72.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  37% 1.84G/4.95G [00:20<00:37, 82.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  37% 1.85G/4.95G [00:20<00:36, 85.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  38% 1.87G/4.95G [00:21<00:33, 92.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  38% 1.88G/4.95G [00:21<00:33, 92.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  38% 1.89G/4.95G [00:21<00:32, 93.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  38% 1.90G/4.95G [00:21<00:31, 95.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  39% 1.91G/4.95G [00:21<00:32, 94.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  39% 1.92G/4.95G [00:21<00:32, 94.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  39% 1.93G/4.95G [00:21<00:32, 93.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  39% 1.94G/4.95G [00:21<00:31, 94.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  39% 1.95G/4.95G [00:21<00:31, 95.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  40% 1.96G/4.95G [00:22<00:31, 95.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  40% 1.97G/4.95G [00:22<00:30, 97.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  40% 1.98G/4.95G [00:22<00:29, 99.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  40% 1.99G/4.95G [00:22<00:29, 100MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:  40% 2.00G/4.95G [00:22<00:29, 101MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  41% 2.01G/4.95G [00:22<00:28, 101MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  41% 2.02G/4.95G [00:22<00:28, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  41% 2.03G/4.95G [00:22<00:28, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  41% 2.04G/4.95G [00:22<00:30, 95.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  42% 2.06G/4.95G [00:23<00:31, 92.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  42% 2.07G/4.95G [00:23<00:32, 88.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  42% 2.08G/4.95G [00:23<00:32, 87.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  42% 2.09G/4.95G [00:23<00:32, 87.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  42% 2.10G/4.95G [00:23<00:32, 87.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  43% 2.11G/4.95G [00:23<00:31, 89.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  43% 2.12G/4.95G [00:23<00:31, 89.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  43% 2.13G/4.95G [00:23<00:31, 89.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  43% 2.14G/4.95G [00:23<00:31, 89.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  43% 2.15G/4.95G [00:24<00:30, 90.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  44% 2.16G/4.95G [00:24<01:30, 30.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  44% 2.17G/4.95G [00:25<01:11, 38.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  44% 2.18G/4.95G [00:25<00:58, 47.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  44% 2.19G/4.95G [00:25<00:48, 56.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  45% 2.21G/4.95G [00:25<00:37, 72.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  45% 2.23G/4.95G [00:25<00:32, 83.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  46% 2.25G/4.95G [00:25<00:29, 90.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  46% 2.28G/4.95G [00:26<00:27, 95.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  46% 2.30G/4.95G [00:26<00:26, 99.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  47% 2.31G/4.95G [00:26<00:26, 98.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  47% 2.32G/4.95G [00:26<00:27, 97.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  47% 2.33G/4.95G [00:26<00:26, 98.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  47% 2.34G/4.95G [00:26<00:26, 97.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  47% 2.35G/4.95G [00:26<00:26, 97.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  48% 2.36G/4.95G [00:26<00:26, 98.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  48% 2.38G/4.95G [00:27<00:25, 101MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:  48% 2.39G/4.95G [00:27<00:25, 101MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  49% 2.41G/4.95G [00:27<00:24, 104MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  49% 2.43G/4.95G [00:27<00:24, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  49% 2.44G/4.95G [00:27<00:25, 96.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  50% 2.45G/4.95G [00:27<00:26, 95.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  50% 2.47G/4.95G [00:28<00:24, 99.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  50% 2.49G/4.95G [00:28<00:24, 99.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  51% 2.51G/4.95G [00:28<00:23, 102MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:  51% 2.52G/4.95G [00:28<00:23, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  51% 2.54G/4.95G [00:28<00:23, 104MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  52% 2.56G/4.95G [00:28<00:22, 106MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  52% 2.58G/4.95G [00:29<00:22, 106MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  53% 2.60G/4.95G [00:29<00:22, 106MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  53% 2.62G/4.95G [00:29<00:22, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  53% 2.63G/4.95G [00:29<00:22, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  53% 2.64G/4.95G [00:29<00:23, 99.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  54% 2.65G/4.95G [00:29<00:24, 94.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  54% 2.66G/4.95G [00:29<00:24, 92.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  54% 2.67G/4.95G [00:30<00:27, 81.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  54% 2.68G/4.95G [00:30<00:27, 83.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  55% 2.71G/4.95G [00:30<00:24, 92.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  55% 2.72G/4.95G [00:30<00:23, 94.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  55% 2.74G/4.95G [00:30<00:22, 99.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  56% 2.76G/4.95G [00:30<00:21, 102MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:  56% 2.78G/4.95G [00:31<00:20, 104MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  57% 2.80G/4.95G [00:31<00:20, 104MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  57% 2.81G/4.95G [00:31<00:21, 100MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  57% 2.82G/4.95G [00:31<00:21, 100MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  57% 2.83G/4.95G [00:31<00:21, 96.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  57% 2.84G/4.95G [00:31<00:22, 92.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  58% 2.85G/4.95G [00:31<00:22, 91.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  58% 2.86G/4.95G [00:32<00:23, 88.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  58% 2.87G/4.95G [00:32<00:22, 92.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  58% 2.88G/4.95G [00:32<00:22, 92.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  59% 2.89G/4.95G [00:32<00:23, 88.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  59% 2.92G/4.95G [00:32<00:21, 94.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  59% 2.93G/4.95G [00:32<00:21, 94.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  60% 2.95G/4.95G [00:32<00:20, 99.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  60% 2.97G/4.95G [00:33<00:19, 101MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:  60% 2.98G/4.95G [00:33<00:20, 98.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  60% 2.99G/4.95G [00:33<00:19, 99.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  61% 3.00G/4.95G [00:33<00:19, 99.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  61% 3.01G/4.95G [00:33<00:19, 98.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  61% 3.02G/4.95G [00:33<00:20, 95.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  61% 3.03G/4.95G [00:33<00:19, 96.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  61% 3.04G/4.95G [00:33<00:20, 93.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  62% 3.05G/4.95G [00:33<00:20, 94.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  62% 3.06G/4.95G [00:34<00:19, 95.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  62% 3.07G/4.95G [00:34<00:19, 95.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  62% 3.08G/4.95G [00:34<00:19, 93.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  63% 3.09G/4.95G [00:35<00:51, 35.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  63% 3.10G/4.95G [00:35<00:42, 43.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  63% 3.11G/4.95G [00:35<00:35, 51.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  63% 3.12G/4.95G [00:35<00:31, 57.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  63% 3.14G/4.95G [00:35<00:27, 66.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  64% 3.15G/4.95G [00:35<00:24, 73.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  64% 3.16G/4.95G [00:35<00:23, 77.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  64% 3.17G/4.95G [00:35<00:21, 83.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  64% 3.18G/4.95G [00:35<00:21, 82.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  65% 3.20G/4.95G [00:36<00:19, 89.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  65% 3.21G/4.95G [00:36<00:20, 84.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  65% 3.22G/4.95G [00:36<00:20, 84.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  65% 3.23G/4.95G [00:36<00:19, 86.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  66% 3.24G/4.95G [00:36<00:20, 84.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  66% 3.25G/4.95G [00:37<00:30, 56.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  66% 3.26G/4.95G [00:37<00:26, 63.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  66% 3.27G/4.95G [00:37<00:24, 68.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  66% 3.28G/4.95G [00:37<00:22, 72.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  67% 3.29G/4.95G [00:37<00:21, 77.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  67% 3.30G/4.95G [00:37<00:20, 78.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  67% 3.32G/4.95G [00:37<00:18, 90.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  68% 3.34G/4.95G [00:38<00:16, 96.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  68% 3.36G/4.95G [00:38<00:17, 92.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  68% 3.38G/4.95G [00:38<00:16, 97.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  68% 3.39G/4.95G [00:38<00:16, 96.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  69% 3.40G/4.95G [00:38<00:16, 95.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  69% 3.41G/4.95G [00:38<00:16, 93.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  69% 3.42G/4.95G [00:38<00:17, 88.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  69% 3.43G/4.95G [00:38<00:17, 87.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  70% 3.44G/4.95G [00:39<00:16, 91.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  70% 3.45G/4.95G [00:39<00:16, 93.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  70% 3.46G/4.95G [00:39<00:15, 93.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  70% 3.47G/4.95G [00:39<00:15, 95.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  70% 3.48G/4.95G [00:39<00:15, 97.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  71% 3.49G/4.95G [00:39<00:14, 97.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  71% 3.50G/4.95G [00:39<00:14, 97.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  71% 3.51G/4.95G [00:39<00:15, 90.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  71% 3.52G/4.95G [00:39<00:15, 90.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  71% 3.53G/4.95G [00:40<00:16, 86.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  72% 3.55G/4.95G [00:40<00:14, 94.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  72% 3.58G/4.95G [00:40<00:13, 99.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  73% 3.60G/4.95G [00:40<00:13, 101MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:  73% 3.61G/4.95G [00:40<00:13, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  73% 3.63G/4.95G [00:40<00:12, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  74% 3.65G/4.95G [00:41<00:12, 104MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  74% 3.66G/4.95G [00:41<00:12, 101MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  74% 3.67G/4.95G [00:41<00:12, 100MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  74% 3.68G/4.95G [00:41<00:12, 99.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  75% 3.69G/4.95G [00:41<00:12, 96.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  75% 3.70G/4.95G [00:41<00:12, 96.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  75% 3.71G/4.95G [00:41<00:12, 95.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  75% 3.72G/4.95G [00:41<00:13, 93.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  76% 3.74G/4.95G [00:42<00:12, 98.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  76% 3.76G/4.95G [00:42<00:11, 101MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:  77% 3.79G/4.95G [00:42<00:11, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  77% 3.80G/4.95G [00:42<00:11, 98.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  77% 3.81G/4.95G [00:42<00:12, 94.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  77% 3.82G/4.95G [00:42<00:11, 94.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  77% 3.83G/4.95G [00:43<00:11, 97.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  78% 3.84G/4.95G [00:43<00:11, 99.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  78% 3.85G/4.95G [00:43<00:11, 97.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  78% 3.86G/4.95G [00:43<00:12, 88.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  78% 3.87G/4.95G [00:43<00:12, 86.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  78% 3.88G/4.95G [00:43<00:12, 87.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  79% 3.89G/4.95G [00:43<00:11, 89.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  79% 3.90G/4.95G [00:43<00:11, 87.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  79% 3.91G/4.95G [00:43<00:11, 89.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  79% 3.92G/4.95G [00:44<00:11, 92.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  80% 3.93G/4.95G [00:44<00:11, 88.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  80% 3.94G/4.95G [00:44<00:11, 85.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  80% 3.95G/4.95G [00:44<00:11, 89.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  80% 3.96G/4.95G [00:44<00:10, 92.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  81% 3.98G/4.95G [00:44<00:09, 98.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  81% 4.01G/4.95G [00:44<00:09, 101MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:  81% 4.02G/4.95G [00:45<00:09, 101MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  82% 4.04G/4.95G [00:45<00:08, 104MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  82% 4.05G/4.95G [00:45<00:09, 99.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  82% 4.06G/4.95G [00:45<00:09, 96.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  82% 4.07G/4.95G [00:45<00:09, 94.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  82% 4.08G/4.95G [00:45<00:09, 90.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  83% 4.09G/4.95G [00:45<00:09, 90.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  83% 4.10G/4.95G [00:45<00:09, 89.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  83% 4.11G/4.95G [00:46<00:09, 90.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  83% 4.12G/4.95G [00:46<00:08, 93.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  84% 4.13G/4.95G [00:46<00:08, 96.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  84% 4.15G/4.95G [00:46<00:07, 102MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:  84% 4.17G/4.95G [00:46<00:07, 104MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  85% 4.19G/4.95G [00:46<00:07, 106MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  85% 4.20G/4.95G [00:46<00:07, 105MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  85% 4.22G/4.95G [00:47<00:08, 89.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  85% 4.23G/4.95G [00:49<00:44, 16.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  86% 4.24G/4.95G [00:50<00:56, 12.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  86% 4.25G/4.95G [00:50<00:44, 15.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  86% 4.26G/4.95G [00:51<00:34, 19.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  86% 4.27G/4.95G [00:51<00:26, 25.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  87% 4.28G/4.95G [00:51<00:21, 31.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  87% 4.30G/4.95G [00:51<00:13, 46.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  87% 4.32G/4.95G [00:51<00:10, 59.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  88% 4.34G/4.95G [00:52<00:08, 70.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  88% 4.35G/4.95G [00:52<00:07, 74.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  88% 4.36G/4.95G [00:52<00:07, 78.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  88% 4.37G/4.95G [00:52<00:07, 80.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  89% 4.38G/4.95G [00:52<00:06, 84.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  89% 4.39G/4.95G [00:52<00:06, 87.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  89% 4.40G/4.95G [00:52<00:06, 88.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  89% 4.41G/4.95G [00:52<00:05, 90.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  89% 4.42G/4.95G [00:52<00:05, 94.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  90% 4.45G/4.95G [00:53<00:05, 99.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  90% 4.47G/4.95G [00:53<00:04, 101MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:  91% 4.49G/4.95G [00:53<00:04, 103MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  91% 4.51G/4.95G [00:53<00:04, 105MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  92% 4.53G/4.95G [00:53<00:03, 105MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  92% 4.55G/4.95G [00:54<00:03, 102MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  92% 4.56G/4.95G [00:54<00:03, 98.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  92% 4.57G/4.95G [00:54<00:03, 96.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  93% 4.58G/4.95G [00:54<00:03, 97.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  93% 4.59G/4.95G [00:54<00:03, 95.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  93% 4.60G/4.95G [00:54<00:03, 96.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  93% 4.61G/4.95G [00:54<00:03, 96.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  94% 4.62G/4.95G [00:54<00:03, 98.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  94% 4.63G/4.95G [00:54<00:03, 96.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  94% 4.65G/4.95G [00:55<00:09, 31.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  94% 4.67G/4.95G [00:56<00:06, 46.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  95% 4.69G/4.95G [00:56<00:04, 59.2MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  95% 4.70G/4.95G [00:56<00:03, 65.3MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  95% 4.72G/4.95G [00:56<00:02, 77.0MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  96% 4.74G/4.95G [00:56<00:02, 84.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  96% 4.76G/4.95G [00:56<00:02, 89.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  96% 4.77G/4.95G [00:57<00:01, 89.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  97% 4.78G/4.95G [00:57<00:01, 91.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  97% 4.79G/4.95G [00:57<00:01, 92.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  97% 4.80G/4.95G [00:57<00:01, 94.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  97% 4.81G/4.95G [00:57<00:01, 93.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  98% 4.82G/4.95G [00:57<00:01, 94.8MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  98% 4.84G/4.95G [00:57<00:01, 99.9MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  98% 4.85G/4.95G [00:57<00:00, 97.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  98% 4.87G/4.95G [00:58<00:00, 94.6MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  99% 4.88G/4.95G [00:58<00:00, 96.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  99% 4.90G/4.95G [00:58<00:00, 101MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:  99% 4.92G/4.95G [00:58<00:00, 104MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors: 100% 4.93G/4.95G [00:58<00:00, 104MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors: 100% 4.95G/4.95G [00:59<00:00, 83.7MB/s]\n",
            "Downloading shards:  50% 1/2 [00:59<00:59, 59.34s/it]\n",
            "model-00002-of-00002.safetensors:   0% 0.00/67.1M [00:00<?, ?B/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  16% 10.5M/67.1M [00:00<00:00, 103MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  31% 21.0M/67.1M [00:00<00:00, 93.7MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  47% 31.5M/67.1M [00:00<00:00, 95.7MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  62% 41.9M/67.1M [00:00<00:00, 95.8MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  78% 52.4M/67.1M [00:00<00:00, 95.6MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors: 100% 67.1M/67.1M [00:00<00:00, 92.4MB/s]\n",
            "Downloading shards: 100% 2/2 [01:00<00:00, 30.12s/it]\n",
            "Loading checkpoint shards: 100% 2/2 [00:30<00:00, 15.06s/it]\n",
            "generation_config.json: 100% 137/137 [00:00<00:00, 723kB/s]\n",
            "tokenizer_config.json: 100% 1.11k/1.11k [00:00<00:00, 5.42MB/s]\n",
            "tokenizer.model: 100% 4.24M/4.24M [00:00<00:00, 15.7MB/s]\n",
            "special_tokens_map.json: 100% 555/555 [00:00<00:00, 3.25MB/s]\n",
            "tokenizer.json: 100% 17.5M/17.5M [00:00<00:00, 105MB/s]\n",
            "Use LoRA: LoraConfig(peft_type=<PeftType.LORA: 'LORA'>, auto_mapping=None, base_model_name_or_path=None, revision=None, task_type='CAUSAL_LM', inference_mode=False, r=8, target_modules={'q_proj', 'v_proj', 'k_proj', 'o_proj'}, lora_alpha=16, lora_dropout=0.05, fan_in_fan_out=False, bias='none', use_rslora=False, modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={}, megatron_config=None, megatron_core='megatron.core', loftq_config={}, use_dora=False, layer_replication=None)\n",
            "trainable params: 1,843,200 || all params: 2,508,015,616 || trainable%: 0.073492365368111\n",
            "WARNING:root:Loading data...\n",
            "Traceback (most recent call last):\n",
            "  File \"/content/GEMMA_ORCAMATH_FINETUNING/GEMMA_ORCAMATH_FINETUNING/train.py\", line 520, in <module>\n",
            "    train()\n",
            "  File \"/content/GEMMA_ORCAMATH_FINETUNING/GEMMA_ORCAMATH_FINETUNING/train.py\", line 505, in train\n",
            "    data_module = make_supervised_data_module(tokenizer=tokenizer, data_args=data_args)\n",
            "  File \"/content/GEMMA_ORCAMATH_FINETUNING/GEMMA_ORCAMATH_FINETUNING/train.py\", line 351, in make_supervised_data_module\n",
            "    train_dataset = SupervisedDataset(tokenizer=tokenizer, data_path=data_args.data_path)\n",
            "  File \"/content/GEMMA_ORCAMATH_FINETUNING/GEMMA_ORCAMATH_FINETUNING/train.py\", line 244, in __init__\n",
            "    list_data_dict = jload(data_path)\n",
            "  File \"/content/GEMMA_ORCAMATH_FINETUNING/GEMMA_ORCAMATH_FINETUNING/utils.py\", line 92, in jload\n",
            "    f = _make_r_io_base(f, mode)\n",
            "  File \"/content/GEMMA_ORCAMATH_FINETUNING/GEMMA_ORCAMATH_FINETUNING/utils.py\", line 49, in _make_r_io_base\n",
            "    f = open(f, mode=mode)\n",
            "FileNotFoundError: [Errno 2] No such file or directory: './data/formatted_orcamath.json'\n",
            "[2024-03-12 10:11:25,890] torch.distributed.elastic.multiprocessing.api: [ERROR] failed (exitcode: 1) local_rank: 0 (pid: 7369) of binary: /usr/bin/python3\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/bin/torchrun\", line 8, in <module>\n",
            "    sys.exit(main())\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py\", line 346, in wrapper\n",
            "    return f(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/run.py\", line 806, in main\n",
            "    run(args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/run.py\", line 797, in run\n",
            "    elastic_launch(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/launcher/api.py\", line 134, in __call__\n",
            "    return launch_agent(self._config, self._entrypoint, list(args))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/distributed/launcher/api.py\", line 264, in launch_agent\n",
            "    raise ChildFailedError(\n",
            "torch.distributed.elastic.multiprocessing.errors.ChildFailedError: \n",
            "============================================================\n",
            "train.py FAILED\n",
            "------------------------------------------------------------\n",
            "Failures:\n",
            "  <NO_OTHER_FAILURES>\n",
            "------------------------------------------------------------\n",
            "Root Cause (first observed failure):\n",
            "[0]:\n",
            "  time      : 2024-03-12_10:11:25\n",
            "  host      : a12c16fb36c0\n",
            "  rank      : 0 (local_rank: 0)\n",
            "  exitcode  : 1 (pid: 7369)\n",
            "  error_file: <N/A>\n",
            "  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html\n",
            "============================================================\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "To do inference with the model just run the following cell and compare results"
      ],
      "metadata": {
        "id": "aInOIWW-I-w7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
        "import safetensors.torch as torchst\n",
        "import torch\n",
        "from peft import PeftModel\n",
        "import torch\n",
        "import os\n",
        "from huggingface_hub import HfFolder\n",
        "\n",
        "# Check if the token is already saved\n",
        "token_path = HfFolder.path_token\n",
        "if not os.path.exists(token_path):\n",
        "    # If the token is not saved, ask for it and save it\n",
        "    access_token = input(\"Add your gemma HF access token: \")\n",
        "    HfFolder.save_token(access_token)  # Save the token for future use\n",
        "else:\n",
        "    print(\"HF access token is already saved.\")\n",
        "\n",
        "\n",
        "# Assuming 'checkpoint_path' is the path to your 'checkpoint-50010' directory\n",
        "checkpoint_path = \"./weights/e10_gemma_2b_qvko_r8_a16_lr5e-5_bs12/checkpoint-50010\"\n",
        "\n",
        "# Load the model from the Hugging Face Hub (without the adapter head)\n",
        "pretrained_model = AutoModelForCausalLM.from_pretrained(\n",
        "    \"google/gemma-2b\", # Load the model from the Hugging Face Hub (without the adapter head)\n",
        "    device_map=\"auto\", # Use the default device (GPU if available, CPU otherwise)\n",
        "    trust_remote_code=True,\n",
        ")\n",
        "\n",
        "# Load the tokenizer from the local directory\n",
        "tokenizer = AutoTokenizer.from_pretrained(checkpoint_path)\n",
        "\n",
        "\n",
        "# Load the adapter head from the local directory\n",
        "model = PeftModel.from_pretrained(pretrained_model, checkpoint_path)\n",
        "\n",
        "# Merge the adapter head into the model and unload it. This will make the model ready for inference.\n",
        "finetuned_model = model.merge_and_unload().to(\"cuda\")\n",
        "\n",
        "# Example of inference\n",
        "input_text = \"explain exponentiation to a child in demographic terms\"\n",
        "\n",
        "# Tokenize the input text and send it to the GPU\n",
        "input_ids = tokenizer(input_text, return_tensors=\"pt\").to(\"cuda\")\n",
        "\n",
        "# Generate the output of the pretrained model for the input text\n",
        "outputs = pretrained_model.generate(**input_ids, max_length=400, num_return_sequences=1)\n",
        "print(\"Pretrained model output:\")\n",
        "print(tokenizer.decode(outputs[0]))\n",
        "print(\"--------------------------------------\\n\")\n",
        "# Generate the output of the finetuned model for the input text\n",
        "print(\"Finetuned model output:\")\n",
        "outputs = finetuned_model.generate(**input_ids, max_length=400, num_return_sequences=1)\n",
        "print(tokenizer.decode(outputs[0]))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 444,
          "referenced_widgets": [
            "32422c5dc0574b6a9e3f0a9a6bb057d9",
            "71cac9ec2c914790a3f24a9ce6298aa2",
            "e198c6bd9fd0422d89c9ff2c6f7d18ec",
            "485d9b49a0814aceae800ceeb9467bdd",
            "93232175929547a099d59580136ed260",
            "563071914fea4a88b7e042805fa2296d",
            "d45336ac9dd54a18aac0b5871f012cca",
            "5999d89f68b4431cbf32c2d75ad3cace",
            "359486581f62496abfc9977b17c59c93",
            "31503e450cfc4ba4b6c8dc1b34c9533c",
            "e68952a5bbfa4358af79c869b5da218a"
          ]
        },
        "id": "14bUK-MpI-Pr",
        "outputId": "a0601d04-88b9-4775-f088-b3fcd3e87bc6"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "HF access token is already saved.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "32422c5dc0574b6a9e3f0a9a6bb057d9"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "You are trying to offload the whole model to the disk. Please use the `disk_offload` function instead.",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-26-333b065502b6>\u001b[0m in \u001b[0;36m<cell line: 23>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;31m# Load the model from the Hugging Face Hub (without the adapter head)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m pretrained_model = AutoModelForCausalLM.from_pretrained(\n\u001b[0m\u001b[1;32m     24\u001b[0m     \u001b[0;34m\"google/gemma-2b\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;31m# Load the model from the Hugging Face Hub (without the adapter head)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0mdevice_map\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"auto\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;31m# Use the default device (GPU if available, CPU otherwise)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/auto/auto_factory.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m    561\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model_mapping\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    562\u001b[0m             \u001b[0mmodel_class\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_get_model_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model_mapping\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 563\u001b[0;31m             return model_class.from_pretrained(\n\u001b[0m\u001b[1;32m    564\u001b[0m                 \u001b[0mpretrained_model_name_or_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mmodel_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mhub_kwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    565\u001b[0m             )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m   3565\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;34m\"skip_keys\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minspect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdispatch_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3566\u001b[0m                 \u001b[0mdevice_map_kwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"skip_keys\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_skip_keys_device_placement\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3567\u001b[0;31m             \u001b[0mdispatch_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mdevice_map_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3568\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3569\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mhf_quantizer\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/accelerate/big_modeling.py\u001b[0m in \u001b[0;36mdispatch_model\u001b[0;34m(model, device_map, main_device, state_dict, offload_dir, offload_index, offload_buffers, skip_keys, preload_module_classes, force_hooks)\u001b[0m\n\u001b[1;32m    474\u001b[0m             \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    475\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 476\u001b[0;31m             raise ValueError(\n\u001b[0m\u001b[1;32m    477\u001b[0m                 \u001b[0;34m\"You are trying to offload the whole model to the disk. Please use the `disk_offload` function instead.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    478\u001b[0m             )\n",
            "\u001b[0;31mValueError\u001b[0m: You are trying to offload the whole model to the disk. Please use the `disk_offload` function instead."
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "bY0tEA9WJpee"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}