diff --git "a/4\354\243\274\354\260\250 \354\204\270\354\205\230 \354\240\225\353\246\254.pdf" "b/4\354\243\274\354\260\250 \354\204\270\354\205\230 \354\240\225\353\246\254.pdf" new file mode 100644 index 0000000..51a1439 Binary files /dev/null and "b/4\354\243\274\354\260\250 \354\204\270\354\205\230 \354\240\225\353\246\254.pdf" differ diff --git a/BOAZ_base_transformer_yunseo.ipynb b/BOAZ_base_transformer_yunseo.ipynb new file mode 100644 index 0000000..d9b0db1 --- /dev/null +++ b/BOAZ_base_transformer_yunseo.ipynb @@ -0,0 +1,2423 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "804b6ce766214273acc5123f5fe35298": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_a4c41c602b0648f3a949999abb832656", + "IPY_MODEL_71baea8c0b804a24bb9cfbb9c5e14848", + "IPY_MODEL_a836170cd4294199999afe5d9e29844b" + ], + "layout": "IPY_MODEL_933efcc469d54abaa3e13ec562ecb82f" + } + }, + "a4c41c602b0648f3a949999abb832656": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_10e77e3e9d234b78a4781a4864e5cde2", + "placeholder": "​", + "style": "IPY_MODEL_3cb4647f5d734911baee034e4daa92a4", + "value": "Train 1: 100%" + } + }, + "71baea8c0b804a24bb9cfbb9c5e14848": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b54a566de8c24b03813fcf5b5a1fb3e6", + "max": 64, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_ed1efb4338b644718104455d5d7a328a", + "value": 64 + } + }, + "a836170cd4294199999afe5d9e29844b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1925402698a8454ea9a39ab8f2f74051", + "placeholder": "​", + "style": "IPY_MODEL_8d71903bee964efcb0e3376db0a24a5c", + "value": " 64/64 [05:21<00:00,  4.75s/it]" + } + }, + "933efcc469d54abaa3e13ec562ecb82f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "10e77e3e9d234b78a4781a4864e5cde2": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3cb4647f5d734911baee034e4daa92a4": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "b54a566de8c24b03813fcf5b5a1fb3e6": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ed1efb4338b644718104455d5d7a328a": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "1925402698a8454ea9a39ab8f2f74051": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8d71903bee964efcb0e3376db0a24a5c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4e157a6ebb5e4cc9be300e71d052cd21": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_86db40fc982e4455a890a3ee250fb337", + "IPY_MODEL_d9cc66a9378244269dcc46fbbda022f5", + "IPY_MODEL_80e28e0170844a9f9eb9f123ee4670bb" + ], + "layout": "IPY_MODEL_eac93e8e6d124765a7f72ebe6a272b3e" + } + }, + "86db40fc982e4455a890a3ee250fb337": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_daf794f81e9842538be4966ca15df99f", + "placeholder": "​", + "style": "IPY_MODEL_052268e785d3458fa78770d233931a50", + "value": "Train 2: 100%" + } + }, + "d9cc66a9378244269dcc46fbbda022f5": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f77a159b25e549e6ab193df25f728b18", + "max": 64, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_bac013322bfd40578f8775c13fb385f4", + "value": 64 + } + }, + "80e28e0170844a9f9eb9f123ee4670bb": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_61acf50ee75f4ec883f2560c09f91dee", + "placeholder": "​", + "style": "IPY_MODEL_18203cc20e5e4baf8c8ef9d4320d68ff", + "value": " 64/64 [05:13<00:00,  6.25s/it]" + } + }, + "eac93e8e6d124765a7f72ebe6a272b3e": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "daf794f81e9842538be4966ca15df99f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "052268e785d3458fa78770d233931a50": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "f77a159b25e549e6ab193df25f728b18": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "bac013322bfd40578f8775c13fb385f4": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "61acf50ee75f4ec883f2560c09f91dee": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "18203cc20e5e4baf8c8ef9d4320d68ff": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a397e9386fc1407a9502b6357c107390": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_d8b46f6c917143b09334a4da7db66aa6", + "IPY_MODEL_7e1cb3526d3940fd83467efeff101353", + "IPY_MODEL_e300a51d0b6147638db40e9b0fba1888" + ], + "layout": "IPY_MODEL_979f222f29eb493190cb882a26bdd66a" + } + }, + "d8b46f6c917143b09334a4da7db66aa6": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e5016eab18c5429a960338f294c7bc39", + "placeholder": "​", + "style": "IPY_MODEL_3d0e355691ed428bb646637df39eafd8", + "value": "Train 3: 100%" + } + }, + "7e1cb3526d3940fd83467efeff101353": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c66821fde8ab4ee5a4b68ceb8fb73998", + "max": 64, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_a2b3809cf0c749b6bfe2fb73fb161884", + "value": 64 + } + }, + "e300a51d0b6147638db40e9b0fba1888": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "model_module_version": "1.5.0", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_cf249fbb234747cab184ed6e75aee387", + "placeholder": "​", + "style": "IPY_MODEL_13e46b4d868d4dac9d4958e0ea0fc74c", + "value": " 64/64 [05:08<00:00,  4.94s/it]" + } + }, + "979f222f29eb493190cb882a26bdd66a": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e5016eab18c5429a960338f294c7bc39": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3d0e355691ed428bb646637df39eafd8": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c66821fde8ab4ee5a4b68ceb8fb73998": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a2b3809cf0c749b6bfe2fb73fb161884": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "cf249fbb234747cab184ed6e75aee387": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "model_module_version": "1.2.0", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "13e46b4d868d4dac9d4958e0ea0fc74c": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "model_module_version": "1.5.0", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + } + } + } + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "## Transformer NMT (English → French) Experiment \n", + "\n", + "이번 복습과제에서는 Transformer Encoder–Decoder 구조를 직접 구현하여 영어 문장을 프랑스어로 번역하는 실험을 진행합니다.\n", + "\n", + "- ManyThings(Anki) 공개 번역 데이터를 활용하여, 데이터 정제 → 토크나이징 → 패딩 → 모델 학습 → 추론까지 전 과정을 포함하고 있습니다.\n", + "\n", + "- Positional Encoding, Padding Mask, Look-Ahead Mask, Multi-Head Attention, Feed-Forward Network 등 Transformer의 핵심 구성요소를 코드로 확인합니다.\n", + "\n", + "- Teacher Forcing 기반 학습을 통해, 디코더가 다음 단어를 예측하도록 학습되는 과정을 관찰합니다.\n", + "\n", + "- 학습된 모델로 임의의 영어 문장을 입력해 Autoregressive 방식으로 번역을 생성해봅니다.\n", + "\n", + "- 중간 출력(텐서 shape, 토큰 id, 마스크 형태 등)을 확인하며, Transformer 내부 흐름을 이해하는 데 초점을 둡니다." + ], + "metadata": { + "id": "9Jv1A9qC85tj" + } + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "xl7WDwjj4Qhv" + }, + "outputs": [], + "source": [ + "#1 Imports + Seed + Utils\n", + "import os\n", + "import torch.optim as optim\n", + "import random\n", + "import math\n", + "import time\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import tensorflow as tf\n", + "\n", + "from tqdm import tqdm, tqdm_notebook, trange\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "import unicodedata\n", + "import re\n", + "\n", + "from tensorflow.keras.preprocessing.sequence import pad_sequences\n", + "from tensorflow.keras.preprocessing.text import Tokenizer\n", + "\n", + "import urllib3\n", + "import zipfile\n", + "import shutil\n", + "import pandas as pd\n", + "\n", + "pd.set_option('display.max_colwidth', None)\n", + "\n", + "# 시드 고정\n", + "SEED = 1234\n", + "random.seed(SEED)\n", + "np.random.seed(SEED)\n", + "torch.manual_seed(SEED)\n", + "torch.cuda.manual_seed(SEED)\n", + "\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "\n", + "ENCODER_LEN = 40 # 입력·출력 문장 토큰 길이를 40으로 고정\n", + "DECODER_LEN = ENCODER_LEN\n", + "BATCH_SIZE = 128 # 한 step에서 학습하는 문장쌍 개수\n", + "N_EPOCHS = 3 # 전체 데이터셋을 반복 학습하는 횟수 (*로컬 환경에서는 모델 학습에 장시간 소요되므로, 줄이는 것을 추천!)\n", + "\n", + "# 디버깅 함수\n", + "def show_shape(name, x):\n", + " if isinstance(x, torch.Tensor):\n", + " print(f\"{name}: shape={tuple(x.shape)}, dtype={x.dtype}, device={x.device}\")\n", + " else:\n", + " print(f\"{name}: type={type(x)}\")\n", + "\n", + "def show_tokens(tokenizer, seq, n=20):\n", + " seq = list(seq)\n", + " print(\"ids :\", seq[:n])\n", + " print(\"text :\", tokenizer.sequences_to_texts([seq[:n]])[0])\n", + "\n", + "def count_pad(x, pad_id=0):\n", + " return int((x == pad_id).sum().item()), x.numel()\n", + "\n", + "DEBUG_ONCE = {\"attn\": True}" + ] + }, + { + "cell_type": "code", + "source": [ + "#2 번역 데이터셋 다운로드 및 로드 (English-French Parallel Corpus)\n", + "!rm -f fra-eng.zip fra.txt\n", + "!wget -q https://www.manythings.org/anki/fra-eng.zip\n", + "!unzip -q fra-eng.zip\n", + "\n", + "import pandas as pd\n", + "\n", + "train_df = pd.read_csv('fra.txt', names=['SRC', 'TRG', 'lic'], sep='\\t')\n", + "del train_df['lic']\n", + "\n", + "print(\"영어-프랑스어 번역 쌍 총 개수:\", len(train_df))\n", + "train_df = train_df.loc[:, 'SRC':'TRG']\n", + "train_df.head()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 225 + }, + "id": "SW-HV8uI6EMr", + "outputId": "09860d54-c7c7-449c-82de-f1f0bf03273d" + }, + "execution_count": 3, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "영어-프랑스어 번역 쌍 총 개수: 239189\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " SRC TRG\n", + "0 Go. Va !\n", + "1 Go. Marche.\n", + "2 Go. En route !\n", + "3 Go. Bouge !\n", + "4 Hi. Salut !" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
SRCTRG
0Go.Va !
1Go.Marche.
2Go.En route !
3Go.Bouge !
4Hi.Salut !
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "variable_name": "train_df" + } + }, + "metadata": {}, + "execution_count": 3 + } + ] + }, + { + "cell_type": "code", + "source": [ + "# Length stats + Dedup + Filter + Sample\n", + "\n", + "train_df[\"src_len\"] = train_df[\"SRC\"].astype(str).str.split().str.len()\n", + "train_df[\"trg_len\"] = train_df[\"TRG\"].astype(str).str.split().str.len()\n", + "\n", + "print('Translation Pair :', len(train_df))\n", + "\n", + "train_df = train_df.drop_duplicates(subset=[\"SRC\"])\n", + "print('After SRC dedup :', len(train_df))\n", + "\n", + "train_df = train_df.drop_duplicates(subset=[\"TRG\"])\n", + "print('After TRG dedup :', len(train_df))\n", + "\n", + "is_within_len = (\n", + " (8 < train_df[\"src_len\"]) & (train_df[\"src_len\"] < 20) &\n", + " (8 < train_df[\"trg_len\"]) & (train_df[\"trg_len\"] < 20)\n", + ")\n", + "train_df = train_df[is_within_len]\n", + "\n", + "dataset_df_8096 = train_df.sample(n=1024*8, random_state=1234)\n", + "print('Sampled Pair :', len(dataset_df_8096))" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "dSuwsvZm4qeC", + "outputId": "99501903-8626-4b26-834b-b61d4c307608" + }, + "execution_count": 4, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Translation Pair : 239189\n", + "After SRC dedup : 171971\n", + "After TRG dedup : 153441\n", + "Sampled Pair : 8192\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# Preprocess (raw_src/raw_trg) + Sanity Print\n", + "raw_src = []\n", + "for sentence in dataset_df_8096['SRC']:\n", + " sentence = sentence.lower().strip()\n", + " sentence = re.sub(r\"([?.!,])\", r\" \\1 \", sentence)\n", + " sentence = re.sub(r'[\" \"]+', \" \", sentence)\n", + "\n", + " sentence = re.sub(r\"i'm\", \"i am\", sentence)\n", + " sentence = re.sub(r\"he's\", \"he is\", sentence)\n", + " sentence = re.sub(r\"she's\", \"she is\", sentence)\n", + " sentence = re.sub(r\"it's\", \"it is\", sentence)\n", + " sentence = re.sub(r\"that's\", \"that is\", sentence)\n", + " sentence = re.sub(r\"what's\", \"that is\", sentence)\n", + " sentence = re.sub(r\"where's\", \"where is\", sentence)\n", + " sentence = re.sub(r\"how's\", \"how is\", sentence)\n", + " sentence = re.sub(r\"\\'ll\", \" will\", sentence)\n", + " sentence = re.sub(r\"\\'ve\", \" have\", sentence)\n", + " sentence = re.sub(r\"\\'re\", \" are\", sentence)\n", + " sentence = re.sub(r\"\\'d\", \" would\", sentence)\n", + " sentence = re.sub(r\"\\'re\", \" are\", sentence)\n", + " sentence = re.sub(r\"won't\", \"will not\", sentence)\n", + " sentence = re.sub(r\"can't\", \"cannot\", sentence)\n", + " sentence = re.sub(r\"n't\", \" not\", sentence)\n", + " sentence = re.sub(r\"n'\", \"ng\", sentence)\n", + " sentence = re.sub(r\"'bout\", \"about\", sentence)\n", + "\n", + " sentence = re.sub(r\"[^a-zA-Z?.!,]+\", \" \", sentence)\n", + " sentence = sentence.strip()\n", + " raw_src.append(sentence)\n", + "\n", + "raw_trg = []\n", + "def unicode_to_ascii(s):\n", + " return ''.join(c for c in unicodedata.normalize('NFD', s)\n", + " if unicodedata.category(c) != 'Mn')\n", + "\n", + "for sentence in dataset_df_8096['TRG']:\n", + " sentence = unicode_to_ascii(sentence.lower())\n", + " sentence = re.sub(r\"([?.!,¿])\", r\" \\1\", sentence)\n", + " sentence = re.sub(r\"[^a-zA-Z!.?]+\", r\" \", sentence)\n", + " sentence = re.sub(r\"\\s+\", \" \", sentence)\n", + " raw_trg.append(sentence)\n", + "\n", + "print(\"=== Preprocess sanity check (top-3) ===\")\n", + "for i in [0, 1, 2]:\n", + " print(f\"\\n[{i}] SRC_raw :\", dataset_df_8096['SRC'].iloc[i])\n", + " print(f\"[{i}] SRC_clean:\", raw_src[i])\n", + " print(f\"[{i}] TRG_raw :\", dataset_df_8096['TRG'].iloc[i])\n", + " print(f\"[{i}] TRG_clean:\", raw_trg[i])" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "siF93PVf4vU2", + "outputId": "09b4a0e8-5703-4167-f528-40251df0115a" + }, + "execution_count": 5, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "=== Preprocess sanity check (top-3) ===\n", + "\n", + "[0] SRC_raw : Was there a dead body in the room? \"No, there was no body there.\"\n", + "[0] SRC_clean: was there a dead body in the room ? no , there was no body there .\n", + "[0] TRG_raw : « Y avait-il un cadavre dans la chambre ? » « Non, il n'y avait pas de cadavre. »\n", + "[0] TRG_clean: y avait il un cadavre dans la chambre ? non il n y avait pas de cadavre . \n", + "\n", + "[1] SRC_raw : Our teacher doesn't just speak English, but French too.\n", + "[1] SRC_clean: our teacher does not just speak english , but french too .\n", + "[1] TRG_raw : Notre professeur ne parle pas juste anglais mais aussi français.\n", + "[1] TRG_clean: notre professeur ne parle pas juste anglais mais aussi francais .\n", + "\n", + "[2] SRC_raw : I don't know whether he is dead or alive.\n", + "[2] SRC_clean: i do not know whether he is dead or alive .\n", + "[2] TRG_raw : Je ne sais pas s'il est mort ou vivant.\n", + "[2] TRG_clean: je ne sais pas s il est mort ou vivant .\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "### Q1. / 토큰의 역할\n", + "이 코드에서는 입력/출력 문장에 ``, `` 토큰을 직접 붙였습니다.\n", + "\n", + "1) ``와 ``는 각각 언제/왜 필요한가요? \n", + " : 디코더가 문장 생성을 시작하는 것을 알리는 토큰 -> 디코더는 를 첫 입력으로 받은 후 단어들을 순차적으로 생성함.\n", + " : 디코더가 문장 생성을 종료해야 할 시점을 판단하기 위해 사용하는 토큰" + ], + "metadata": { + "id": "L0yx1hKo6ud5" + } + }, + { + "cell_type": "code", + "source": [ + "# Build train_df + Add SOS/EOS + Tokenizer + Vocab Print\n", + "df1 = pd.DataFrame(raw_src)\n", + "df2 = pd.DataFrame(raw_trg)\n", + "df1.rename(columns={0: \"SRC\"}, errors=\"raise\", inplace=True)\n", + "df2.rename(columns={0: \"TRG\"}, errors=\"raise\", inplace=True)\n", + "train_df = pd.concat([df1, df2], axis=1)\n", + "print('Translation Pair :',len(train_df))\n", + "\n", + "raw_src = train_df['SRC']\n", + "raw_trg = train_df['TRG']\n", + "\n", + "src_sentence = raw_src.apply(lambda x: \" \" + str(x) + \" \")\n", + "trg_sentence = raw_trg.apply(lambda x: \" \"+ x + \" \")\n", + "\n", + "print(\"=== SOS/EOS check ===\")\n", + "print(\"SRC example:\", src_sentence.iloc[0])\n", + "print(\"TRG example:\", trg_sentence.iloc[0])\n", + "\n", + "filters = '!\"#$%&()*+,-./:;=?@[\\\\]^_`{|}~\\t\\n'\n", + "oov_token = ''\n", + "\n", + "SRC_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters=filters, oov_token=oov_token)\n", + "TRG_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters=filters, oov_token=oov_token)\n", + "\n", + "SRC_tokenizer.fit_on_texts(src_sentence)\n", + "TRG_tokenizer.fit_on_texts(trg_sentence)\n", + "\n", + "n_enc_vocab = len(SRC_tokenizer.word_index) + 1\n", + "n_dec_vocab = len(TRG_tokenizer.word_index) + 1\n", + "\n", + "print('Encoder vocab size :', n_enc_vocab)\n", + "print('Decoder vocab size :', n_dec_vocab)\n", + "\n", + "print(\"\\n=== Tokenizer vocab check ===\")\n", + "print(\"SRC '' id:\", SRC_tokenizer.word_index.get(\"\"))\n", + "print(\"SRC '' id:\", SRC_tokenizer.word_index.get(\"\"))\n", + "print(\"TRG '' id:\", TRG_tokenizer.word_index.get(\"\"))\n", + "print(\"TRG '' id:\", TRG_tokenizer.word_index.get(\"\"))\n", + "print(\"TRG '' id:\", TRG_tokenizer.word_index.get(\"\"))\n", + "\n", + "print(\"\\nSRC top-10:\", list(SRC_tokenizer.word_index.items())[:10])\n", + "print(\"TRG top-10:\", list(TRG_tokenizer.word_index.items())[:10])\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "bv61caIm4yng", + "outputId": "80812071-cd67-4d54-d15e-b16ede7e065f" + }, + "execution_count": 6, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Translation Pair : 8192\n", + "=== SOS/EOS check ===\n", + "SRC example: was there a dead body in the room ? no , there was no body there . \n", + "TRG example: y avait il un cadavre dans la chambre ? non il n y avait pas de cadavre . \n", + "Encoder vocab size : 5924\n", + "Decoder vocab size : 7773\n", + "\n", + "=== Tokenizer vocab check ===\n", + "SRC '' id: 2\n", + "SRC '' id: 3\n", + "TRG '' id: 2\n", + "TRG '' id: 3\n", + "TRG '' id: 1\n", + "\n", + "SRC top-10: [('', 1), ('', 2), ('', 3), ('the', 4), ('to', 5), ('i', 6), ('you', 7), ('a', 8), ('not', 9), ('is', 10)]\n", + "TRG top-10: [('', 1), ('', 2), ('', 3), ('de', 4), ('a', 5), ('je', 6), ('que', 7), ('la', 8), ('le', 9), ('pas', 10)]\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# Tokenization Demo + Pad + Decode Check\n", + "lines = [\n", + " \"It is winter and the weather is very cold.\",\n", + " \"Will this Christmas be a white Christmas?\",\n", + " \"Be careful not to catch a cold in winter and have a happy new year.\"\n", + "]\n", + "print(\"=== SRC tokenizer demo ===\")\n", + "for line in lines:\n", + " txt_2_ids = SRC_tokenizer.texts_to_sequences([line])\n", + " ids_2_txt = SRC_tokenizer.sequences_to_texts(txt_2_ids)\n", + " print(\"Input :\", line)\n", + " print(\"txt_2_ids :\", txt_2_ids)\n", + " print(\"ids_2_txt :\", ids_2_txt[0],\"\\n\")\n", + "\n", + "lines = [\n", + " \"C'est l'hiver et il fait très froid.\",\n", + " \"Ce Noël sera-t-il un Noël blanc ?\",\n", + " \"Attention à ne pas attraper froid en hiver et bonne année.\"\n", + "]\n", + "print(\"=== TRG tokenizer demo ===\")\n", + "for line in lines:\n", + " txt_2_ids = TRG_tokenizer.texts_to_sequences([line])\n", + " ids_2_txt = TRG_tokenizer.sequences_to_texts(txt_2_ids)\n", + " print(\"Input :\", line)\n", + " print(\"txt_2_ids :\", txt_2_ids)\n", + " print(\"ids_2_txt :\", ids_2_txt[0],\"\\n\")\n", + "\n", + "tokenized_inputs = SRC_tokenizer.texts_to_sequences(src_sentence)\n", + "tokenized_outputs = TRG_tokenizer.texts_to_sequences(trg_sentence)\n", + "\n", + "tkn_sources = tf.keras.preprocessing.sequence.pad_sequences(\n", + " tokenized_inputs, maxlen=ENCODER_LEN, padding='post', truncating='post'\n", + ")\n", + "tkn_targets = tf.keras.preprocessing.sequence.pad_sequences(\n", + " tokenized_outputs, maxlen=DECODER_LEN, padding='post', truncating='post'\n", + ")\n", + "\n", + "print(\"=== Padding check ===\")\n", + "print(\"tkn_sources[0][:30]:\", tkn_sources[0][:30])\n", + "print(\"tkn_targets[0][:30]:\", tkn_targets[0][:30])\n", + "\n", + "print(\"\\nSRC decoded (first 30):\")\n", + "show_tokens(SRC_tokenizer, tkn_sources[0], n=30)\n", + "print(\"TRG decoded (first 30):\")\n", + "show_tokens(TRG_tokenizer, tkn_targets[0], n=30)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "KULY2MOO42bM", + "outputId": "77693d44-f426-4347-90b9-c3be6d399ea4" + }, + "execution_count": 7, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "=== SRC tokenizer demo ===\n", + "Input : It is winter and the weather is very cold.\n", + "txt_2_ids : [[15, 10, 926, 23, 4, 458, 10, 96, 304]]\n", + "ids_2_txt : it is winter and the weather is very cold \n", + "\n", + "Input : Will this Christmas be a white Christmas?\n", + "txt_2_ids : [[28, 25, 138, 26, 8, 466, 138]]\n", + "ids_2_txt : will this christmas be a white christmas \n", + "\n", + "Input : Be careful not to catch a cold in winter and have a happy new year.\n", + "txt_2_ids : [[26, 575, 9, 5, 372, 8, 304, 14, 926, 23, 17, 8, 319, 110, 179]]\n", + "ids_2_txt : be careful not to catch a cold in winter and have a happy new year \n", + "\n", + "=== TRG tokenizer demo ===\n", + "Input : C'est l'hiver et il fait très froid.\n", + "txt_2_ids : [[1, 1, 30, 11, 55, 1, 451]]\n", + "ids_2_txt : et il fait froid \n", + "\n", + "Input : Ce Noël sera-t-il un Noël blanc ?\n", + "txt_2_ids : [[17, 1, 245, 70, 11, 14, 1, 691]]\n", + "ids_2_txt : ce sera t il un blanc \n", + "\n", + "Input : Attention à ne pas attraper froid en hiver et bonne année.\n", + "txt_2_ids : [[740, 1, 13, 10, 666, 451, 24, 1001, 30, 190, 1]]\n", + "ids_2_txt : attention ne pas attraper froid en hiver et bonne \n", + "\n", + "=== Padding check ===\n", + "tkn_sources[0][:30]: [ 2 19 50 8 561 706 14 4 167 67 50 19 67 706 50 3 0 0\n", + " 0 0 0 0 0 0 0 0 0 0 0 0]\n", + "tkn_targets[0][:30]: [ 2 45 73 11 14 1665 31 8 324 199 11 21 45 73\n", + " 10 4 1665 3 0 0 0 0 0 0 0 0 0 0\n", + " 0 0]\n", + "\n", + "SRC decoded (first 30):\n", + "ids : [np.int32(2), np.int32(19), np.int32(50), np.int32(8), np.int32(561), np.int32(706), np.int32(14), np.int32(4), np.int32(167), np.int32(67), np.int32(50), np.int32(19), np.int32(67), np.int32(706), np.int32(50), np.int32(3), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0)]\n", + "text : was there a dead body in the room no there was no body there \n", + "TRG decoded (first 30):\n", + "ids : [np.int32(2), np.int32(45), np.int32(73), np.int32(11), np.int32(14), np.int32(1665), np.int32(31), np.int32(8), np.int32(324), np.int32(199), np.int32(11), np.int32(21), np.int32(45), np.int32(73), np.int32(10), np.int32(4), np.int32(1665), np.int32(3), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0), np.int32(0)]\n", + "text : y avait il un cadavre dans la chambre non il n y avait pas de cadavre \n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# Torch tensors + DataLoader + Batch Shape Check\n", + "tensors_src = torch.tensor(tkn_sources).to(device)\n", + "tensors_trg = torch.tensor(tkn_targets).to(device)\n", + "\n", + "from torch.utils.data import TensorDataset, DataLoader\n", + "\n", + "dataset = TensorDataset(tensors_src, tensors_trg)\n", + "dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True)\n", + "\n", + "print(\"=== One batch shape check ===\")\n", + "src_batch, trg_batch = next(iter(dataloader))\n", + "show_shape(\"src_batch\", src_batch)\n", + "show_shape(\"trg_batch\", trg_batch)\n", + "pad_cnt, total = count_pad(src_batch, pad_id=0)\n", + "print(f\"SRC PAD ratio: {pad_cnt}/{total} = {pad_cnt/total:.3f}\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "kII7o1iq44e_", + "outputId": "8bfc316f-fca6-4b77-db41-8f97ede61929" + }, + "execution_count": 8, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "=== One batch shape check ===\n", + "src_batch: shape=(128, 40), dtype=torch.int32, device=cpu\n", + "trg_batch: shape=(128, 40), dtype=torch.int32, device=cpu\n", + "SRC PAD ratio: 3415/5120 = 0.667\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "### Q2. Positional Encoding\n", + "\n", + "1) Positional Encoding을 하는 이유가 무엇인가요?\n", + "Transformer는 단어들을 동시에 처리하므로 앞에서 등장한 단어인지 뒤에서 등장한 단어인지 구분하지 못함 -> 따라서, 각 토큰의 위치 정보를 임베딩에 추가로 제공해야 함." + ], + "metadata": { + "id": "t8lmXdL17cis" + } + }, + { + "cell_type": "code", + "source": [ + "# Hyperparams + Positional Encoding + Mask Functions + Mask Debug\n", + "n_layers = 2\n", + "hid_dim = 256\n", + "pf_dim = 1024\n", + "n_heads = 8\n", + "dropout = 0.3\n", + "pe_source = 512\n", + "pe_target = 512\n", + "layer_norm_epsilon = 1e-12\n", + "\n", + "class get_sinusoid_encoding_table(nn.Module):\n", + " def __init__(self, position, hid_dim):\n", + " super().__init__()\n", + " self.hid_dim = hid_dim\n", + " pe = torch.zeros(position, hid_dim).to(device)\n", + "\n", + " for pos in range(position):\n", + " for i in range(0, hid_dim, 2):\n", + " pe[pos, i] = math.sin(pos / (10000 ** ((2 * i)/hid_dim)))\n", + " pe[pos, i + 1] = math.cos(pos / (10000 ** ((2 * i)/hid_dim)))\n", + "\n", + " self.pe = pe.unsqueeze(0)\n", + " self.pe.requires_grad = False\n", + "\n", + " def forward(self, x):\n", + " ret = math.sqrt(self.hid_dim)*x + self.pe[:, :x.size(1)]\n", + " return ret\n", + "\n", + "def create_padding_mask(x):\n", + " input_pad = 0\n", + " mask = (x == input_pad).float()\n", + " mask = mask.unsqueeze(1).unsqueeze(1)\n", + " return mask\n", + "\n", + "def create_look_ahead_mask(seq):\n", + " seq_len = seq.shape[1]\n", + " look_ahead_mask = torch.ones(seq_len, seq_len)\n", + " look_ahead_mask = torch.triu(look_ahead_mask, diagonal=1).to(device)\n", + " return look_ahead_mask\n", + "\n", + "print(\"=== Positional Encoding sanity ===\")\n", + "x = torch.zeros(2, 5, hid_dim).to(device)\n", + "pe = get_sinusoid_encoding_table(10, hid_dim).to(device)\n", + "y = pe(x)\n", + "show_shape(\"x\", x)\n", + "show_shape(\"y\", y)\n", + "print(\"y[0,0,0:6] =\", y[0,0,:6].detach().cpu().numpy())\n", + "print(\"y[0,1,0:6] =\", y[0,1,:6].detach().cpu().numpy(), \"(different position)\")\n", + "\n", + "print(\"\\n=== Mask check ===\")\n", + "src_batch, trg_batch = next(iter(dataloader))\n", + "enc_pad = create_padding_mask(src_batch)\n", + "look = create_look_ahead_mask(trg_batch)\n", + "dec_pad = create_padding_mask(trg_batch)\n", + "combined = torch.maximum(dec_pad, look)\n", + "\n", + "show_shape(\"enc_padding_mask\", enc_pad)\n", + "show_shape(\"look_ahead_mask\", look)\n", + "show_shape(\"dec_target_padding_mask\", dec_pad)\n", + "show_shape(\"combined_mask\", combined)\n", + "print(\"\\nlook_ahead_mask[0:8,0:8]:\\n\", look[:8,:8].detach().cpu().numpy())" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "yZw-ECKG47sJ", + "outputId": "66489fb0-e1eb-4bac-8ffa-ed33a834933f" + }, + "execution_count": 9, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "=== Positional Encoding sanity ===\n", + "x: shape=(2, 5, 256), dtype=torch.float32, device=cpu\n", + "y: shape=(2, 5, 256), dtype=torch.float32, device=cpu\n", + "y[0,0,0:6] = [0. 1. 0. 1. 0. 1.]\n", + "y[0,1,0:6] = [0.84147096 0.5403023 0.7617204 0.6479059 0.68156135 0.731761 ] (different position)\n", + "\n", + "=== Mask check ===\n", + "enc_padding_mask: shape=(128, 1, 1, 40), dtype=torch.float32, device=cpu\n", + "look_ahead_mask: shape=(40, 40), dtype=torch.float32, device=cpu\n", + "dec_target_padding_mask: shape=(128, 1, 1, 40), dtype=torch.float32, device=cpu\n", + "combined_mask: shape=(128, 1, 40, 40), dtype=torch.float32, device=cpu\n", + "\n", + "look_ahead_mask[0:8,0:8]:\n", + " [[0. 1. 1. 1. 1. 1. 1. 1.]\n", + " [0. 0. 1. 1. 1. 1. 1. 1.]\n", + " [0. 0. 0. 1. 1. 1. 1. 1.]\n", + " [0. 0. 0. 0. 1. 1. 1. 1.]\n", + " [0. 0. 0. 0. 0. 1. 1. 1.]\n", + " [0. 0. 0. 0. 0. 0. 1. 1.]\n", + " [0. 0. 0. 0. 0. 0. 0. 1.]\n", + " [0. 0. 0. 0. 0. 0. 0. 0.]]\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "### Q3. look-ahead mask의 의미(Decoder self-attention)\n", + "1) look-ahead mask는 어떤 정보를 “가리기” 위해 존재하나요? \n", + "디코더는 문장을 왼쪽에서 오른쪽으로 한 토큰씩 생성함.\n", + "-> 하지만 self-attention은 모든 위치를 동시에 참조할 수 있기 때문에 mask가 없다면 생성되지 않은 미래의 토큰을 미리 보게되는 문제가 발생함.\n", + "-> 따라서, 디코더가 미래 정보를 가려서 현재까지 생성된 토큰들만을 기반으로 다음 토큰을 예측하게 하기 위해 look-ahead mask를 사용함." + ], + "metadata": { + "id": "G8yXCv_E7184" + } + }, + { + "cell_type": "code", + "source": [ + "# Attention + MHA + Encoder/Decoder/Transformer (with 1-time debug print)\n", + "class ScaledDotProductAttention(nn.Module):\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.dropout = nn.Dropout(0.3)\n", + "\n", + " def forward(self, query, key, value, mask):\n", + " if DEBUG_ONCE[\"attn\"]:\n", + " print(\"=== [DEBUG] ScaledDotProductAttention ===\")\n", + " show_shape(\"query\", query)\n", + " show_shape(\"key\", key)\n", + " show_shape(\"value\", value)\n", + " if mask is not None: show_shape(\"mask\", mask)\n", + "\n", + " matmul_qk = torch.matmul(query, torch.transpose(key,2,3))\n", + " dk = key.shape[-1]\n", + " scaled_attention_logits = matmul_qk / math.sqrt(dk)\n", + "\n", + " if mask is not None:\n", + " scaled_attention_logits += (mask * -1e9)\n", + "\n", + " attention_weights = F.softmax(scaled_attention_logits, dim=-1)\n", + " output = torch.matmul(attention_weights, value)\n", + "\n", + " if DEBUG_ONCE[\"attn\"]:\n", + " show_shape(\"attention_weights\", attention_weights)\n", + " show_shape(\"output\", output)\n", + " print(\"attn_weights sample (head0, q0, first 8):\",\n", + " attention_weights[0,0,0,:8].detach().cpu().numpy())\n", + " DEBUG_ONCE[\"attn\"] = False\n", + "\n", + " return output, attention_weights\n", + "\n", + "class MultiHeadAttentionLayer(nn.Module):\n", + " def __init__(self, hid_dim, n_heads):\n", + " super(MultiHeadAttentionLayer, self).__init__()\n", + " self.n_heads = n_heads\n", + " assert hid_dim % self.n_heads == 0\n", + " self.hid_dim = hid_dim\n", + " self.depth = int(hid_dim/self.n_heads)\n", + "\n", + " self.q_linear = nn.Linear(hid_dim, hid_dim)\n", + " self.k_linear = nn.Linear(hid_dim, hid_dim)\n", + " self.v_linear = nn.Linear(hid_dim, hid_dim)\n", + "\n", + " self.scaled_dot_attn = ScaledDotProductAttention()\n", + " self.out = nn.Linear(hid_dim, hid_dim)\n", + "\n", + " def split_heads(self, inputs, batch_size):\n", + " inputs = torch.reshape(inputs, (batch_size, -1, self.n_heads, self.depth))\n", + " return torch.transpose(inputs, 1,2)\n", + "\n", + " def forward(self, inputs):\n", + " query, key, value, mask = inputs['query'], inputs['key'], inputs['value'], inputs['mask']\n", + " batch_size = query.shape[0]\n", + "\n", + " query = self.q_linear(query)\n", + " key = self.k_linear(key)\n", + " value = self.v_linear(value)\n", + "\n", + " query = self.split_heads(query, batch_size)\n", + " key = self.split_heads(key, batch_size)\n", + " value = self.split_heads(value, batch_size)\n", + "\n", + " scaled_attention, _ = self.scaled_dot_attn(query, key, value, mask)\n", + "\n", + " scaled_attention = torch.transpose(scaled_attention, 1,2)\n", + " concat_attention = torch.reshape(scaled_attention, (batch_size, -1, self.hid_dim))\n", + " outputs = self.out(concat_attention)\n", + " return outputs\n", + "\n", + "class PositionwiseFeedforwardLayer(nn.Module):\n", + " def __init__(self, hid_dim, pf_dim):\n", + " super(PositionwiseFeedforwardLayer, self).__init__()\n", + " self.linear_1 = nn.Linear(hid_dim, pf_dim)\n", + " self.linear_2 = nn.Linear(pf_dim, hid_dim)\n", + "\n", + " def forward(self, attention):\n", + " output = self.linear_1(attention)\n", + " output = F.relu(output)\n", + " output = self.linear_2(output)\n", + " return output\n", + "\n", + "class EncoderLayer(nn.Module):\n", + " def __init__(self):\n", + " super(EncoderLayer, self).__init__()\n", + " self.attn = MultiHeadAttentionLayer(hid_dim, n_heads)\n", + " self.ffn = PositionwiseFeedforwardLayer(hid_dim, pf_dim)\n", + "\n", + " self.layernorm1 = nn.LayerNorm(hid_dim)\n", + " self.layernorm2 = nn.LayerNorm(hid_dim)\n", + "\n", + " self.dropout1 = nn.Dropout(dropout)\n", + " self.dropout2 = nn.Dropout(dropout)\n", + "\n", + " def forward(self, inputs, padding_mask):\n", + " attention = self.attn({'query': inputs, 'key': inputs, 'value': inputs, 'mask': padding_mask})\n", + " attention = self.dropout1(attention)\n", + " attention = self.layernorm1(inputs + attention)\n", + "\n", + " ffn_outputs = self.ffn(attention)\n", + " ffn_outputs = self.dropout2(ffn_outputs)\n", + " ffn_outputs = self.layernorm2(attention + ffn_outputs)\n", + " return ffn_outputs\n", + "\n", + "class Encoder(nn.Module):\n", + " def __init__(self):\n", + " super(Encoder, self).__init__()\n", + " self.embedding = nn.Embedding(n_enc_vocab, hid_dim)\n", + " self.pos_encoding = get_sinusoid_encoding_table(pe_source, hid_dim)\n", + " self.enc_layers = EncoderLayer()\n", + " self.dropout1 = nn.Dropout(dropout)\n", + "\n", + " def forward(self, x, padding_mask):\n", + " emb = self.embedding(x)\n", + " emb *= math.sqrt(hid_dim)\n", + " emb = self.pos_encoding(emb)\n", + " output = self.dropout1(emb)\n", + "\n", + " for i in range(n_layers):\n", + " output = self.enc_layers(output, padding_mask)\n", + " return output\n", + "\n", + "class DecoderLayer(nn.Module):\n", + " def __init__(self):\n", + " super(DecoderLayer, self).__init__()\n", + " self.attn = MultiHeadAttentionLayer(hid_dim, n_heads)\n", + " self.attn_2 = MultiHeadAttentionLayer(hid_dim, n_heads)\n", + " self.ffn = PositionwiseFeedforwardLayer(hid_dim, pf_dim)\n", + "\n", + " self.layernorm1 = nn.LayerNorm(hid_dim)\n", + " self.layernorm2 = nn.LayerNorm(hid_dim)\n", + " self.layernorm3 = nn.LayerNorm(hid_dim)\n", + "\n", + " self.dropout1 = nn.Dropout(dropout)\n", + " self.dropout2 = nn.Dropout(dropout)\n", + " self.dropout3 = nn.Dropout(dropout)\n", + "\n", + " def forward(self, inputs, enc_outputs, padding_mask, look_ahead_mask):\n", + " attention1 = self.attn({'query': inputs, 'key': inputs, 'value': inputs, 'mask': look_ahead_mask})\n", + " attention1 = self.dropout1(attention1)\n", + " attention1 = self.layernorm1(inputs + attention1)\n", + "\n", + " attention2 = self.attn_2({'query': attention1, 'key': enc_outputs, 'value': enc_outputs, 'mask': padding_mask})\n", + " attention2 = self.dropout2(attention2)\n", + " attention2 = self.layernorm2(attention1 + attention2)\n", + "\n", + " ffn_outputs = self.ffn(attention2)\n", + " ffn_outputs = self.dropout3(ffn_outputs)\n", + " ffn_outputs = self.layernorm3(attention2 + ffn_outputs)\n", + " return ffn_outputs\n", + "\n", + "class Decoder(nn.Module):\n", + " def __init__(self):\n", + " super(Decoder, self).__init__()\n", + " self.embedding = nn.Embedding(n_dec_vocab, hid_dim)\n", + " self.pos_encoding = get_sinusoid_encoding_table(pe_target, hid_dim)\n", + " self.dec_layers = DecoderLayer()\n", + " self.dropout = nn.Dropout(dropout)\n", + "\n", + " def forward(self, enc_output, dec_input, padding_mask, look_ahead_mask):\n", + " emb = self.embedding(dec_input)\n", + " emb *= math.sqrt(hid_dim)\n", + " emb = self.pos_encoding(emb)\n", + " output = self.dropout(emb)\n", + "\n", + " for i in range(n_layers):\n", + " output = self.dec_layers(output, enc_output, padding_mask, look_ahead_mask)\n", + " return output\n", + "\n", + "class Transformer(nn.Module):\n", + " def __init__(self, n_enc_vocab, n_dec_vocab,\n", + " n_layers, pf_dim, hid_dim, n_heads,\n", + " pe_source, pe_target, dropout):\n", + " super(Transformer, self).__init__()\n", + "\n", + " self.encoder = Encoder()\n", + " self.decoder = Decoder()\n", + " self.fin_output = nn.Linear(hid_dim, n_dec_vocab)\n", + "\n", + " def forward(self, enc_inputs, dec_inputs):\n", + " enc_padding_mask = create_padding_mask(enc_inputs)\n", + " dec_padding_mask = create_padding_mask(enc_inputs)\n", + "\n", + " look_ahead_mask = create_look_ahead_mask(dec_inputs)\n", + " dec_target_padding_mask = create_padding_mask(dec_inputs).to(device)\n", + " look_ahead_mask = torch.maximum(dec_target_padding_mask, look_ahead_mask)\n", + "\n", + " enc_output = self.encoder(enc_inputs, enc_padding_mask)\n", + " dec_output = self.decoder(enc_output, dec_inputs, dec_padding_mask, look_ahead_mask)\n", + " final_output = self.fin_output(dec_output)\n", + " return final_output" + ], + "metadata": { + "id": "jW0LSuEK4_HB" + }, + "execution_count": 10, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# Build model + Init + Forward Pass Shape Check (before training)\n", + "model = Transformer(\n", + " n_enc_vocab = n_enc_vocab,\n", + " n_dec_vocab = n_dec_vocab,\n", + " n_layers = n_layers,\n", + " pf_dim = pf_dim,\n", + " hid_dim = hid_dim,\n", + " n_heads = n_heads,\n", + " pe_source = 512,\n", + " pe_target = 512,\n", + " dropout = dropout\n", + ").to(device)\n", + "\n", + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "print(f'The model has {count_parameters(model):,} trainable parameters')\n", + "\n", + "def initialize_weights(m):\n", + " classname = m.__class__.__name__\n", + " if classname.find('Linear') != -1:\n", + " nn.init.kaiming_normal_(m.weight)\n", + " if m.bias is not None:\n", + " nn.init.constant_(m.bias, 0.0)\n", + "\n", + "model.apply(initialize_weights)\n", + "\n", + "import os.path\n", + "if os.path.isfile('./checkpoints/transformermodel.pt'):\n", + " model.load_state_dict(torch.load('./checkpoints/transformermodel.pt'))\n", + "\n", + "print('네트워크 초기화 완료')\n", + "\n", + "print(\"=== Forward pass shape check (no training) ===\")\n", + "src_batch, trg_batch = next(iter(dataloader))\n", + "with torch.no_grad():\n", + " out = model(src_batch, trg_batch)\n", + "show_shape(\"model_output(logits)\", out)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "smFtliiK4_2S", + "outputId": "3d3c042b-a5a1-45d8-f165-5c3089188f44" + }, + "execution_count": 11, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "The model has 7,347,293 trainable parameters\n", + "네트워크 초기화 완료\n", + "=== Forward pass shape check (no training) ===\n", + "=== [DEBUG] ScaledDotProductAttention ===\n", + "query: shape=(128, 8, 40, 32), dtype=torch.float32, device=cpu\n", + "key: shape=(128, 8, 40, 32), dtype=torch.float32, device=cpu\n", + "value: shape=(128, 8, 40, 32), dtype=torch.float32, device=cpu\n", + "mask: shape=(128, 1, 1, 40), dtype=torch.float32, device=cpu\n", + "attention_weights: shape=(128, 8, 40, 40), dtype=torch.float32, device=cpu\n", + "output: shape=(128, 8, 40, 32), dtype=torch.float32, device=cpu\n", + "attn_weights sample (head0, q0, first 8): [0. 0. 0. 0. 0. 1. 0. 0.]\n", + "model_output(logits): shape=(128, 40, 7773), dtype=torch.float32, device=cpu\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# Loss + Optimizer + Label Shift Debug (one batch)\n", + "criterion = nn.CrossEntropyLoss()\n", + "learning_rate = 0.0005\n", + "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n", + "\n", + "print(\"=== Label shift check ===\")\n", + "src_batch, trg_batch = next(iter(dataloader))\n", + "with torch.no_grad():\n", + " logits = model(src_batch, trg_batch)\n", + "\n", + "preds_id = torch.transpose(logits, 1, 2)\n", + "pad = torch.LongTensor(trg_batch.size(0), 1).fill_(0).to(device)\n", + "labels_lm = torch.cat((trg_batch[:, 1:], pad), -1)\n", + "\n", + "show_shape(\"preds_id\", preds_id)\n", + "show_shape(\"labels_lm\", labels_lm)\n", + "\n", + "print(\"TRG original first row:\", trg_batch[0, :15].detach().cpu().tolist())\n", + "print(\"TRG shifted first row:\", labels_lm[0, :15].detach().cpu().tolist())" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "9Z9wVB9L5FCJ", + "outputId": "65beadab-89c3-4f87-c637-d24ef6eeb69a" + }, + "execution_count": 12, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "=== Label shift check ===\n", + "preds_id: shape=(128, 7773, 40), dtype=torch.float32, device=cpu\n", + "labels_lm: shape=(128, 40), dtype=torch.int64, device=cpu\n", + "TRG original first row: [2, 11, 170, 2515, 7241, 420, 34, 16, 56, 182, 15, 828, 3, 0, 0]\n", + "TRG shifted first row: [11, 170, 2515, 7241, 420, 34, 16, 56, 182, 15, 828, 3, 0, 0, 0]\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# Train Loop + Save Best + Loss Plot\n", + "def train(epoch, model, dataloader, optimizer, criterion, clip):\n", + " model.train()\n", + " epoch_loss = 0\n", + "\n", + " with tqdm_notebook(total=len(dataloader), desc=f\"Train {epoch+1}\") as pbar:\n", + " for batch_idx, samples in enumerate(dataloader):\n", + " src_inputs, trg_outputs = samples\n", + "\n", + " logits_lm = model(src_inputs, trg_outputs)\n", + "\n", + " pad = torch.LongTensor(trg_outputs.size(0), 1).fill_(0).to(device)\n", + " preds_id = torch.transpose(logits_lm,1,2)\n", + " labels_lm = torch.cat((trg_outputs[:, 1:], pad), -1)\n", + "\n", + " optimizer.zero_grad()\n", + " loss = criterion(preds_id, labels_lm)\n", + " loss.backward()\n", + " torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n", + " optimizer.step()\n", + "\n", + " epoch_loss += loss.item()\n", + " pbar.update(1)\n", + "\n", + " return epoch_loss / len(dataloader)\n", + "\n", + "CLIP = 0.5\n", + "epoch_ = []\n", + "epoch_train_loss = []\n", + "\n", + "torch.backends.cudnn.benchmark = True\n", + "best_epoch_loss = float(\"inf\")\n", + "\n", + "for epoch in range(N_EPOCHS):\n", + " train_loss = train(epoch, model, dataloader, optimizer, criterion, CLIP)\n", + "\n", + " if train_loss < best_epoch_loss:\n", + " if not os.path.isdir(\"checkpoints\"):\n", + " os.makedirs(\"checkpoints\")\n", + " best_epoch_loss = train_loss\n", + " torch.save(model.state_dict(), './checkpoints/transformermodel.pt')\n", + "\n", + " epoch_.append(epoch)\n", + " epoch_train_loss.append(train_loss)\n", + " print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\n", + "\n", + "fig = plt.figure(figsize=(8,8))\n", + "fig.set_facecolor('white')\n", + "ax = fig.add_subplot()\n", + "ax.plot(epoch_, epoch_train_loss, label='Average loss')\n", + "ax.legend()\n", + "ax.set_xlabel('epoch')\n", + "ax.set_ylabel('loss')\n", + "plt.show()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 903, + "referenced_widgets": [ + "804b6ce766214273acc5123f5fe35298", + "a4c41c602b0648f3a949999abb832656", + "71baea8c0b804a24bb9cfbb9c5e14848", + "a836170cd4294199999afe5d9e29844b", + "933efcc469d54abaa3e13ec562ecb82f", + "10e77e3e9d234b78a4781a4864e5cde2", + "3cb4647f5d734911baee034e4daa92a4", + "b54a566de8c24b03813fcf5b5a1fb3e6", + "ed1efb4338b644718104455d5d7a328a", + "1925402698a8454ea9a39ab8f2f74051", + "8d71903bee964efcb0e3376db0a24a5c", + "4e157a6ebb5e4cc9be300e71d052cd21", + "86db40fc982e4455a890a3ee250fb337", + "d9cc66a9378244269dcc46fbbda022f5", + "80e28e0170844a9f9eb9f123ee4670bb", + "eac93e8e6d124765a7f72ebe6a272b3e", + "daf794f81e9842538be4966ca15df99f", + "052268e785d3458fa78770d233931a50", + "f77a159b25e549e6ab193df25f728b18", + "bac013322bfd40578f8775c13fb385f4", + "61acf50ee75f4ec883f2560c09f91dee", + "18203cc20e5e4baf8c8ef9d4320d68ff", + "a397e9386fc1407a9502b6357c107390", + "d8b46f6c917143b09334a4da7db66aa6", + "7e1cb3526d3940fd83467efeff101353", + "e300a51d0b6147638db40e9b0fba1888", + "979f222f29eb493190cb882a26bdd66a", + "e5016eab18c5429a960338f294c7bc39", + "3d0e355691ed428bb646637df39eafd8", + "c66821fde8ab4ee5a4b68ceb8fb73998", + "a2b3809cf0c749b6bfe2fb73fb161884", + "cf249fbb234747cab184ed6e75aee387", + "13e46b4d868d4dac9d4958e0ea0fc74c" + ] + }, + "id": "iyXwiMB98bva", + "outputId": "4eef32f4-8e65-4975-ff7d-f805272e0a40" + }, + "execution_count": 13, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "/tmp/ipython-input-1074719180.py:6: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n", + "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n", + " with tqdm_notebook(total=len(dataloader), desc=f\"Train {epoch+1}\") as pbar:\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "Train 1: 0%| | 0/64 [00:00" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAArMAAAKnCAYAAACVoMWWAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAbIhJREFUeJzt3Xd4VGXexvF7Jp1UQggECL33Ir0qVRCNDQQUXduqoVuxK2pQkY7gIraVpkhxpaMmoSMkIL2GTuik9znvH2heo7RAkjOT+X6ua641k2dm7jkesrcPvzmxGIZhCAAAAHBAVrMDAAAAADeLMgsAAACHRZkFAACAw6LMAgAAwGFRZgEAAOCwKLMAAABwWJRZAAAAOCzKLAAAAByWq9kBiprNZtPJkyfl6+sri8VidhwAAAD8jWEYSkpKUrly5WS1Xnvv1enK7MmTJxUaGmp2DAAAAFzHsWPHVKFChWuucboy6+vrK+nywfHz8zM5DQAAAP4uMTFRoaGhub3tWpyuzP45WuDn50eZBQAAsGM3MhLKB8AAAADgsCizAAAAcFiUWQAAADgsp5uZBQAARcMwDGVnZysnJ8fsKLBDbm5ucnFxueXnocwCAIACl5mZqVOnTik1NdXsKLBTFotFFSpUkI+Pzy09D2UWAAAUKJvNpri4OLm4uKhcuXJyd3fnFxUhD8MwdPbsWR0/flw1atS4pR1ayiwAAChQmZmZstlsCg0NVYkSJcyOAztVunRpHT58WFlZWbdUZvkAGAAAKBTX+zWkcG4FtVvPWQYAAACHRZkFAABwYocPH5bFYtHWrVvNjnJTKLMAAAB/s379erm4uKhXr15mR8F1UGYBAAD+ZsaMGRo8eLCio6N18uTJQn2tP6/Hi5tDmQUAAPiL5ORkzZ07V88++6x69eqlr776Kvd7/fv3V9++ffOsz8rKUlBQkL755htJly9NFhERoSpVqsjLy0uNGjXSvHnzctdHRkbKYrFo6dKlatasmTw8PLRmzRodPHhQ99xzj8qUKSMfHx81b95cq1atyvNap06dUq9eveTl5aUqVapo1qxZqly5ssaPH5+75tKlS3ryySdVunRp+fn56Y477tC2bdvydQyioqLUokULeXh4KCQkRK+88kqewj1v3jw1aNBAXl5eKlWqlLp06aKUlJTc99eiRQt5e3srICBAbdu21ZEjR/L1+vnBpbkAAEChMwxDaVlF/5vAvNxc8v2p+e+++061a9dWrVq19PDDD2vYsGEaOXKkLBaLBgwYoAcffFDJycm5F/tfvny5UlNTde+990qSIiIi9O2332ratGmqUaOGoqOj9fDDD6t06dLq2LFj7uu88sorGjNmjKpWraqSJUvq2LFj6tmzp95//315eHjom2++Ue/evbV3715VrFhRkjRw4ECdO3dOkZGRcnNz04gRI3TmzJk8+R988EF5eXlp6dKl8vf312effabOnTtr3759CgwMvO77P3HihHr27KnHHntM33zzjfbs2aOnnnpKnp6eevvtt3Xq1Cn169dPH330ke69914lJSVp9erVuTvMYWFheuqppzR79mxlZmZq06ZNhXqdYcosAAAodGlZOar75vIif91d73ZXCff81Z0ZM2bo4YcfliT16NFDCQkJioqKUqdOndS9e3d5e3trwYIFeuSRRyRJs2bN0t133y1fX19lZGTogw8+0KpVq9S6dWtJUtWqVbVmzRp99tlnecrsu+++q65du+Z+HRgYqEaNGuV+PWrUKC1YsEA//vijBg0apD179mjVqlX67bffdNttt0mSPv/8c9WoUSP3MWvWrNGmTZt05swZeXh4SJLGjBmjhQsXat68eXr66aev+/4//fRThYaGavLkybJYLKpdu7ZOnjypl19+WW+++aZOnTql7Oxs3XfffapUqZIkqUGDBpKkCxcuKCEhQXfddZeqVasmSapTp06+jn9+MWYAAADwh71792rTpk3q16+fJMnV1VV9+/bVjBkzcr/u06ePZs6cKUlKSUnRokWLNGDAAEnSgQMHlJqaqq5du8rHxyf39s033+jgwYN5XuvPQvqn5ORkvfDCC6pTp44CAgLk4+Oj3bt36+jRo7nZXF1d1bRp09zHVK9eXSVLlsz9etu2bUpOTlapUqXyvH5cXNw/Xv9qdu/erdatW+fZTW3btq2Sk5N1/PhxNWrUSJ07d1aDBg304IMPavr06bp48aKky4X8scceU/fu3dW7d29NmDBBp06duqHXvVnszAIAgELn5eaiXe92N+V182PGjBnKzs5WuXLlcu8zDEMeHh6aPHmy/P39NWDAAHXs2FFnzpzRypUr5eXlpR49eki6XEglafHixSpfvnye5/5zp/RP3t7eeb5+4YUXtHLlSo0ZM0bVq1eXl5eXHnjgAWVmZt5w/uTkZIWEhCgyMvIf3wsICLjh57kWFxcXrVy5UuvWrdOKFSs0adIkvfbaa9q4caOqVKmiL7/8UkOGDNGyZcs0d+5cvf7661q5cqVatWpVIK//d5RZAABQ6CwWS77/ur+oZWdn65tvvtEnn3yibt265fleWFiYZs+erWeeeUZt2rRRaGio5s6dq6VLl+rBBx+Um5ubJKlu3bry8PDQ0aNH84wU3Ii1a9fqsccey529TU5O1uHDh3O/X6tWLWVnZys2NlbNmjWTdHkn+M9dUUlq2rSp4uPj5erqqsqVK9/EUbg8FvDDDz/IMIzc3dm1a9fK19dXFSpUkHT532fbtm3Vtm1bvfnmm6pUqZIWLFigESNGSJKaNGmiJk2aaOTIkWrdurVmzZpVaGXW1DGDqVOnqmHDhvLz85Ofn59at26tpUuXXnX99OnT1b59e5UsWVIlS5ZUly5dtGnTpiJMDAAAiquffvpJFy9e1BNPPKH69evnud1///25owbS5asaTJs2TStXrswdMZAkX19fvfDCCxo+fLi+/vprHTx4UDExMZo0aZK+/vrra75+jRo1NH/+fG3dulXbtm1T//79ZbPZcr9fu3ZtdenSRU8//bQ2bdqk2NhYPf300/Ly8sotnV26dFHr1q0VFhamFStW6PDhw1q3bp1ee+01bd68+YaOw3PPPadjx45p8ODB2rNnjxYtWqS33npLI0aMkNVq1caNG/XBBx9o8+bNOnr0qObPn6+zZ8+qTp06iouL08iRI7V+/XodOXJEK1as0P79+wt3btYw0Y8//mgsXrzY2Ldvn7F3717j1VdfNdzc3IwdO3ZccX3//v2NKVOmGLGxscbu3buNxx57zPD39zeOHz9+w6+ZkJBgSDISEhIK6m0AAIC/SEtLM3bt2mWkpaWZHSVf7rrrLqNnz55X/N7GjRsNSca2bdsMwzCMXbt2GZKMSpUqGTabLc9am81mjB8/3qhVq5bh5uZmlC5d2ujevbsRFRVlGIZh/Prrr4Yk4+LFi3keFxcXZ9x+++2Gl5eXERoaakyePNno2LGjMXTo0Nw1J0+eNO68807Dw8PDqFSpkjFr1iwjODjYmDZtWu6axMREY/DgwUa5cuUMNzc3IzQ01BgwYIBx9OjRK763uLg4Q5IRGxube19kZKTRvHlzw93d3Shbtqzx8ssvG1lZWbnvvXv37kbp0qUNDw8Po2bNmsakSZMMwzCM+Ph4IywszAgJCTHc3d2NSpUqGW+++aaRk5Pzj9e91nmSn75mMQzDKLyqnH+BgYH6+OOP9cQTT1x3bU5OjkqWLKnJkydr4MCBN/T8iYmJ8vf3V0JCgvz8/G41LgAA+Jv09HTFxcWpSpUq8vT0NDtOsXb8+HGFhoZq1apV6ty5s9lx8uVa50l++prdDK/k5OTo+++/V0pKSu6lLK4nNTVVWVlZ17xmWkZGhjIyMnK/TkxMvOWsAAAAZvjll1+UnJysBg0a6NSpU3rppZdUuXJldejQwexopjG9zG7fvl2tW7dWenq6fHx8tGDBAtWtW/eGHvvyyy+rXLly6tKly1XXRERE6J133imouAAAAKbJysrSq6++qkOHDsnX11dt2rTRzJkzcz+A5oxMHzPIzMzU0aNHlZCQoHnz5unzzz9XVFTUdQvt6NGj9dFHHykyMlINGza86ror7cyGhoYW2ZhBWmaOpkUd1LOdqskzn5cHAQDAETFmgBtRbMYM3N3dVb16dUlSs2bN9Ntvv2nChAn67LPPrvqYMWPGaPTo0Vq1atU1i6x0+Zpuf7+uW1F64uvftO7geZ1NztAH9zYwLQcAAEBxZHe/Acxms+XZSf27jz76SKNGjdKyZcv+8Zsz7NGznarJYpFmbTyqRVtPmB0HAACgWDG1zI4cOVLR0dE6fPiwtm/frpEjRyoyMjL3em0DBw7UyJEjc9d/+OGHeuONN/TFF1+ocuXKio+PV3x8fO5v27BH7WuU1uDbL+88vzp/uw6etd+sAAAUJDu7YBLsTEGdH6aW2TNnzmjgwIGqVauWOnfurN9++03Lly9X165dJUlHjx7N8/t8p06dqszMTD3wwAMKCQnJvY0ZM8ast3BDhnapqVZVA5WSmaPwmTFKy8wxOxIAAIXmzw8jpaammpwE9uzPX9Pr4nJrnyky/QNgRc2s68yeSUxXz4lrdC45Qw81D9Xo+6896wsAgCM7deqULl26pODgYJUoUSL3N1QB0uWx0pMnT8rNzU0VK1b8x/nhUB8AcxbBfp6a8FBjPTxjo+b8dkwtqwbq3iYVzI4FAEChKFu2rKTLfwsLXInVar1ikc0vymwRals9SEPuqKEJP+/Xq/N3qEF5f1UP9jU7FgAABc5isSgkJETBwcHKysoyOw7skLu7u6zWW594pcwWsSGda2jzkQtae+C8npsZo0Xh7eTlzvVnAQDFk4uLyy3PRALXYneX5iruXKwWje/bRKV9PbTvdLLeXLTD7EgAAAAOizJrgtK+HprwUGNZLdL3W45r3pbjZkcCAABwSJRZk7SpFqRhXWpKkl5fuF37TieZnAgAAMDxUGZNFH57dbWvEaT0LJuemxmj1MxssyMBAAA4FMqsiVysFo3r21jBvh46cCZZry/cwW9LAQAAyAfKrMmCfDw0sV8TWS3S/JgT+n4z87MAAAA3ijJrB1pVLaXnu9WSJL2xaIf2xCeanAgAAMAxUGbtxLMdq6ljzdLKyL48P5uSwfwsAADA9VBm7YTVatHYPo1U1s9Th86m6LUF25mfBQAAuA7KrB0p5eOhSf2byMVq0cKtJzXnt2NmRwIAALBrlFk707xyoF74Y372rR93atdJ5mcBAACuhjJrh/7doapur1Vamdk2DZoVo2TmZwEAAK6IMmuHrFaLPunTWCH+njp0LkUj5zM/CwAAcCWUWTsV6O2uyf2byNVq0f+2ndTMjUfNjgQAAGB3KLN2rFmlQL3U4/L87Ls/7dKOEwkmJwIAALAvlFk791T7qupSJzh3fjYpPcvsSAAAAHaDMmvnLBaLxjzYSOUDvHT4fKpe+YH5WQAAgD9RZh1AQAl3Tfpjfnbx9lP6dsMRsyMBAADYBcqsg2hasaReubO2JGnUT7u1/TjzswAAAJRZB/JEuyrqWreMMnNsCp8Vo0TmZwEAgJOjzDoQi8WiMQ80UoWSXjp6IVUvff8787MAAMCpUWYdjH8JN03p31RuLhYt2xmvr9cdNjsSAACAaSizDqhRaIBe7VlHkvT+kt3aduySuYEAAABMQpl1UI+1qawe9coqK8dQ+KwYJaQyPwsAAJwPZdZBWSwWffhAQ4UGeun4xTS9OG8b87MAAMDpUGYdmL+Xmz7t30zuLlat2HVaX6w9bHYkAACAIkWZdXANKvjrtV6X52cjluxW7NGLJicCAAAoOpTZYmBg60rq1SBE2TZDg2bF6lJqptmRAAAAigRlthiwWCyKuL+BKpUqoROX0vTC98zPAgAA50CZLSb8PC9ff9bdxapVu8/o89VxZkcCAAAodJTZYqR+eX+90buuJOnDZXu05QjzswAAoHijzBYzD7esqLsaXp6fHTwrRhdTmJ8FAADFF2W2mLFYLIq4r4GqBHnrZEK6nv9+m2w25mcBAEDxRJkthnw93TS5fxO5u1r1y54z+s/qQ2ZHAgAAKBSU2WKqXjl/vd27niTp4+V79dvhCyYnAgAAKHiU2WKsX4tQ3dO4nHJshgbPitUF5mcBAEAxQ5ktxiwWiz64t4GqlvZWfGK6hs/dyvwsAAAoViizxZy3h6um9G8qD1erovad1dSog2ZHAgAAKDCUWSdQJ8RP795zeX72kxV7tSmO+VkAAFA8UGadRJ/bQnVfk/KyGdLg2TE6l5xhdiQAAIBbRpl1EhaLRaPC6qtaaW+dTsxgfhYAABQLlFkn4u3hqk8HNJOnm1Wr95/TlF8PmB0JAADgllBmnUytsr4adU99SdK4Vfu0/uB5kxMBAADcPMqsE3rwtlA90KyCbIY0ZE6sziYxPwsAABwTZdZJvXtPPdUI9tHZpAwNmxurHOZnAQCAA6LMOqkS7q76dEBTebm5aO2B85r8C/OzAADA8VBmnViNMr56L+zy/Oz4n/dp3YFzJicCAADIH8qsk7u/WQX1ua2CDEMaMmerziSlmx0JAADghlFmoXfurq9aZXx1LjlDQ2dvZX4WAAA4DMos5OXuoikDmqqEu4vWHzqvCT/vNzsSAADADaHMQpJUPdhHH9zbQJI06Zf9Wr3/rMmJAAAAro8yi1xhTcqrX4tQGYY0bM5WnU5kfhYAANg3yizyeKt3PdUJ8dP5lEwNmR2r7Byb2ZEAAACuijKLPDzdXDSlfxN5u7toY9wFjV/F/CwAALBflFn8Q9XSPoq4v6EkaUrkAUXtY34WAADYJ8osrujuRuU0oGVFGYY0fO5WnUpIMzsSAADAP1BmcVVv3FVX9cr56QLzswAAwE5RZnFVl+dnm8rHw1W/Hb6oT1buMzsSAABAHpRZXFPlIG99+Mf87NTIg/p1zxmTEwEAAPw/yiyuq1fDEA1sXUmSNOK7rTp5iflZAABgHyizuCGv9aqjBuX9dTE1S4NnxyqL+VkAAGAHKLO4IR6ul+dnfT1cteXIRY1ZvtfsSAAAAJRZ3LiKpUroowcuz89+Fn1IP+8+bXIiAADg7CizyJc7G4TosTaVJUkjvtumE8zPAgAAE1FmkW+v9qyjRhX8lZCWpUGzYpSZzfwsAAAwB2UW+ebuatXk/k3l5+mq2KOX9NGyPWZHAgAATooyi5sSGlhCHz/YSJL0+Zo4rdgZb3IiAADgjCizuGnd65XVE+2qSJJe+H6bjl1INTkRAABwNpRZ3JKXe9RW49AAJaZna9DsWOZnAQBAkaLM4pZcnp9tIn8vN207dkkRS3ebHQkAADgRyixuWYWSJfTJH/OzX649rGU7mJ8FAABFgzKLAtGlbhk93aGqJOnFedt09DzzswAAoPBRZlFgXuxeS00rBigpPVvhs2KUkZ1jdiQAAFDMUWZRYNxcLl9/NqCEm7afSFDEEq4/CwAAChdlFgWqXICXxvVpLEn6at1hLdl+ytxAAACgWKPMosDdXjtYz3SsJkl6ed7vOnI+xeREAACguKLMolA8362mbqtUUkkZ2XpuZozSs5ifBQAABY8yi0Lh5mLVpP5NFOjtrp0nE/X+Yq4/CwAACh5lFoUmxN9LY/tcvv7sfzcc0f+2nTQ5EQAAKG4osyhUnWoF67lOl+dnR87frrhzzM8CAICCQ5lFoRvRtaZaVAlUMvOzAACggFFmUehcXaya1K+JSnm7a/epRL370y6zIwEAgGKCMosiUcbPU+P6NpbFIs3aeFSLtp4wOxIAACgGKLMoMh1qltag26tLkl6dv10HzyabnAgAADg6yiyK1LAuNdWqaqBSMnMUzvwsAAC4RZRZFCkXq0UTH2qiIB937YlP0ts/7jQ7EgAAcGCUWRS5YD9PTXioiSwWac5vx7Qg9rjZkQAAgIOizMIUbasHacgdNSRJr87foQNnkkxOBAAAHJGpZXbq1Klq2LCh/Pz85Ofnp9atW2vp0qXXfMz333+v2rVry9PTUw0aNNCSJUuKKC0K2pDONdS2eimlZeUofGas0jKZnwUAAPljapmtUKGCRo8erS1btmjz5s264447dM8992jnzivPUa5bt079+vXTE088odjYWIWFhSksLEw7duwo4uQoCC5Wi8b3baLSvh7aezpJby7i3yMAAMgfi2EYhtkh/iowMFAff/yxnnjiiX98r2/fvkpJSdFPP/2Ue1+rVq3UuHFjTZs27YaePzExUf7+/kpISJCfn1+B5cbNW3fwnB7+fKNshjTmwUZ6oFkFsyMBAAAT5aev2c3MbE5OjubMmaOUlBS1bt36imvWr1+vLl265Lmve/fuWr9+/VWfNyMjQ4mJiXlusC9tqgVpWJeakqTXF27XvtPMzwIAgBtjepndvn27fHx85OHhoWeeeUYLFixQ3bp1r7g2Pj5eZcqUyXNfmTJlFB8ff9Xnj4iIkL+/f+4tNDS0QPOjYITfXl3tawQpPcum8JkxSs3MNjsSAABwAKaX2Vq1amnr1q3auHGjnn32WT366KPatWtXgT3/yJEjlZCQkHs7duxYgT03Co6L1aJxfRsr2NdD+88k6/WFO2RnEzAAAMAOmV5m3d3dVb16dTVr1kwRERFq1KiRJkyYcMW1ZcuW1enTp/Pcd/r0aZUtW/aqz+/h4ZF7tYQ/b7BPQT4emtiviawWaX7MCX2/hevPAgCAazO9zP6dzWZTRkbGFb/XunVr/fzzz3nuW7ly5VVnbOF4WlUtpee71ZIkvbloh/bGMz8LAACuztQyO3LkSEVHR+vw4cPavn27Ro4cqcjISA0YMECSNHDgQI0cOTJ3/dChQ7Vs2TJ98skn2rNnj95++21t3rxZgwYNMustoBA827GaOtQsrfQsm56buUUpGczPAgCAKzO1zJ45c0YDBw5UrVq11LlzZ/32229avny5unbtKkk6evSoTp06lbu+TZs2mjVrlv7zn/+oUaNGmjdvnhYuXKj69eub9RZQCKxWi8b1aaSyfp46eDZFry3YzvwsAAC4Iru7zmxh4zqzjuO3wxf00H82KMdmaPR9DfRQi4pmRwIAAEXAIa8zC/xd88qBeuGP+dm3ftyp3ae4RjAAAMiLMgu79u8OVXV7rdLKyL58/dlk5mcBAMBfUGZh16xWiz7p01gh/p46dC5Fr85nfhYAAPw/yizsXqC3uyb3byJXq0U/bjupWZuOmh0JAADYCcosHEKzSoF6qcfl+dl3/rdLO04kmJwIAADYA8osHMaT7aqqc+1gZWbbNGhWjJLSs8yOBAAATEaZhcO4PD/bSOUDvHT4fKpeYX4WAACnR5mFQwko4a5Jf8zPLv79lL7dcMTsSAAAwESUWTicphVL6pU7a0uSRv20W9uPMz8LAICzoszCIT3Rroq61i2jzBybwmfFKJH5WQAAnBJlFg7JYrFozAONVKGkl45eSNXL835nfhYAACdEmYXD8i/hpsn9m8rNxaKlO+L19brDZkcCAABFjDILh9Y4NEAj76wjSXp/yW5tO3bJ3EAAAKBIUWbh8P7VtrJ61CurrBxD4bNilJDG/CwAAM6CMguHZ7FY9OEDDRUa6KXjF9P00rxtzM8CAOAkKLMoFvy93DSlf1O5u1i1fOdpfbH2sNmRAABAEaDMothoWCFAr/W6PD87eulubWV+FgCAYo8yi2JlYOtK6tUg5PL87MwYXUrNNDsSAAAoRJRZFCsWi0UR9zdQpVIldOJSml74nuvPAgBQnFFmUez4ef7//Oyq3af1+eo4syMBAIBCQplFsVS/vL/e6F1XkvThsj3acuSiyYkAAEBhoMyi2Hq4ZUXd1TBE2TZDg2fF6GIK87MAABQ3lFkUWxaLRRH3NVCVIG+dTEjX899vk83G/CwAAMUJZRbFmq+nmyb3byJ3V6t+2XNG01cfMjsSAAAoQJRZFHv1yvnr7d71JEkfLd+rzYcvmJwIAAAUFMosnEK/FqG6p3E55dgMDZoVqwvMzwIAUCxQZuEULBaL3r+3gaoGeSs+MV3D525lfhYAgGKAMgun4ePhqikDmsrD1aqofWc1Lfqg2ZEAAMAtoszCqdQJ8dO791yen/1kxT5timN+FgAAR0aZhdPpc1uo7m1SXjk2Q4Nnx+hccobZkQAAwE2izMLpWCwWvRdWX9VKe+t0YgbzswAAODDKLJySt4erPh3QTJ5uVq3ef06fRh4wOxIAALgJlFk4rVplfTXqnvqSpLEr92n9wfMmJwIAAPlFmYVTe/C2UN3ftIJshjRkTqzOJjE/CwCAI6HMwumNCqunGsE+Opt0eX42h/lZAAAcBmUWTq+Eu6s+HdBUXm4uWnPgnCb/wvwsAACOgjILSKpRxlfvhV2enx3/8z6tO3DO5EQAAOBGUGaBP9zfrIL63FZBhiENmbNVZ5LSzY4EAACugzIL/MU7d9dXrTK+OpecoaGzmZ8FAMDeUWaBv/Byd9GUAU1Vwt1F6w+d14Sf95sdCQAAXANlFvib6sE++uDeBpKkSb/s1+r9Z01OBAAAroYyC1xBWJPy6tciVIYhDZuzVacTmZ8FAMAeUWaBq3irdz3VLuur8ymZGjI7Vtk5NrMjAQCAv6HMAlfh6eaiTwc0lbe7izbGXdD4VczPAgBgbyizwDVULe2jiPsbSpKmRB5Q1D7mZwEAsCeUWeA67m5UTgNaVpRhSMPnblV8AvOzAADYC8oscAPeuKuu6ob46QLzswAA2BXKLHAD/pyf9fFw1abDF/TJyn1mRwIAAKLMAjescpC3PvxjfnZq5EH9uveMyYkAAABlFsiHXg1DNLB1JUnSiLlbdfJSmsmJAABwbpRZIJ9e61VH9cv76WJqlgbPjlUW87MAAJiGMgvkk4eri6b0bypfD1dtOXJRY5bvNTsSAABOizIL3IRKpbz10QOX52c/iz6kn3efNjkRAADOiTIL3KQ7G4TosTaVJUnPf79NJ5ifBQCgyFFmgVswsmdtNargr0upWRo0K0aZ2czPAgBQlCizwC3wcHXR5P5N5efpqtijl/Tx8j1mRwIAwKlQZoFbFBpYQh8/2EiSNH11nFbuYn4WAICiQpkFCkD3emX1eNsqkqTnv9uqYxdSTU4EAIBzoMwCBeSVO2urUWiAEtOzNWh2LPOzAAAUAcosUEDcXa2a0r+J/L3ctO3YJY1eyvwsAACFjTILFKAKJUvokz/mZ79YG6dlO+JNTgQAQPFGmQUKWJe6ZfRU+8vzsy/O26aj55mfBQCgsFBmgULwUo/aaloxQEnp2Ro0O0YZ2TlmRwIAoFiizAKFwM3Fqsn9myqghJt+P56giCXMzwIAUBgos0AhKRfgpbF9Ls/PfrXusJZsP2VyIgAAih/KLFCI7qhdRv/uWFWS9PK833XkfIrJiQAAKF4os0Ahe6FbLd1WqaSSMrL13MwYpWcxPwsAQEGhzAKFzM3Fqkn9m6hkCTftPJmo9xfvNjsSAADFBmUWKAIh/l4a27exJOm/G47of9tOmhsIAIBigjILFJHbawXruU7VJEkj529X3DnmZwEAuFWUWaAIjehaUy2qBCo5I1vhzM8CAHDLKLNAEXJ1sWpSvyYq5e2uXacS9e5Pu8yOBACAQ6PMAkWsjJ+nxvVtLItFmrXxqBZtPWF2JAAAHBZlFjBBh5qlNej26pKkV+dv18GzySYnAgDAMVFmAZMM61JTraoGKiUzh/lZAABuEmUWMImL1aKJDzVRkI+79sQn6e0fd5odCQAAh0OZBUwU7OepCQ81kcUizfntmBbEHjc7EgAADoUyC5isbfUgDbmjhiTptQU7dOAM87MAANwoyixgB4Z0rqE21Uop9Y/52bRM5mcBALgRlFnADrhYLRr/UGMF+Xho7+kkvbloh9mRAABwCJRZwE4E+3pqYr/Gslqk77cc17wtzM8CAHA9lFnAjrSpFqRhXWpKkt5YuEP7TyeZnAgAAPtGmQXsTPjt1dWuepDSsnL03MwYpWZmmx0JAAC7RZkF7Myf87PBvh7afyZZbyzk+rMAAFwNZRawQ0E+HprYr4msFumHmOP6bvMxsyMBAGCXKLOAnWpVtZSe71ZLkvTmoh3aG8/8LAAAf0eZBezYsx2rqUPN0krPsum5mVuUksH8LAAAf0WZBeyY1WrRuD6NVNbPUwfPpuj1hTtkGIbZsQAAsBuUWcDOlfLx0KT+TeRitWhB7AnN/Y35WQAA/kSZBRxA88qBer7b5evPvvXjTu0+lWhyIgAA7ANlFnAQz3SopttrlVZGtk3hM2OUzPwsAADmltmIiAg1b95cvr6+Cg4OVlhYmPbu3Xvdx40fP161atWSl5eXQkNDNXz4cKWnpxdBYsA8VqtFn/RprBB/Tx06l6JX529nfhYA4PRMLbNRUVEKDw/Xhg0btHLlSmVlZalbt25KSUm56mNmzZqlV155RW+99ZZ2796tGTNmaO7cuXr11VeLMDlgjkBvd03+Y372x20nNWvTUbMjAQBgKothR1s7Z8+eVXBwsKKiotShQ4crrhk0aJB2796tn3/+Ofe+559/Xhs3btSaNWuu+xqJiYny9/dXQkKC/Pz8Ciw7UJQ+izqoiKV75O5q1fxn26h+eX+zIwEAUGDy09fsamY2ISFBkhQYGHjVNW3atNGWLVu0adMmSdKhQ4e0ZMkS9ezZ84rrMzIylJiYmOcGOLqn2ldV59rBysy2adCsGCWlZ5kdCQAAU9hNmbXZbBo2bJjatm2r+vXrX3Vd//799e6776pdu3Zyc3NTtWrV1KlTp6uOGURERMjf3z/3FhoaWlhvASgyl+dnG6l8gJcOn0/VK8zPAgCclN2U2fDwcO3YsUNz5sy55rrIyEh98MEH+vTTTxUTE6P58+dr8eLFGjVq1BXXjxw5UgkJCbm3Y8e4RieKh4AS7prUv4lcrRYt/v2Uvt1wxOxIAAAUObuYmR00aJAWLVqk6OhoValS5Zpr27dvr1atWunjjz/Ove/bb7/V008/reTkZFmt1+7nzMyiuPl89SG9t3i33F2smv8c87MAAMfnMDOzhmFo0KBBWrBggX755ZfrFllJSk1N/UdhdXFxyX0+wNk80a6KutYto8wcm56bGaNE5mcBAE7E1DIbHh6ub7/9VrNmzZKvr6/i4+MVHx+vtLS03DUDBw7UyJEjc7/u3bu3pk6dqjlz5iguLk4rV67UG2+8od69e+eWWsCZWCwWjXng8vzs0Qupenne7/yHHQDAabia+eJTp06VJHXq1CnP/V9++aUee+wxSdLRo0fz7MS+/vrrslgsev3113XixAmVLl1avXv31vvvv19UsQG741/CTVMGNNWD09Zp6Y54fb3usB5re/2/6QAAwNHZxcxsUWJmFsXZF2vi9O5Pu+TmYtEPz7ZRwwoBZkcCACDfHGZmFkDB+lfbyuper4yycgyFz4pRQhrzswCA4o0yCxQjFotFHz3QSKGBXjp2IU0vzdvG/CwAoFijzALFjL+Xm6b0byp3F6uW7zytL9ceNjsSAACFhjILFEMNKwTotV51JEkRS3dr67FL5gYCAKCQUGaBYmpg60rq2aDs5fnZmTG6lJppdiQAAAocZRYopiwWi0bf31CVSpXQiUtpeuF7rj8LACh+KLNAMebn+f/zs6t2n9aMNXFmRwIAoEBRZoFirn55f73Ru64kafTSPYo5etHkRAAAFBzKLOAEHm5ZUXc1DFG2zdCgmTG6mML8LACgeKDMAk7AYrEo4r4GqhLkrZMJ6Xr++22y2ZifBQA4Psos4CR8Pd00uX8Tubta9cueM5q++pDZkQAAuGWUWcCJ1Cvnr7d715MkfbR8rzYfvmByIgAAbg1lFnAy/VqE6u5G5ZRjMzRoVqwuMD8LAHBglFnAyVgsFn1wXwNVDfJWfGK6Rny3lflZAIDDoswCTsjHw1VTBjSVh6tVkXvPalr0QbMjAQBwUyizgJOqE+Knd+6+PD/7yYp92hTH/CwAwPFQZgEn1rd5qO5tUl45NkODZ8fofHKG2ZEAAMgXyizgxCwWi94Lq69qpb11OjFDw7/j+rMAAMdCmQWcnLeHqz4d0EyeblZF7zurTyMPmB0JAIAbRpkFoFplffXuPfUlSWNX7tP6g+dNTgQAwI2hzAKQJPW5LVT3N60gmyENmROrs0nMzwIA7B9lFkCuUWH1VCPYR2eTMjR87lblMD8LALBzlFkAuUq4u+rTAU3l5eaiNQfOafIvzM8CAOwbZRZAHjXK+Oq9sMvzs+N/3qd1B86ZnAgAgKujzAL4h/ubVVCf2yrIMKQhc7bqTFK62ZEAALgiyiyAK3rn7vqqVcZX55IzNHQ287MAAPtEmQVwRV7uLpoyoKlKuLto/aHzmvDzfrMjAQDwD5RZAFdVPdhHH9zbQJI06Zf9WrOf+VkAgH2hzAK4prAm5dWvRagMQxo2N1ZnEpmfBQDYD8osgOt6q3c91S7rq3PJmRo8O1bZOTazIwEAIIkyC+AGeLq56NMBTeXt7qKNcReYnwUA2A3KLIAbUrW0jyLubyhJmvzrAUXvO2tyIgAAKLMA8uHuRuU0oGXFP+Zntyo+gflZAIC5KLMA8uWNu+qqboifLqRkagjzswAAk1FmAeTLn/OzPh6u2nT4gsau3Gd2JACAE6PMAsi3ykHeGn3/5evPfhp5UL/uPWNyIgCAs6LMArgpdzUsp0daVZIkjZi7VScvpZmcCADgjCizAG7a63fVUf3yfrqYmqXBs2OVxfwsAKCIUWYB3DQPVxdN6d9Uvh6u2nLkosas2Gt2JACAk6HMArgllUp566MHLl9/9rOoQ/p592mTEwEAnAllFsAtu7NBiB5rU1mS9Pz323SC+VkAQBGhzAIoECN71lajCv66lJqlQbNimJ8FABQJyiyAAuHh6qLJ/ZvK19NVsUcv6aNle8yOBABwApRZAAUmNLCEPn6gkSRp+uo4rdzF/CwAoHBRZgEUqB71y+rxtlUkSc9/t1XHLqSanAgAUJxRZgEUuFfurK1GoQFKTM/WoNmxysxmfhYAUDgoswAKnLurVVP6N5Gfp6u2Hbuk0UuZnwUAFA7KLIBCUaFkCX3Sp7Ek6Yu1cVq2I97cQACAYokyC6DQdK1bRk+1vzw/++K8bczPAgAKHGUWQKF6qUdtNakYoKT0bIXPilFGdo7ZkQAAxQhlFkChcnOxanL/pgoo4abfjycoYgnzswCAgkOZBVDoygd4aWyfy9ef/WrdYS3dfsrkRACA4oIyC6BI3FG7jP7dsaok6aV5v+vI+RSTEwEAigPKLIAi80K3WrqtUkklZVyen03PYn4WAHBrKLMAioybi1WT+jdRyRJu2nEiUe8v3m12JACAg6PMAihSIf5eGtu3sSTpvxuO6KffT5obCADg0CizAIrc7bWC9VynapKkV37YrrhzzM8CAG4OZRaAKUZ0rakWlQOVnJGt8JnMzwIAbg5lFoApXF2smtiviUp5u2vXqUSN+mmX2ZEAAA6IMgvANGX9PTWub2NZLNLMjUe1aOsJsyMBABwMZRaAqTrULK1Bt1eXJL06f7sOnk02OREAwJFQZgGYbmjnGmpZJVApmTnMzwIA8uWmyuzXX3+txYsX53790ksvKSAgQG3atNGRI0cKLBwA5+DqYtWkfk0U5OOuPfFJeud/O82OBABwEDdVZj/44AN5eXlJktavX68pU6boo48+UlBQkIYPH16gAQE4h2A/T43v20QWizR70zEtjGV+FgBwfTdVZo8dO6bq1S/PuC1cuFD333+/nn76aUVERGj16tUFGhCA82hXI0iD76ghSXp1wXYdOMP8LADg2m6qzPr4+Oj8+fOSpBUrVqhr166SJE9PT6WlpRVcOgBOZ2jnGmpTrZRS/5ifTctkfhYAcHU3VWa7du2qJ598Uk8++aT27dunnj17SpJ27typypUrF2Q+AE7GxWrR+IcaK8jHQ3tPJ+mtH3eYHQkAYMduqsxOmTJFrVu31tmzZ/XDDz+oVKlSkqQtW7aoX79+BRoQgPMJ9vXUxH6NZbVI320+rh+2HDc7EgDATlkMwzDMDlGUEhMT5e/vr4SEBPn5+ZkdB8A1TFi1X+NW7ZOXm4t+HNRWNcr4mh0JAFAE8tPXbmpndtmyZVqzZk3u11OmTFHjxo3Vv39/Xbx48WaeEgD+YdAd1dWuepDSsnL03MwYpWZmmx0JAGBnbqrMvvjii0pMTJQkbd++Xc8//7x69uypuLg4jRgxokADAnBef87PBvt6aP+ZZL2xkOvPAgDyuqkyGxcXp7p160qSfvjhB91111364IMPNGXKFC1durRAAwJwbkE+HprYr4msFumHmOP6bvMxsyMBAOzITZVZd3d3paamSpJWrVqlbt26SZICAwNzd2wBoKC0qlpKI7rWlCS9uWiH9sYnmZwIAGAvbqrMtmvXTiNGjNCoUaO0adMm9erVS5K0b98+VahQoUADAoAkPdepujrULK30LJuem7lFKRnMzwIAbrLMTp48Wa6urpo3b56mTp2q8uXLS5KWLl2qHj16FGhAAJAkq9WicX0aqYyfhw6eTdHrC3fIyS7GAgC4Ai7NBcChbIq7oH7TNyjHZmj0fQ30UIuKZkcCABSw/PQ115t9kZycHC1cuFC7d++WJNWrV0933323XFxcbvYpAeC6WlQJ1PPdauqjZXv11o871Sg0QHVC+A9TAHBWNzVmcODAAdWpU0cDBw7U/PnzNX/+fD388MOqV6+eDh48WNAZASCPZzpU0+21Sisj26bwmTFKZn4WAJzWTZXZIUOGqFq1ajp27JhiYmIUExOjo0ePqkqVKhoyZEhBZwSAPKxWiz7p01gh/p46dC5Fr87fzvwsADipmyqzUVFR+uijjxQYGJh7X6lSpTR69GhFRUUVWDgAuJpAb3dN6tdELlaLftx2UrM3cf1ZAHBGN1VmPTw8lJT0z+s8Jicny93d/ZZDAcCNuK1yoF7qXkuS9Pb/dmrnyQSTEwEAitpNldm77rpLTz/9tDZu3CjDMGQYhjZs2KBnnnlGd999d0FnBICreqp9VXWuHazMP+Znk9KzzI4EAChCN1VmJ06cqGrVqql169by9PSUp6en2rRpo+rVq2v8+PEFHBEArs5qtWjMg41Uzt9Th8+n6hXmZwHAqdzSdWYPHDiQe2muOnXqqHr16gUWrLBwnVmgeIo5elF9pq1Xts3QqLD6eqRVJbMjAQBuUn762g2X2REjRtxwgLFjx97w2qJGmQWKr89XH9J7i3fL3cWq+c+1Uf3y/mZHAgDchEL5pQmxsbE3tM5isdzoUwJAgXqiXRVtOHRBq3af1nMzY/TTkHby83QzOxYAoBDx62wBFCsJqVnqOXG1TlxKU88GZTWlf1P+IxsAHEx++tpNfQAMAOyVfwk3TRnQVG4uFi3ZHq9v1h8xOxIAoBBRZgEUO41DAzTyzjqSpPcW79Lvxy+ZGwgAUGgoswCKpX+1razu9cooK8dQ+KwYJaRx/VkAKI4oswCKJYvFoo8eaKTQQC8du5Cml+Zt4/qzAFAMmVpmIyIi1Lx5c/n6+io4OFhhYWHau3fvdR936dIlhYeHKyQkRB4eHqpZs6aWLFlSBIkBOBJ/LzdN6d9U7i5WLd95Wl+uPWx2JABAATO1zEZFRSk8PFwbNmzQypUrlZWVpW7duiklJeWqj8nMzFTXrl11+PBhzZs3T3v37tX06dNVvnz5IkwOwFE0rBCg13pdnp+NWLpbW49dMjcQAKBA2dWluc6ePavg4GBFRUWpQ4cOV1wzbdo0ffzxx9qzZ4/c3PJ//UguzQU4H8O4PDe7ZHu8ygd4acmQ9vIvwfVnAcBeOeyluRISEiRJgYGBV13z448/qnXr1goPD1eZMmVUv359ffDBB8rJybni+oyMDCUmJua5AXAuFotFo+9vqEqlSujEpTS9wPwsABQbdlNmbTabhg0bprZt26p+/fpXXXfo0CHNmzdPOTk5WrJkid544w198skneu+99664PiIiQv7+/rm30NDQwnoLAOyYn+f/z8+u3HVaM9bEmR0JAFAA7GbM4Nlnn9XSpUu1Zs0aVahQ4arratasqfT0dMXFxcnFxUWSNHbsWH388cc6derUP9ZnZGQoIyMj9+vExESFhoYyZgA4qf+uP6w3Fu2Uq9Wi755praYVS5odCQDwNw43ZjBo0CD99NNP+vXXX69ZZCUpJCRENWvWzC2yklSnTh3Fx8crMzPzH+s9PDzk5+eX5wbAeT3cqpLuahiibJuhQTNjdCn1nz83AACOw9QyaxiGBg0apAULFuiXX35RlSpVrvuYtm3b6sCBA7LZbLn37du3TyEhIXJ3dy/MuACKAYvFooj7GqhKkLdOJqTr+e+2yWazi7+gAgDcBFPLbHh4uL799lvNmjVLvr6+io+PV3x8vNLS0nLXDBw4UCNHjsz9+tlnn9WFCxc0dOhQ7du3T4sXL9YHH3yg8PBwM94CAAfk6+mmyf2byN3Vqp/3nNH01YfMjgQAuEmmltmpU6cqISFBnTp1UkhISO5t7ty5uWuOHj2aZxY2NDRUy5cv12+//aaGDRtqyJAhGjp0qF555RUz3gIAB1WvnL/e6l1XkvTR8r3afPiCyYkAADfDbj4AVlS4ziyAPxmGoaFzturHbScV4u+pxUPaK9CbcSUAMJvDfQAMAMxgsVj0wX0NVDXIW6cS0jXiu63MzwKAg6HMAnBqPh6umjKgqTxcrYrce1bTog+aHQkAkA+UWQBOr06In965u54k6ZMV+7QpjvlZAHAUlFkAkNS3eajubVJeOTZDg2fH6HxyxvUfBAAwHWUWAHR5fva9sPqqVtpbpxMzNJzrzwKAQ6DMAsAfvD1c9emAZvJ0syp631l9GnnA7EgAgOugzALAX9Qq66t376kvSRq7cp82HDpvciIAwLVQZgHgb/rcFqr7m1aQzZCGzI7V2STmZwHAXlFmAeAKRoXVU41gH51JytDwuVuVw/wsANglyiwAXEEJd1d9OqCpvNxctObAOU35lflZALBHlFkAuIoaZXz1Xtjl+dnxq/Zp3cFzJicCAPwdZRYAruH+ZhXU57Y/52e36kxSutmRAAB/QZkFgOt45+76qlXGV+eSMzR0NvOzAGBPKLMAcB1e7i6aMqCpSri7aP2h85r4836zIwEA/kCZBYAbUD3YRx/c20CSNPGX/Vqzn/lZALAHlFkAuEFhTcrroeahMgxp2NxYnUlkfhYAzEaZBYB8ePvueqpd1lfnkjM1eHassnNsZkcCAKdGmQWAfPB0c9GnA5rK291FG+MuaALzswBgKsosAORT1dI++uC+y/Ozk389oOh9Z01OBADOizILADfhnsbl1b9lxT/mZ7cqPoH5WQAwA2UWAG7Sm3fVVd0QP11IydQQ5mcBwBSUWQC4SZ5ul68/6+Phqk2HL2jsyn1mRwIAp0OZBYBbUCXIW6Pvvzw/+2nkQf2694zJiQDAuVBmAeAW3dWwnB5pVUmSNGLuVp1KSDM5EQA4D8osABSA1++qo/rl/XQxNUuDZ8Uqi/lZACgSlFkAKAAeri6a0r+pfD1ctfnIRY1ZsdfsSADgFCizAFBAKpXy1kcPNJQkfRZ1SD/vPm1yIgAo/iizAFCA7mwQosfaVJYkPf/9Np24xPwsABQmyiwAFLCRPWurYQV/XUrN0uBZMczPAkAhoswCQAHLnZ/1dFXM0Uv6aNkesyMBQLFFmQWAQhAaWEIfP9BIkjR9dZxW7mJ+FgAKA2UWAApJj/pl9XjbKpKkF77fpuMXU01OBADFD2UWAArRK3fWVqPQACWkZSl8Vqwys5mfBYCCRJkFgELk7mrV5H5N5Ofpqm3HLmn0UuZnAaAgUWYBoJCFBpbQJ30aS5K+WBun5TvjzQ0EAMUIZRYAikDXumX0VPv/n589doH5WQAoCJRZACgiL/WorSYVA5SUnq3wWTHKyM4xOxIAODzKLAAUETcXqyb3b6qAEm76/XiCIpYwPwsAt4oyCwBFqHyAl8b2uXz92a/WHdbS7adMTgQAjo0yCwBF7I7aZfTvjlUlSS/N+11HzqeYnAgAHBdlFgBM8EK3WmpWqaSSMi7Pz6ZnMT8LADeDMgsAJrg8P9tEJUu4aceJRH2wZLfZkQDAIVFmAcAkIf5eGtu3sSTpm/VH9NPvJ80NBAAOiDILACa6vVawnu1UTZL0yg/bFXeO+VkAyA/KLACY7PmuNdWicqCSM7IVPpP5WQDID8osAJjM1cWqif2aqJS3u3adStSon3aZHQkAHAZlFgDsQFl/T43r21gWizRz41Et2nrC7EgA4BAoswBgJzrULK3wTtUlSa/O366DZ5NNTgQA9o8yCwB2ZFiXGmpZJVApmTnMzwLADaDMAoAdcXWxalK/Jgrycdee+CS987+dZkcCALtGmQUAOxPs56nxfZvIYpFmbzqmhbHMzwLA1VBmAcAOtasRpMF31JAkvbpguw6cYX4WAK6EMgsAdmpo5xpqU62UUv+Yn03LZH4WAP6OMgsAdsrFatH4hxoryMdDe08n6a0fd5gdCQDsDmUWAOxYsK+nJj7UWFaL9N3m4/phy3GzIwGAXaHMAoCda1M9SEM715Qkvb5wh/afTjI5EQDYD8osADiAQXdUV7vqQUrLytFzM2OUmpltdiQAsAuUWQBwAC5Wi8b1bazSvh7afyZZbyzk+rMAIFFmAcBhlPb10KR+TWS1SD/EHNf3m4+ZHQkATEeZBQAH0qpqKY3oenl+9o1FO7Q3nvlZAM6NMgsADua5TtXVvkaQ0rNsem7mFqVkMD8LwHlRZgHAwVitFo3v21hl/Dx08GyKXl+4Q4ZhmB0LAExBmQUAB1TKx0OT+jWVi9WiBbEn9B3zswCcFGUWABxUiyqBer7b5fnZNxft1ILY4+zQAnA6lFkAcGDPdKimLnWClZFt0/C52/Tk15sVn5BudiwAKDKUWQBwYFarRVMfbqYXutWUu4tVP+85o67jovTdb8fYpQXgFCizAODg3FysGnRHDf00pJ0ahQYoKT1bL/3wuwZ+sUnHL6aaHQ8AChVlFgCKiZplfPXDM631as/a8nC1avX+c+o+Llr/3XBENhu7tACKJ8osABQjri5WPd2hmpYOba/bKpVUSmaO3li4QwM+36ij59mlBVD8UGYBoBiqWtpHc//dWm/1risvNxetP3Re3cdH68u1cezSAihWKLMAUEy5WC36V9sqWjasvVpVDVRaVo7e+d8u9flsvQ6dTTY7HgAUCMosABRzlUp5a9aTrfReWH15u7to85GLunPCan0WdVA57NICcHCUWQBwAlarRQ+3qqTlwzuofY0gZWTbFLF0j+6buk77TyeZHQ8AbhplFgCcSIWSJfTN4y300f0N5evpqm3HLqnXxDWa8usBZeXYzI4HAPlGmQUAJ2OxWNSneahWDu+oO2oHKzPHpo+X71XYlLXadTLR7HgAkC+UWQBwUmX9PTXj0ds0rm8j+Xu5aefJRN09eY3GrtynzGx2aQE4BsosADgxi8Wie5tU0MoRHdS9Xhll2wxN/Hm/7p68RtuPJ5gdDwCuizILAFCwr6emPdxMk/s3UaC3u/bEJyns07X6cNkepWflmB0PAK6KMgsAkHR5l/auhuW0cngH9W5UTjk2Q1MjD6rXxNXacuSi2fEA4IooswCAPEr5eGhSvyb67JFmCvLx0MGzKXpg2jq999MupWWySwvAvlBmAQBX1L1eWa0a0UH3NS0vw5A+XxOnOydEa+Oh82ZHA4BclFkAwFUFlHDX2D6N9eVjzVXWz1OHz6eq73826K1FO5SSkW12PACgzAIAru/22sFaMaKDHmoeKkn6ev0RdR8frbUHzpmcDICzo8wCAG6In6ebRt/fUP99ooXKB3jp+MU0Dfh8o0bO366k9Cyz4wFwUpRZAEC+tK9RWsuHd9DA1pUkSbM3HVW3cdGK3HvG5GQAnBFlFgCQbz4ernr3nvqa83QrVSpVQqcS0vXYl7/phe+3KSGVXVoARYcyCwC4aa2qltLSoe31eNsqslikeVuOq+u4KK3cddrsaACcBGUWAHBLSri76s3edTXvmdaqWtpbZ5Iy9NQ3mzV0TqwupGSaHQ9AMUeZBQAUiGaVArVkSHv9u2NVWS3Soq0n1W1clJZsP2V2NADFGGUWAFBgPN1cNPLOOlrwXFvVLOOjc8mZem5mjJ79dovOJmWYHQ9AMUSZBQAUuEahAfrf4HYafEd1uVgtWrojXt3GRWnR1hMyDMPseACKEVPLbEREhJo3by5fX18FBwcrLCxMe/fuveHHz5kzRxaLRWFhYYUXEgBwUzxcXfR8t1paFN5WdUP8dDE1S0PnbNVT32zW6cR0s+MBKCZMLbNRUVEKDw/Xhg0btHLlSmVlZalbt25KSUm57mMPHz6sF154Qe3bty+CpACAm1W/vL8WDWqr57vWlJuLRat2n1HXsVH6fvMxdmkB3DKLYUc/Sc6ePavg4GBFRUWpQ4cOV12Xk5OjDh066PHHH9fq1at16dIlLVy48IZeIzExUf7+/kpISJCfn18BJQcA3Ii98Ul6cd42/X48QZLUsWZpfXBfA5UP8DI5GQB7kp++ZlczswkJl3+4BQYGXnPdu+++q+DgYD3xxBPXfc6MjAwlJibmuQEAzFGrrK/mP9tGr9xZW+6uVkXtO6vu46I1c+MRdmkB3BS7KbM2m03Dhg1T27ZtVb9+/auuW7NmjWbMmKHp06ff0PNGRETI398/9xYaGlpQkQEAN8HVxapnOlbTkiHt1bRigJIzsvXagh0a8PlGHbuQanY8AA7GbspseHi4duzYoTlz5lx1TVJSkh555BFNnz5dQUFBN/S8I0eOVEJCQu7t2LFjBRUZAHALqgf76Ptn2uiNu+rK082qdQfPq9u4aH21Nk42G7u0AG6MXczMDho0SIsWLVJ0dLSqVKly1XVbt25VkyZN5OLiknufzWaTJFmtVu3du1fVqlW75msxMwsA9ufwuRS9/MPv2hh3QZLUonKgPnygoaoEeZucDIAZ8tPXTC2zhmFo8ODBWrBggSIjI1WjRo1rrk9PT9eBAwfy3Pf6668rKSlJEyZMUM2aNeXu7n7N56DMAoB9stkMzdx4RBFL9yg1M0cerla90K2WHm9XRS5Wi9nxABSh/PQ11yLKdEXh4eGaNWuWFi1aJF9fX8XHx0uS/P395eV1+ZOtAwcOVPny5RURESFPT89/zNMGBARI0jXnbAEA9s9qteiR1pXVqVawRs7frjUHzun9Jbu1ePspffxAQ9Uo42t2RAB2yNSZ2alTpyohIUGdOnVSSEhI7m3u3Lm5a44ePapTp/i93gDgLEIDS+i/T7TQ6PsayNfDVVuPXVKviWs05dcDys6xmR0PgJ2xi5nZosSYAQA4jlMJaXp1/nb9uvesJKlBeX999EBD1Qnh5zdQnDnsdWYBAPirEH8vffFYc33yYCP5ebpq+4kE3T15jcav2qfMbHZpAVBmAQB2zmKx6P5mFbRqREd1q1tGWTmGxq/ar7snr9GOEwlmxwNgMsosAMAhBPt56rNHmmlivyYqWcJNe+KTdM+Utfp4+R5lZOeYHQ+ASSizAACHYbFYdHejclo5oqN6NQxRjs3QlF8PqtfENYo9etHseABMQJkFADicIB8PTenfVNMebqogHw8dOJOs+6eu0wdLdis9i11awJlQZgEADqtH/RCtHN5B9zYpL5sh/Sf6kO6csFq/Hb5gdjQARYQyCwBwaCW93TWub2PNePQ2lfHzUNy5FPX5bL3e/nGnUjOzzY4HoJBRZgEAxULnOmW0YnhH9b0tVIYhfbXusLqPj9a6g+fMjgagEFFmAQDFhr+Xmz58oKG+ebyFygd46diFNPWfvlGvLdiu5Ax2aYHiiDILACh2OtQsrWXD2uvhVhUlSTM3HlX3cdGK2nfW5GQAChplFgBQLPl6uum9sAaa9VRLhQZ66cSlND36xSa9NG+bEtKyzI4HoIBQZgEAxVqbakFaPqyDHmtTWRaL9N3m4+o2Lko/7z5tdjQABYAyCwAo9kq4u+rtu+vpu3+3VtUgb51OzNATX2/W8LlbdTEl0+x4AG4BZRYA4DSaVw7UkqHt9XSHqrJapAWxJ9R1XLSW7ThldjQAN4kyCwBwKp5uLnq1Zx398Gwb1Qj20bnkDD3zbYzCZ8boXHKG2fEA5BNlFgDglJpULKmfhrTToNury8Vq0eLtp9RtXLR+3HZShmGYHQ/ADaLMAgCcloeri17oXkuLwtuqdllfXUjJ1JDZsXr6v1t0JjHd7HgAbgBlFgDg9OqX99ePg9ppeJeacnOxaOWu0+oyNkrzthxnlxawc5RZAAAkubtaNbRLDf1vcDs1KO+vxPRsvfD9Nj3+1W86lZBmdjwAV0GZBQDgL2qX9dOC59ropR615O5i1a97z6rb2GjN3nSUXVrADlFmAQD4G1cXq57rVF1LhrZTk4oBSsrI1sj52/XIjE06diHV7HgA/oIyCwDAVVQP9tW8Z9ro9V515OFq1ZoD59R9fLS+WX9YNhu7tIA9oMwCAHANLlaLnmxfVcuGdVCLyoFKzczRm4t2qt/0DTp8LsXseIDTo8wCAHADqgR5a87TrfTO3fVUwt1FG+MuqMeEaH2++pBy2KUFTEOZBQDgBlmtFj3aprKWD+ugNtVKKT3LpvcW79aD09bpwJlks+MBTokyCwBAPoUGltDMJ1vqg3sbyMfDVTFHL6nnxNWaGnlQ2Tk2s+MBToUyCwDATbBYLOrfsqJWDO+gjjVLKzPbpg+X7dF9U9dpb3yS2fEAp0GZBQDgFpQL8NJX/2qujx9oKD9PV/1+PEF3TVqtiT/vVxa7tECho8wCAHCLLBaLHrwtVCtHdFSXOmWUlWNo7Mp9unvyWu04kWB2PKBYo8wCAFBAyvh5avrAZprwUGOVLOGm3acSFTZlrT5ZsVcZ2TlmxwOKJcosAAAFyGKx6J7G5bVieEf1bFBW2TZDk345oN6T1mjbsUtmxwOKHcosAACFoLSvhz4d0EyfDmiqIB937TudrHs/XauIpbuVnsUuLVBQKLMAABSing1CtGJ4R4U1LiebIX0WdUg9J67WliMXzI4GFAuUWQAAClmgt7vGP9RE0wfepmBfDx06m6IHpq3Xu//bpdTMbLPjAQ6NMgsAQBHpWreMVg7vqAebVZBhSF+sjdOdE1Zr/cHzZkcDHBZlFgCAIuRfwk0fP9hIX/2ruUL8PXXkfKr6Td+gNxbuUHIGu7RAflFmAQAwQadawVoxvIP6t6woSfrvhiPqPi5aq/efNTkZ4FgoswAAmMTX000f3NtAM59sqQolvXTiUpoembFJr/zwuxLTs8yOBzgEyiwAACZrWz1Iy4d10GNtKkuS5vx2TN3GRuuXPafNDQY4AMosAAB2wNvDVW/fXU/f/bu1KpcqofjEdD3+1WaNmLtVl1IzzY4H2C3KLAAAdqRFlUAtHdpBT7WvIqtFmh97Ql3HRWv5znizowF2iTILAICd8XJ30Wu96mres21UrbS3ziZl6N//3aJBs2J0PjnD7HiAXaHMAgBgp5pWLKnFQ9rruU7V5GK16KffT6nbuGj99PtJGYZhdjzALlBmAQCwY55uLnqpR20tfK6tapf11fmUTA2aFatnvt2iM0npZscDTEeZBQDAATSo4K8fB7XT0M415Gq1aPnO0+o6NlrzY46zSwunRpkFAMBBuLtaNbxrTf04qJ3qlfNTQlqWRny3TU9+vVnxCezSwjlRZgEAcDB1y/lpYXhbvdi9ltxdrPp5zxl1HRulub8dZZcWTocyCwCAA3JzsSr89ur6aUg7NQoNUFJGtl7+YbsGfrFJxy+mmh0PKDKUWQAAHFjNMr6a/2wbvdqztjxcrVq9/5y6j4vWfzcckc3GLi2KP8osAAAOzsVq0dMdqmnp0PZqXrmkUjJz9MbCHer/+QYdOZ9idjygUFFmAQAoJqqW9tHcp1vrrd515eXmog2HLqjH+NX6Yk0cu7QotiizAAAUI1arRf9qW0XLh3VQ66qllJaVo3d/2qU+n63XwbPJZscDChxlFgCAYqhiqRKa+WRLvRdWX97uLtp85KJ6Tlitz6IOKoddWhQjlFkAAIopq9Wih1tV0ooRHdW+RpAysm2KWLpH901dp32nk8yOBxQIyiwAAMVc+QAvffN4C330QEP5erpq27FLumviGk3+Zb+ycmxmxwNuCWUWAAAnYLFY1Oe2UK0c3lGdawcrM8emMSv2KWzKWu06mWh2POCmUWYBAHAiZf099fmjt2l838YKKOGmnScTdffkNRq7cp8ys9mlheOhzAIA4GQsFovCmpTXiuEd1KNeWWXbDE38eb96T1qj349fMjsekC+UWQAAnFSwr6emPdJMU/o3VSlvd+09naSwKWs1eukepWflmB0PuCGUWQAAnFyvhiFaMbyD7m5UTjZDmhZ1UL0mrtaWIxfNjgZcF2UWAAColI+HJvZrov880kylfT108GyKHpi2Tu/9tEtpmezSwn5RZgEAQK5u9cpq5fAOur9pBRmG9PmaON05IVobD503OxpwRZRZAACQR0AJd33Sp5G+fKy5Qvw9dfh8qvr+Z4PeWrRDKRnZZscD8qDMAgCAK7q9drCWD++gfi1CJUlfrz+i7uOjtfbAOZOTAf+PMgsAAK7Kz9NNEfc11LdPtFT5AC8dv5imAZ9v1Mj525WYnmV2PIAyCwAArq9djSCtGN5BA1tXkiTN3nRU3cdF69e9Z0xOBmdHmQUAADfE28NV795TX3OebqVKpUroVEK6/vXlb3r+u21KSGWXFuagzAIAgHxpVbWUlg3toCfaVZHFIv0Qc1xdx0Vp5a7TZkeDE6LMAgCAfPNyd9Ebd9XVvGfaqFppb51JytBT32zWkNmxupCSaXY8OBHKLAAAuGnNKpXU4iHt9UzHarJapB+3nVTXsVFa/Psps6PBSVBmAQDALfF0c9Erd9bWgufaqlYZX51PyVT4rBg9++0WnU3KMDseijnKLAAAKBCNQgP04+C2GnJHdblaLVq6I15dx0VpYewJGYZhdjwUU5RZAABQYDxcXTSiWy0tGtRWdUP8dCk1S8PmbtVT32zW6cR0s+OhGKLMAgCAAlevnL8WDWqrF7rVlJuLRat2n1GXsVH6bvMxdmlRoCizAACgULi5WDXojhpaPKS9GlXwV1J6tl6a97se/fI3nbiUZnY8FBOUWQAAUKhqlvHVD8+20St31pa7q1XR+86q+7hozdx4hF1a3DLKLAAAKHSuLlY907Galg5tr2aVSio5I1uvLdihAZ9v1NHzqWbHgwOjzAIAgCJTrbSPvvt3a715V115ulm17uB5dR8fra/WxslmY5cW+UeZBQAARcrFatHj7apo+bAOalklUGlZOXr7f7vU9z/rFXcuxex4cDCUWQAAYIpKpbw1+6lWGhVWX97uLvrt8EX1GB+t6dGHlMMuLW4QZRYAAJjGarXokVaVtHx4B7WvEaSMbJveX7Jb909dp/2nk8yOBwdAmQUAAKarULKEvnm8hT68v4F8PVy19dgl9Zq4RlN+PaDsHJvZ8WDHKLMAAMAuWCwW9W1eUStGdNDttUorM8emj5fvVdina7X7VKLZ8WCnKLMAAMCuhPh76YvHmmtsn0by93LTjhOJunvyGo1ftU+Z2ezSIi/KLAAAsDsWi0X3Na2glcM7qFvdMsrKMTR+1X7dPXmNth9PMDse7AhlFgAA2K1gP0999kgzTerXRIHe7toTn6SwT9fqo2V7lJGdY3Y82AHKLAAAsGsWi0W9G5XTyuEddFfDEOXYDH0aeVC9Jq5R7NGLZseDySizAADAIZTy8dDk/k017eFmCvLx0IEzybp/6jq9v3iX0rPYpXVWlFkAAOBQetQvq1UjOui+JuVlM6Tpq+N054TV+u3wBbOjwQSUWQAA4HACSrhrbN/GmvHobSrj56G4cynq89l6vf3jTqVmZpsdD0XI1DIbERGh5s2by9fXV8HBwQoLC9PevXuv+Zjp06erffv2KlmypEqWLKkuXbpo06ZNRZQYAADYk851ymjF8I7qe1uoDEP6at1hdR8frXUHzpkdDUXE1DIbFRWl8PBwbdiwQStXrlRWVpa6deumlJSUqz4mMjJS/fr106+//qr169crNDRU3bp104kTJ4owOQAAsBf+Xm768IGG+ubxFiof4KVjF9LU//ONenXBdiWlZ5kdD4XMYhiGYXaIP509e1bBwcGKiopShw4dbugxOTk5KlmypCZPnqyBAwded31iYqL8/f2VkJAgPz+/W40MAADsSHJGtkYv3a1vNxyVJJXz91TE/Q3VsWZpk5MhP/LT1+xqZjYh4fJFkAMDA2/4MampqcrKysrXYwAAQPHk4+Gq98IaaPZTrVQxsIROJqTr0S826cXvtykhjV3a4shudmZtNpvuvvtuXbp0SWvWrLnhxz333HNavny5du7cKU9Pz398PyMjQxkZGblfJyYmKjQ0lJ1ZAACKudTMbH28fK++WndYhiGV8fPQB/c2UOc6ZcyOhutwyJ3Z8PBw7dixQ3PmzLnhx4wePVpz5szRggULrlhkpcsfMvP398+9hYaGFlRkAABgx0q4u+qt3vX0/b9bq2qQt04nZuiJrzdr2JxYXUzJNDseCohd7MwOGjRIixYtUnR0tKpUqXJDjxkzZozee+89rVq1SrfddttV17EzCwAA0rNyNG7lPk1ffUg2Qwrycdeoe+rrzgYhZkfDFeRnZ9bUMmsYhgYPHqwFCxYoMjJSNWrUuKHHffTRR3r//fe1fPlytWrVKl+vyQfAAABwXluPXdKL32/T/jPJkqReDUL0zj31FOTjYXIy/JXDjBmEh4fr22+/1axZs+Tr66v4+HjFx8crLS0td83AgQM1cuTI3K8//PBDvfHGG/riiy9UuXLl3MckJyeb8RYAAIADaRwaoJ+GtNPgO6rLxWrR4u2n1HVslBZtPSE7+Mtq3ARTd2YtFssV7//yyy/12GOPSZI6deqkypUr66uvvpIkVa5cWUeOHPnHY9566y29/fbb131NdmYBAIAk7TiRoBfn/a7dpxIlSV3rltH7YfUV7Hflz+Gg6DjMmIEZKLMAAOBPmdk2TYs6qEm/7FdWjiE/T1e92bue7m9a/qqbbih8DjNmAAAAYCZ3V6uGdK6h/w1up4YV/JWYnq0Xvt+mf331m04lpF3/CWA6yiwAAHB6tcv6af6zbfRyj9pyd7Uqcu9ZdRsbrdmbjjJLa+coswAAAJJcXax6tlM1LRnSTk0qBigpI1sj52/XwzM26tiFVLPj4SooswAAAH9RPdhX855po9d71ZGnm1VrD5xX9/HR+mb9Ydls7NLaG8osAADA37hYLXqyfVUtG9pBLaoEKjUzR28u2qmHpm/Q4XMpZsfDX1BmAQAArqJykLfmPNVK795TTyXcXbQp7oJ6TIjW56sPKYddWrtAmQUAALgGq9Wiga0ra/mwDmpXPUjpWTa9t3i3Hpy2TgfO8EubzEaZBQAAuAGhgSX03ydaKOK+BvLxcFXM0UvqOXG1pkYeVHaOzex4TosyCwAAcIMsFov6taioFcM7qFOt0srMtunDZXt039R12hOfaHY8p0SZBQAAyKdyAV768rHmGvNgI/l5uur34wnqPWmNJqzaryx2aYsUZRYAAOAmWCwWPdCsglaO6KgudcooK8fQuFX7dPfktdpxIsHseE6DMgsAAHALyvh5avrAZprwUGOVLOGm3acSdc+UtRqzfK8ysnPMjlfsUWYBAABukcVi0T2Ny2vliI7q1SBEOTZDk389oN6T1mjrsUtmxyvWKLMAAAAFJMjHQ1MGNNXUAU0V5OOufaeTdd+naxWxZLfSs9ilLQyUWQAAgAJ2Z4MQrRzeUWGNy8lmSJ9FH1LPCau15cgFs6MVO5RZAACAQlDS213jH2qizwfepjJ+Hjp0LkUPTFuvd/63U6mZ2WbHKzYoswAAAIWoS90yWjG8ox5sVkGGIX259rB6jF+t9QfPmx2tWKDMAgAAFDJ/Lzd9/GAjff14C5Xz99TRC6nqN32DXl+4XckZ7NLeCsosAABAEelYs7SWD++g/i0rSpK+3XBU3cdFK3rfWZOTOS7KLAAAQBHy9XTTB/c20KwnWyo00EsnLqVp4Beb9PK835WQlmV2PIdDmQUAADBBm+pBWja0gx5rU1mSNHfzMXUfF61f9pw2N5iDocwCAACYxNvDVW/fXU/f/bu1qgR5Kz4xXY9/tVkj5m7VpdRMs+M5BMosAACAyVpUCdSSIe31dIeqslqk+bEn1GVstJbtiDc7mt2jzAIAANgBL3cXvdqzjn54to2qB/voXHKGnvl2iwbNitH55Ayz49ktyiwAAIAdaVKxpH4a3E7ht1eTi9Win34/pa7jovW/bSdlGIbZ8ewOZRYAAMDOeLq56MXutbXwubaqXdZXF1IyNXh2rJ75dovOJKWbHc+uUGYBAADsVIMK/vpxUDsN61JDrlaLlu88ra5jozU/5ji7tH+gzAIAANgxd1erhnWpqf8Nbqf65f2UkJalEd9t0xNfb9aphDSz45mOMgsAAOAA6oT4aeFzbfVi91pyd7Hqlz1n1G1stOb+dtSpd2kpswAAAA7C1cWq8Nura/GQdmocGqCkjGy9/MN2Dfxik45fTDU7nikoswAAAA6mRhlf/fBsG73Ws448XK1avf+cuo+L1n83HJHN5ly7tJRZAAAAB+RiteipDlW1bFgHtagcqJTMHL2xcIf6f75BR86nmB2vyFBmAQAAHFiVIG/NebqV3u5dV15uLtpw6IJ6jF+tL9bEKccJdmkpswAAAA7OarXosbZVtHxYB7WpVkppWTl696dd6vPZeh08m2x2vEJFmQUAACgmKpYqoZlPttT799aXj4erthy5qJ4TVuuzqIPKzrGZHa9QUGYBAACKEYvFogEtK2n58A7qULO0MrJtili6R/dPXad9p5PMjlfgKLMAAADFUPkAL339r+b6+IGG8vV01bbjCeo1cbUm/bxfWcVol5YyCwAAUExZLBY9eFuoVo3oqC51gpWVY+iTlfsUNmWtdp5MMDtegaDMAgAAFHNl/Dw1feBtGt+3sQJKuGnnyUTdM3mtxq7Yq8xsx96lpcwCAAA4AYvForAm5bVyeEfdWb+ssm2GJv5yQL0nrdHvxy+ZHe+mUWYBAACcSGlfD019uJk+HdBUpbzdtfd0ksKmrNXopXuUnpVjdrx8o8wCAAA4oZ4NQrRyREfd3aicbIY0Leqgek1crS1HLpodLV8oswAAAE4q0NtdE/s10X8eaaZgXw8dPJuiB6at06ifdikt0zF2aSmzAAAATq5bvbJaObyj7m9aQYYhzVgTpx4TorXh0Hmzo10XZRYAAADyL+GmT/o00pf/aq4Qf08dOZ+qh/6zQW8u2qGUjGyz410VZRYAAAC5bq8VrOXDO6hfi1BJ0jfrj6j7+Git2X/O5GRXRpkFAABAHn6eboq4r6G+faKlKpT00vGLaXp4xkb9vPu02dH+gTILAACAK2pXI0jLh3XQo60rqV45P3WoWdrsSP/ganYAAAAA2C9vD1e9c099pWflyM3F/vZB7S8RAAAA7I6nm4vZEa6IMgsAAACHRZkFAACAw6LMAgAAwGFRZgEAAOCwKLMAAABwWJRZAAAAOCzKLAAAABwWZRYAAAAOizILAAAAh0WZBQAAgMOizAIAAMBhUWYBAADgsCizAAAAcFiUWQAAADgsyiwAAAAcFmUWAAAADosyCwAAAIdFmQUAAIDDoswCAADAYVFmAQAA4LAoswAAAHBYlFkAAAA4LMosAAAAHBZlFgAAAA7L1ewARc0wDElSYmKiyUkAAABwJX/2tD9727U4XZlNSkqSJIWGhpqcBAAAANeSlJQkf3//a66xGDdSeYsRm82mkydPytfXVxaLpdBfLzExUaGhoTp27Jj8/PwK/fUcCcfmyjguV8exuTKOy9VxbK6M43J1HJsrK+rjYhiGkpKSVK5cOVmt156KdbqdWavVqgoVKhT56/r5+fGH4io4NlfGcbk6js2VcVyujmNzZRyXq+PYXFlRHpfr7cj+iQ+AAQAAwGFRZgEAAOCwKLOFzMPDQ2+99ZY8PDzMjmJ3ODZXxnG5Oo7NlXFcro5jc2Ucl6vj2FyZPR8Xp/sAGAAAAIoPdmYBAADgsCizAAAAcFiUWQAAADgsyiwAAAAcFmX2JkyZMkWVK1eWp6enWrZsqU2bNl1z/ffff6/atWvL09NTDRo00JIlS/J83zAMvfnmmwoJCZGXl5e6dOmi/fv3F+ZbKBT5OS7Tp09X+/btVbJkSZUsWVJdunT5x/rHHntMFoslz61Hjx6F/TYKRX6OzVdfffWP9+3p6ZlnjTOeM506dfrHcbFYLOrVq1fumuJwzkRHR6t3794qV66cLBaLFi5ceN3HREZGqmnTpvLw8FD16tX11Vdf/WNNfn9u2aP8Hpv58+era9euKl26tPz8/NS6dWstX748z5q33377H+dM7dq1C/FdFLz8HpfIyMgr/lmKj4/Ps84Zz5kr/QyxWCyqV69e7pricM5ERESoefPm8vX1VXBwsMLCwrR3797rPs5e+wxlNp/mzp2rESNG6K233lJMTIwaNWqk7t2768yZM1dcv27dOvXr109PPPGEYmNjFRYWprCwMO3YsSN3zUcffaSJEydq2rRp2rhxo7y9vdW9e3elp6cX1du6Zfk9LpGRkerXr59+/fVXrV+/XqGhoerWrZtOnDiRZ12PHj106tSp3Nvs2bOL4u0UqPweG+nyb1j56/s+cuRInu874zkzf/78PMdkx44dcnFx0YMPPphnnaOfMykpKWrUqJGmTJlyQ+vj4uLUq1cv3X777dq6dauGDRumJ598Mk9pu5lz0B7l99hER0era9euWrJkibZs2aLbb79dvXv3VmxsbJ519erVy3POrFmzpjDiF5r8Hpc/7d27N8/7Dg4Ozv2es54zEyZMyHNMjh07psDAwH/8nHH0cyYqKkrh4eHasGGDVq5cqaysLHXr1k0pKSlXfYxd9xkD+dKiRQsjPDw89+ucnByjXLlyRkRExBXX9+nTx+jVq1ee+1q2bGn8+9//NgzDMGw2m1G2bFnj448/zv3+pUuXDA8PD2P27NmF8A4KR36Py99lZ2cbvr6+xtdff51736OPPmrcc889BR21yOX32Hz55ZeGv7//VZ+Pc+aycePGGb6+vkZycnLufcXlnPmTJGPBggXXXPPSSy8Z9erVy3Nf3759je7du+d+favH2h7dyLG5krp16xrvvPNO7tdvvfWW0ahRo4ILZrIbOS6//vqrIcm4ePHiVddwzly2YMECw2KxGIcPH869r7idM4ZhGGfOnDEkGVFRUVddY899hp3ZfMjMzNSWLVvUpUuX3PusVqu6dOmi9evXX/Ex69evz7Nekrp37567Pi4uTvHx8XnW+Pv7q2XLlld9TntzM8fl71JTU5WVlaXAwMA890dGRio4OFi1atXSs88+q/Pnzxdo9sJ2s8cmOTlZlSpVUmhoqO655x7t3Lkz93ucM5fNmDFDDz30kLy9vfPc7+jnTH5d72dMQRzr4sJmsykpKekfP2f279+vcuXKqWrVqhowYICOHj1qUsKi1bhxY4WEhKhr165au3Zt7v2cM/9vxowZ6tKliypVqpTn/uJ2ziQkJEjSP/5s/JU99xnKbD6cO3dOOTk5KlOmTJ77y5Qp849Zoz/Fx8dfc/2f/5uf57Q3N3Nc/u7ll19WuXLl8vwh6NGjh7755hv9/PPP+vDDDxUVFaU777xTOTk5BZq/MN3MsalVq5a++OILLVq0SN9++61sNpvatGmj48ePS+KckaRNmzZpx44devLJJ/PcXxzOmfy62s+YxMREpaWlFcifz+JizJgxSk5OVp8+fXLva9mypb766istW7ZMU6dOVVxcnNq3b6+kpCQTkxaukJAQTZs2TT/88IN++OEHhYaGqlOnToqJiZFUMD/Ti4OTJ09q6dKl//g5U9zOGZvNpmHDhqlt27aqX7/+VdfZc59xLdRnB27A6NGjNWfOHEVGRub5oNNDDz2U+88NGjRQw4YNVa1aNUVGRqpz585mRC0SrVu3VuvWrXO/btOmjerUqaPPPvtMo0aNMjGZ/ZgxY4YaNGigFi1a5LnfWc8ZXN+sWbP0zjvvaNGiRXlmQ++8887cf27YsKFatmypSpUq6bvvvtMTTzxhRtRCV6tWLdWqVSv36zZt2ujgwYMaN26c/vvf/5qYzL58/fXXCggIUFhYWJ77i9s5Ex4erh07djjc3O9fsTObD0FBQXJxcdHp06fz3H/69GmVLVv2io8pW7bsNdf/+b/5eU57czPH5U9jxozR6NGjtWLFCjVs2PCaa6tWraqgoCAdOHDgljMXlVs5Nn9yc3NTkyZNct+3s58zKSkpmjNnzg39n4YjnjP5dbWfMX5+fvLy8iqQc9DRzZkzR08++aS+++67f/w16d8FBASoZs2axfqcuZIWLVrkvmfOmcufyv/iiy/0yCOPyN3d/ZprHfmcGTRokH766Sf9+uuvqlChwjXX2nOfoczmg7u7u5o1a6aff/459z6bzaaff/45z07aX7Vu3TrPeklauXJl7voqVaqobNmyedYkJiZq48aNV31Oe3Mzx0W6/KnHUaNGadmyZbrtttuu+zrHjx/X+fPnFRISUiC5i8LNHpu/ysnJ0fbt23PftzOfM9LlS8NkZGTo4Ycfvu7rOOI5k1/X+xlTEOegI5s9e7b+9a9/afbs2Xku43Y1ycnJOnjwYLE+Z65k69atue/Z2c8Z6fKn/Q8cOHBD/9HsiOeMYRgaNGiQFixYoF9++UVVqlS57mPsus8U6sfLiqE5c+YYHh4exldffWXs2rXLePrpp42AgAAjPj7eMAzDeOSRR4xXXnkld/3atWsNV1dXY8yYMcbu3buNt956y3BzczO2b9+eu2b06NFGQECAsWjRIuP333837rnnHqNKlSpGWlpakb+/m5Xf4zJ69GjD3d3dmDdvnnHq1KncW1JSkmEYhpGUlGS88MILxvr16424uDhj1apVRtOmTY0aNWoY6enpprzHm5XfY/POO+8Yy5cvNw4ePGhs2bLFeOihhwxPT09j586duWuc8Zz5U7t27Yy+ffv+4/7ics4kJSUZsbGxRmxsrCHJGDt2rBEbG2scOXLEMAzDeOWVV4xHHnkkd/2hQ4eMEiVKGC+++KKxe/duY8qUKYaLi4uxbNmy3DXXO9aOIr/HZubMmYarq6sxZcqUPD9nLl26lLvm+eefNyIjI424uDhj7dq1RpcuXYygoCDjzJkzRf7+blZ+j8u4ceOMhQsXGvv37ze2b99uDB061LBarcaqVaty1zjrOfOnhx9+2GjZsuUVn7M4nDPPPvus4e/vb0RGRub5s5Gampq7xpH6DGX2JkyaNMmoWLGi4e7ubrRo0cLYsGFD7vc6duxoPProo3nWf/fdd0bNmjUNd3d3o169esbixYvzfN9msxlvvPGGUaZMGcPDw8Po3LmzsXfv3qJ4KwUqP8elUqVKhqR/3N566y3DMAwjNTXV6Natm1G6dGnDzc3NqFSpkvHUU0853A/SP+Xn2AwbNix3bZkyZYyePXsaMTExeZ7PGc8ZwzCMPXv2GJKMFStW/OO5iss58+dlk/5++/NYPProo0bHjh3/8ZjGjRsb7u7uRtWqVY0vv/zyH897rWPtKPJ7bDp27HjN9YZx+TJmISEhhru7u1G+fHmjb9++xoEDB4r2jd2i/B6XDz/80KhWrZrh6elpBAYGGp06dTJ++eWXfzyvM54zhnH5clJeXl7Gf/7znys+Z3E4Z650TCTl+dnhSH3G8sebAgAAABwOM7MAAABwWJRZAAAAOCzKLAAAABwWZRYAAAAOizILAAAAh0WZBQAAgMOizAIAAMBhUWYBwElFRkbKYrHo0qVLZkcBgJtGmQUAAIDDoswCAADAYVFmAcAkNptNERERqlKliry8vNSoUSPNmzdP0v+PACxevFgNGzaUp6enWrVqpR07duR5jh9++EH16tWTh4eHKleurE8++STP9zMyMvTyyy8rNDRUHh4eql69umbMmJFnzZYtW3TbbbepRIkSatOmjfbu3Vu4bxwAChBlFgBMEhERoW+++UbTpk3Tzp07NXz4cD388MOKiorKXfPiiy/qk08+0W+//abSpUurd+/eysrKknS5hPbp00cPPfSQtm/frrfffltvvPGGvvrqq9zHDxw4ULNnz9bEiRO1e/duffbZZ/Lx8cmT47XXXtMnn3yizZs3y9XVVY8//niRvH8AKAgWwzAMs0MAgLPJyMhQYGCgVq1apdatW+fe/+STTyo1NVVPP/20br/9ds2ZM0d9+/aVJF24cEEVKlTQV199pT59+mjAgAE6e/asVqxYkfv4l156SYsXL9bOnTu1b98+1apVSytXrlSXLl3+kSEyMlK33367Vq1apc6dO0uSlixZol69eiktLU2enp6FfBQA4NaxMwsAJjhw4IBSU1PVtWtX+fj45N6++eYbHTx4MHfdX4tuYGCgatWqpd27d0uSdu/erbZt2+Z53rZt22r//v3KycnR1q1b5eLioo4dO14zS8OGDXP/OSQkRJJ05syZW36PAFAUXM0OAADOKDk5WZK0ePFilS9fPs/3PDw88hTam+Xl5XVD69zc3HL/2WKxSLo8zwsAjoCdWQAwQd26deXh4aGjR4+qevXqeW6hoaG56zZs2JD7zxcvXtS+fftUp04dSVKdOnW0du3aPM+7du1a1axZUy4uLmrQoIFsNlueGVwAKG7YmQUAE/j6+uqFF17Q8OHDZbPZ1K5dOyUkJGjt2rXy8/NTpUqVJEnvvvuuSpUqpTJlyui1115TUFCQwsLCJEnPP/+8mjdvrlGjRqlv375av369Jk+erE8//VSSVLlyZT366KN6/PHHNXHiRDVq1EhHjhzRmTNn1KdPH7PeOgAUKMosAJhk1KhRKl26tCIiInTo0CEFBASoadOmevXVV3P/mn/06NEaOnSo9u/fr8aNG+t///uf3N3dJUlNmzbVd999pzfffFOjRo1SSEiI3n33XT322GO5rzF16lS9+uqreu6553T+/HlVrFhRr776qhlvFwAKBVczAAA79OeVBi5evKiAgACz4wCA3WJmFgAAAA6LMgsAAACHxZgBAAAAHBY7swAAAHBYlFkAAAA4LMosAAAAHBZlFgAAAA6LMgsAAACHRZkFAACAw6LMAgAAwGFRZgEAAOCwKLMAAABwWP8HzoAbeDbva6gAAAAASUVORK5CYII=\n" + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "source": [ + "# Load Best Model + Greedy Decode + Sample Predictions\n", + "trained_model = Transformer(\n", + " n_enc_vocab = n_enc_vocab,\n", + " n_dec_vocab = n_dec_vocab,\n", + " n_layers = n_layers,\n", + " pf_dim = pf_dim,\n", + " hid_dim = hid_dim,\n", + " n_heads = n_heads,\n", + " pe_source = 512,\n", + " pe_target = 512,\n", + " dropout = dropout\n", + ").to(device)\n", + "\n", + "trained_model.load_state_dict(torch.load('./checkpoints/transformermodel.pt'))\n", + "trained_model.eval()\n", + "\n", + "def evaluate(text):\n", + " text_ids = SRC_tokenizer.texts_to_sequences([text])\n", + " text_ids = pad_sequences(text_ids, maxlen=ENCODER_LEN, padding='post', truncating='post')\n", + "\n", + " sos_id = TRG_tokenizer.word_index['']\n", + " eos_id = TRG_tokenizer.word_index['']\n", + "\n", + " decoder_input = [sos_id]\n", + " input_tensor = torch.tensor(text_ids).to(device)\n", + " output_tensor = torch.tensor([decoder_input]).to(device)\n", + "\n", + " for i in range(DECODER_LEN):\n", + " with torch.no_grad():\n", + " predictions = trained_model(input_tensor, output_tensor)\n", + " predictions = predictions[:, -1:, :]\n", + "\n", + " predicted_id = torch.argmax(predictions[:,:,3:], axis=-1) + 3\n", + "\n", + " if int(predicted_id.item()) == eos_id:\n", + " break\n", + " output_tensor = torch.cat((output_tensor, predicted_id), -1)\n", + " return output_tensor\n", + "\n", + "def predict(text):\n", + " prediction = evaluate(text)\n", + " out_list = prediction.tolist()\n", + " out_list[0].pop(0) # drop \n", + " output_indexes = out_list[0]\n", + " predicted_sentence = TRG_tokenizer.sequences_to_texts([output_indexes])\n", + " return predicted_sentence\n", + "\n", + "for idx in (11, 21, 31, 41, 51):\n", + " print(\"Input :\", raw_src[idx])\n", + " print(\"Prediction :\", predict(raw_src[idx]))\n", + " print(\"Ground Truth :\", raw_trg[idx], \"\\n\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "3b-Y2ICx5B3C", + "outputId": "028dfb79-6914-4c68-f4ad-070509182027" + }, + "execution_count": 14, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Input : he said to her under his breath , i love you .\n", + "Prediction : ['je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je']\n", + "Ground Truth : il lui a dit a voix basse je t aime . \n", + "\n", + "Input : we were wakened by the whistle of the steam locomotive at dawn .\n", + "Prediction : ['je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je']\n", + "Ground Truth : nous avons ete reveilles a l aube par le sifflement d un train . \n", + "\n", + "Input : first of all , let me say how glad i am to be here .\n", + "Prediction : ['je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je']\n", + "Ground Truth : tout d abord laissez moi vous dire combien je suis heureux d etre ici . \n", + "\n", + "Input : this is a sentence that has the syllable count of a haiku .\n", + "Prediction : ['je je je je je je de de de de de de de de de de de de de de de de de de de de de de de de de de de de de de de de de de']\n", + "Ground Truth : on trouve en la phrase qui suit autant de syllabes qu en a un haiku . \n", + "\n", + "Input : i think we are the only people on this island .\n", + "Prediction : ['je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je je']\n", + "Ground Truth : je pense que nous sommes les seules personnes sur cette ile . \n", + "\n" + ] + } + ] + } + ] +} \ No newline at end of file