yo_parlai_8.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"yo_parlai_8.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyNR9NgPwYMFQaPHplF+IiCf"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"1DKbX_VjDGYr","executionInfo":{"status":"ok","timestamp":1636892170939,"user_tz":-60,"elapsed":47216,"user":{"displayName":"Tosin Adewumi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gg5Jzk7yK5iD3Qlys_LI6ytxZDPGvDKD_rFgZtE=s64","userId":"13705980824171128224"}},"outputId":"f3e6cbf5-4b66-482b-e0c7-6d546b5bc48f"},"source":["pip install parlai"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting parlai\n","  Downloading parlai-1.5.1-py3-none-any.whl (1.5 MB)\n","\u001b[K     |████████████████████████████████| 1.5 MB 4.2 MB/s \n","\u001b[?25hCollecting docutils<0.16,>=0.14\n","  Downloading docutils-0.15.2-py3-none-any.whl (547 kB)\n","\u001b[K     |████████████████████████████████| 547 kB 40.4 MB/s \n","\u001b[?25hCollecting datasets>=1.4.1\n","  Downloading datasets-1.15.1-py3-none-any.whl (290 kB)\n","\u001b[K     |████████████████████████████████| 290 kB 52.7 MB/s \n","\u001b[?25hCollecting Unidecode\n","  Downloading Unidecode-1.3.2-py3-none-any.whl (235 kB)\n","\u001b[K     |████████████████████████████████| 235 kB 39.7 MB/s \n","\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from parlai) (1.4.1)\n","Collecting tensorboardX\n","  Downloading tensorboardX-2.4-py2.py3-none-any.whl (124 kB)\n","\u001b[K     |████████████████████████████████| 124 kB 49.3 MB/s \n","\u001b[?25hCollecting tokenizers>=0.8.0\n","  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n","\u001b[K     |████████████████████████████████| 3.3 MB 36.8 MB/s \n","\u001b[?25hRequirement already satisfied: tornado in /usr/local/lib/python3.7/dist-packages (from parlai) (5.1.1)\n","Collecting Sphinx~=2.2.0\n","  Downloading Sphinx-2.2.2-py3-none-any.whl (2.7 MB)\n","\u001b[K     |████████████████████████████████| 2.7 MB 21.6 MB/s \n","\u001b[?25hRequirement already satisfied: pytest in /usr/local/lib/python3.7/dist-packages (from parlai) (3.6.4)\n","Requirement already satisfied: pyzmq in /usr/local/lib/python3.7/dist-packages (from parlai) (22.3.0)\n","Collecting emoji\n","  Downloading emoji-1.6.1.tar.gz (170 kB)\n","\u001b[K     |████████████████████████████████| 170 kB 53.5 MB/s \n","\u001b[?25hRequirement already satisfied: torchtext>=0.5.0 in /usr/local/lib/python3.7/dist-packages (from parlai) (0.11.0)\n","Requirement already satisfied: Pillow in /usr/local/lib/python3.7/dist-packages (from parlai) (7.1.2)\n","Requirement already satisfied: ipython in /usr/local/lib/python3.7/dist-packages (from parlai) (5.5.0)\n","Collecting requests-mock\n","  Downloading requests_mock-1.9.3-py2.py3-none-any.whl (27 kB)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from parlai) (3.10.0.2)\n","Collecting urllib3>=1.26.5\n","  Downloading urllib3-1.26.7-py2.py3-none-any.whl (138 kB)\n","\u001b[K     |████████████████████████████████| 138 kB 49.9 MB/s \n","\u001b[?25hCollecting omegaconf~=2.1.1\n","  Downloading omegaconf-2.1.1-py3-none-any.whl (74 kB)\n","\u001b[K     |████████████████████████████████| 74 kB 3.1 MB/s \n","\u001b[?25hCollecting sphinx-rtd-theme\n","  Downloading sphinx_rtd_theme-1.0.0-py2.py3-none-any.whl (2.8 MB)\n","\u001b[K     |████████████████████████████████| 2.8 MB 20.6 MB/s \n","\u001b[?25hCollecting myst-parser~=0.12.2\n","  Downloading myst_parser-0.12.10-py3-none-any.whl (34 kB)\n","Collecting websocket-client\n","  Downloading websocket_client-1.2.1-py2.py3-none-any.whl (52 kB)\n","\u001b[K     |████████████████████████████████| 52 kB 1.4 MB/s \n","\u001b[?25hCollecting gitdb2\n","  Downloading gitdb2-4.0.2-py3-none-any.whl (1.1 kB)\n","Collecting flake8-bugbear\n","  Downloading flake8_bugbear-21.9.2-py36.py37.py38-none-any.whl (16 kB)\n","Collecting attrs~=20.2.0\n","  Downloading attrs-20.2.0-py2.py3-none-any.whl (48 kB)\n","\u001b[K     |████████████████████████████████| 48 kB 4.9 MB/s \n","\u001b[?25hCollecting py-rouge\n","  Downloading py_rouge-1.1-py3-none-any.whl (56 kB)\n","\u001b[K     |████████████████████████████████| 56 kB 4.6 MB/s \n","\u001b[?25hCollecting subword-nmt\n","  Downloading subword_nmt-0.3.7-py2.py3-none-any.whl (26 kB)\n","Collecting flake8\n","  Downloading flake8-4.0.1-py2.py3-none-any.whl (64 kB)\n","\u001b[K     |████████████████████████████████| 64 kB 2.4 MB/s \n","\u001b[?25hCollecting pytest-regressions\n","  Downloading pytest_regressions-2.2.0-py3-none-any.whl (18 kB)\n","Collecting coloredlogs\n","  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n","\u001b[K     |████████████████████████████████| 46 kB 3.7 MB/s \n","\u001b[?25hCollecting jsonlines\n","  Downloading jsonlines-2.0.0-py3-none-any.whl (6.3 kB)\n","Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from parlai) (0.22.2.post1)\n","Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from parlai) (1.1.5)\n","Requirement already satisfied: tqdm~=4.62.1 in /usr/local/lib/python3.7/dist-packages (from parlai) (4.62.3)\n","Collecting sh\n","  Downloading sh-1.14.2-py2.py3-none-any.whl (40 kB)\n","\u001b[K     |████████████████████████████████| 40 kB 5.5 MB/s \n","\u001b[?25hCollecting iopath~=0.1.8\n","  Downloading iopath-0.1.9-py3-none-any.whl (27 kB)\n","Collecting fairscale\n","  Downloading fairscale-0.4.2.tar.gz (222 kB)\n","\u001b[K     |████████████████████████████████| 222 kB 57.1 MB/s \n","\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n","  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n","  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n","    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n","Requirement already satisfied: torch>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from parlai) (1.10.0+cu111)\n","Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from parlai) (2.23.0)\n","Collecting hydra-core~=1.1.0\n","  Downloading hydra_core-1.1.1-py3-none-any.whl (145 kB)\n","\u001b[K     |████████████████████████████████| 145 kB 45.5 MB/s \n","\u001b[?25hCollecting botocore\n","  Downloading botocore-1.23.5-py3-none-any.whl (8.1 MB)\n","\u001b[K     |████████████████████████████████| 8.1 MB 38.9 MB/s \n","\u001b[?25hCollecting websocket-server\n","  Downloading websocket_server-0.6.0-py3-none-any.whl (7.3 kB)\n","Collecting sphinx-autodoc-typehints~=1.10.3\n","  Downloading sphinx_autodoc_typehints-1.10.3-py3-none-any.whl (8.4 kB)\n","Collecting boto3\n","  Downloading boto3-1.20.5-py3-none-any.whl (131 kB)\n","\u001b[K     |████████████████████████████████| 131 kB 36.1 MB/s \n","\u001b[?25hRequirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (from parlai) (3.2.5)\n","Collecting GitPython\n","  Downloading GitPython-3.1.24-py3-none-any.whl (180 kB)\n","\u001b[K     |████████████████████████████████| 180 kB 52.5 MB/s \n","\u001b[?25hRequirement already satisfied: tensorboard in /usr/local/lib/python3.7/dist-packages (from parlai) (2.7.0)\n","Collecting docformatter\n","  Downloading docformatter-1.4.tar.gz (208 kB)\n","\u001b[K     |████████████████████████████████| 208 kB 52.8 MB/s \n","\u001b[?25hCollecting py-gfm\n","  Downloading py_gfm-1.0.2-py2.py3-none-any.whl (15 kB)\n","Collecting importlib-metadata<4.3\n","  Downloading importlib_metadata-4.2.0-py3-none-any.whl (16 kB)\n","Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from parlai) (3.13)\n","Collecting regex>=2021.8.3\n","  Downloading regex-2021.11.10-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (749 kB)\n","\u001b[K     |████████████████████████████████| 749 kB 40.2 MB/s \n","\u001b[?25hRequirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from parlai) (1.1.0)\n","Requirement already satisfied: pexpect in /usr/local/lib/python3.7/dist-packages (from parlai) (4.8.0)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from parlai) (1.19.5)\n","Collecting fsspec[http]>=2021.05.0\n","  Downloading fsspec-2021.11.0-py3-none-any.whl (132 kB)\n","\u001b[K     |████████████████████████████████| 132 kB 44.1 MB/s \n","\u001b[?25hCollecting aiohttp\n","  Downloading aiohttp-3.8.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n","\u001b[K     |████████████████████████████████| 1.1 MB 40.7 MB/s \n","\u001b[?25hCollecting xxhash\n","  Downloading xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243 kB)\n","\u001b[K     |████████████████████████████████| 243 kB 51.0 MB/s \n","\u001b[?25hRequirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets>=1.4.1->parlai) (0.3.4)\n","Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets>=1.4.1->parlai) (21.2)\n","Collecting huggingface-hub<1.0.0,>=0.1.0\n","  Downloading huggingface_hub-0.1.2-py3-none-any.whl (59 kB)\n","\u001b[K     |████████████████████████████████| 59 kB 6.5 MB/s \n","\u001b[?25hRequirement already satisfied: pyarrow!=4.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets>=1.4.1->parlai) (3.0.0)\n","Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets>=1.4.1->parlai) (0.70.12.2)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets>=1.4.1->parlai) (3.3.2)\n","Requirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from hydra-core~=1.1.0->parlai) (5.4.0)\n","Collecting antlr4-python3-runtime==4.8\n","  Downloading antlr4-python3-runtime-4.8.tar.gz (112 kB)\n","\u001b[K     |████████████████████████████████| 112 kB 45.5 MB/s \n","\u001b[?25hCollecting pyyaml\n","  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n","\u001b[K     |████████████████████████████████| 596 kB 41.1 MB/s \n","\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata<4.3->parlai) (3.6.0)\n","Collecting portalocker\n","  Downloading portalocker-2.3.2-py2.py3-none-any.whl (15 kB)\n","Collecting markdown-it-py~=0.5.4\n","  Downloading markdown_it_py-0.5.8-py3-none-any.whl (110 kB)\n","\u001b[K     |████████████████████████████████| 110 kB 50.9 MB/s \n","\u001b[?25hRequirement already satisfied: pyparsing<3,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets>=1.4.1->parlai) (2.4.7)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->parlai) (2.10)\n","Collecting requests<3,>=2.21.0\n","  Downloading requests-2.26.0-py2.py3-none-any.whl (62 kB)\n","\u001b[K     |████████████████████████████████| 62 kB 868 kB/s \n","\u001b[?25hRequirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->parlai) (2.0.7)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->parlai) (2021.10.8)\n","Collecting sphinxcontrib-applehelp\n","  Downloading sphinxcontrib_applehelp-1.0.2-py2.py3-none-any.whl (121 kB)\n","\u001b[K     |████████████████████████████████| 121 kB 60.0 MB/s \n","\u001b[?25hRequirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.7/dist-packages (from Sphinx~=2.2.0->parlai) (2.1.0)\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from Sphinx~=2.2.0->parlai) (57.4.0)\n","Requirement already satisfied: Jinja2>=2.3 in /usr/local/lib/python3.7/dist-packages (from Sphinx~=2.2.0->parlai) (2.11.3)\n","Requirement already satisfied: babel!=2.0,>=1.3 in /usr/local/lib/python3.7/dist-packages (from Sphinx~=2.2.0->parlai) (2.9.1)\n","Collecting sphinxcontrib-devhelp\n","  Downloading sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl (84 kB)\n","\u001b[K     |████████████████████████████████| 84 kB 3.4 MB/s \n","\u001b[?25hRequirement already satisfied: sphinxcontrib-serializinghtml in /usr/local/lib/python3.7/dist-packages (from Sphinx~=2.2.0->parlai) (1.1.5)\n","Requirement already satisfied: imagesize in /usr/local/lib/python3.7/dist-packages (from Sphinx~=2.2.0->parlai) (1.3.0)\n","Requirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.7/dist-packages (from Sphinx~=2.2.0->parlai) (2.6.1)\n","Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.7/dist-packages (from Sphinx~=2.2.0->parlai) (0.7.12)\n","Collecting sphinxcontrib-htmlhelp\n","  Downloading sphinxcontrib_htmlhelp-2.0.0-py2.py3-none-any.whl (100 kB)\n","\u001b[K     |████████████████████████████████| 100 kB 10.3 MB/s \n","\u001b[?25hCollecting sphinxcontrib-jsmath\n","  Downloading sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl (5.1 kB)\n","Collecting sphinxcontrib-qthelp\n","  Downloading sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl (90 kB)\n","\u001b[K     |████████████████████████████████| 90 kB 10.4 MB/s \n","\u001b[?25hRequirement already satisfied: pytz>=2015.7 in /usr/local/lib/python3.7/dist-packages (from babel!=2.0,>=1.3->Sphinx~=2.2.0->parlai) (2018.9)\n","Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2>=2.3->Sphinx~=2.2.0->parlai) (2.0.1)\n","Collecting async-timeout<5.0,>=4.0.0a3\n","  Downloading async_timeout-4.0.1-py3-none-any.whl (5.7 kB)\n","Collecting frozenlist>=1.1.1\n","  Downloading frozenlist-1.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (192 kB)\n","\u001b[K     |████████████████████████████████| 192 kB 52.5 MB/s \n","\u001b[?25hCollecting aiosignal>=1.1.2\n","  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n","Collecting asynctest==0.13.0\n","  Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\n","Collecting yarl<2.0,>=1.0\n","  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n","\u001b[K     |████████████████████████████████| 271 kB 42.4 MB/s \n","\u001b[?25hCollecting multidict<7.0,>=4.5\n","  Downloading multidict-5.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (160 kB)\n","\u001b[K     |████████████████████████████████| 160 kB 46.4 MB/s \n","\u001b[?25hCollecting jmespath<1.0.0,>=0.7.1\n","  Downloading jmespath-0.10.0-py2.py3-none-any.whl (24 kB)\n","Collecting s3transfer<0.6.0,>=0.5.0\n","  Downloading s3transfer-0.5.0-py3-none-any.whl (79 kB)\n","\u001b[K     |████████████████████████████████| 79 kB 8.1 MB/s \n","\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.7/dist-packages (from botocore->parlai) (2.8.2)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore->parlai) (1.15.0)\n","Collecting humanfriendly>=9.1\n","  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n","\u001b[K     |████████████████████████████████| 86 kB 5.3 MB/s \n","\u001b[?25hCollecting untokenize\n","  Downloading untokenize-0.1.1.tar.gz (3.1 kB)\n","Collecting pycodestyle<2.9.0,>=2.8.0\n","  Downloading pycodestyle-2.8.0-py2.py3-none-any.whl (42 kB)\n","\u001b[K     |████████████████████████████████| 42 kB 803 kB/s \n","\u001b[?25hCollecting pyflakes<2.5.0,>=2.4.0\n","  Downloading pyflakes-2.4.0-py2.py3-none-any.whl (69 kB)\n","\u001b[K     |████████████████████████████████| 69 kB 7.3 MB/s \n","\u001b[?25hCollecting mccabe<0.7.0,>=0.6.0\n","  Downloading mccabe-0.6.1-py2.py3-none-any.whl (8.6 kB)\n","Collecting gitdb>=4.0.1\n","  Downloading gitdb-4.0.9-py3-none-any.whl (63 kB)\n","\u001b[K     |████████████████████████████████| 63 kB 1.6 MB/s \n","\u001b[?25hCollecting smmap<6,>=3.0.1\n","  Downloading smmap-5.0.0-py3-none-any.whl (24 kB)\n","Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython->parlai) (0.7.5)\n","Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.7/dist-packages (from ipython->parlai) (5.1.1)\n","Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython->parlai) (0.8.1)\n","Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython->parlai) (1.0.18)\n","Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython->parlai) (4.4.2)\n","Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython->parlai) (0.2.5)\n","Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.7/dist-packages (from pexpect->parlai) (0.7.0)\n","Requirement already satisfied: markdown~=3.2 in /usr/local/lib/python3.7/dist-packages (from py-gfm->parlai) (3.3.4)\n","Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from pytest->parlai) (8.10.0)\n","Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.7/dist-packages (from pytest->parlai) (1.4.0)\n","Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.7/dist-packages (from pytest->parlai) (0.7.1)\n","Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from pytest->parlai) (1.11.0)\n","Collecting pytest-datadir>=1.2.0\n","  Downloading pytest_datadir-1.3.1-py2.py3-none-any.whl (5.9 kB)\n","Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard->parlai) (0.6.1)\n","Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorboard->parlai) (0.37.0)\n","Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard->parlai) (0.4.6)\n","Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard->parlai) (1.0.1)\n","Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard->parlai) (3.17.3)\n","Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard->parlai) (1.8.0)\n","Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard->parlai) (1.35.0)\n","Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard->parlai) (0.12.0)\n","Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard->parlai) (1.41.1)\n","Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard->parlai) (4.2.4)\n","Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard->parlai) (4.7.2)\n","Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard->parlai) (0.2.8)\n","Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard->parlai) (1.3.0)\n","Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard->parlai) (0.4.8)\n","Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard->parlai) (3.1.1)\n","Building wheels for collected packages: antlr4-python3-runtime, docformatter, emoji, fairscale, untokenize\n","  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.8-py3-none-any.whl size=141230 sha256=88edbdd45acb167be8386a7ce13dc11e0e74680e9f9324fa5db38ad214dcdde3\n","  Stored in directory: /root/.cache/pip/wheels/ca/33/b7/336836125fc9bb4ceaa4376d8abca10ca8bc84ddc824baea6c\n","  Building wheel for docformatter (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for docformatter: filename=docformatter-1.4-py3-none-any.whl size=12406 sha256=3a79aa852f2d841dd5476645d5f76f2e3ac6f263e169d4b541ff9864c2abe465\n","  Stored in directory: /root/.cache/pip/wheels/f7/62/81/a91b835e57388111fbe5f8c0cc99c5be6e257ba1fda172dd19\n","  Building wheel for emoji (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for emoji: filename=emoji-1.6.1-py3-none-any.whl size=169314 sha256=63bbbf4e384501fc41684dfdcca637dddbeb4aa4de4547bab5c13d11f8978183\n","  Stored in directory: /root/.cache/pip/wheels/ea/5f/d3/03d313ddb3c2a1a427bb4690f1621eea60fe6f2a30cc95940f\n","  Building wheel for fairscale (PEP 517) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for fairscale: filename=fairscale-0.4.2-py3-none-any.whl size=282740 sha256=0fd8ec2b27935e01a92d30e48ee82b33f4e5a37bd80a9531a160171fc552a0ff\n","  Stored in directory: /root/.cache/pip/wheels/3d/9c/6d/10e43b9036ee48c14424db261c519a4942d568c1793bda5f8b\n","  Building wheel for untokenize (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for untokenize: filename=untokenize-0.1.1-py3-none-any.whl size=2889 sha256=939a8afa5027b243eae1f9c93ae91357d3b8246ba52dcfd6734fee30bff7774b\n","  Stored in directory: /root/.cache/pip/wheels/5e/d4/22/3b662355e9a2faa5fe462c17b6fae2e9757066c36cd72c4497\n","Successfully built antlr4-python3-runtime docformatter emoji fairscale untokenize\n","Installing collected packages: urllib3, multidict, frozenlist, yarl, requests, jmespath, attrs, asynctest, async-timeout, aiosignal, sphinxcontrib-qthelp, sphinxcontrib-jsmath, sphinxcontrib-htmlhelp, sphinxcontrib-devhelp, sphinxcontrib-applehelp, smmap, pyyaml, pyflakes, pycodestyle, mccabe, importlib-metadata, fsspec, docutils, botocore, antlr4-python3-runtime, aiohttp, xxhash, untokenize, Sphinx, s3transfer, pytest-datadir, portalocker, omegaconf, markdown-it-py, humanfriendly, huggingface-hub, gitdb, flake8, websocket-server, websocket-client, Unidecode, tokenizers, tensorboardX, subword-nmt, sphinx-rtd-theme, sphinx-autodoc-typehints, sh, requests-mock, regex, pytest-regressions, py-rouge, py-gfm, myst-parser, jsonlines, iopath, hydra-core, GitPython, gitdb2, flake8-bugbear, fairscale, emoji, docformatter, datasets, coloredlogs, boto3, parlai\n","  Attempting uninstall: urllib3\n","    Found existing installation: urllib3 1.24.3\n","    Uninstalling urllib3-1.24.3:\n","      Successfully uninstalled urllib3-1.24.3\n","  Attempting uninstall: requests\n","    Found existing installation: requests 2.23.0\n","    Uninstalling requests-2.23.0:\n","      Successfully uninstalled requests-2.23.0\n","  Attempting uninstall: attrs\n","    Found existing installation: attrs 21.2.0\n","    Uninstalling attrs-21.2.0:\n","      Successfully uninstalled attrs-21.2.0\n","  Attempting uninstall: pyyaml\n","    Found existing installation: PyYAML 3.13\n","    Uninstalling PyYAML-3.13:\n","      Successfully uninstalled PyYAML-3.13\n","  Attempting uninstall: importlib-metadata\n","    Found existing installation: importlib-metadata 4.8.2\n","    Uninstalling importlib-metadata-4.8.2:\n","      Successfully uninstalled importlib-metadata-4.8.2\n","  Attempting uninstall: docutils\n","    Found existing installation: docutils 0.18\n","    Uninstalling docutils-0.18:\n","      Successfully uninstalled docutils-0.18\n","  Attempting uninstall: Sphinx\n","    Found existing installation: Sphinx 1.8.5\n","    Uninstalling Sphinx-1.8.5:\n","      Successfully uninstalled Sphinx-1.8.5\n","  Attempting uninstall: regex\n","    Found existing installation: regex 2019.12.20\n","    Uninstalling regex-2019.12.20:\n","      Successfully uninstalled regex-2019.12.20\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","google-colab 1.0.0 requires requests~=2.23.0, but you have requests 2.26.0 which is incompatible.\n","datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n","Successfully installed GitPython-3.1.24 Sphinx-2.2.2 Unidecode-1.3.2 aiohttp-3.8.0 aiosignal-1.2.0 antlr4-python3-runtime-4.8 async-timeout-4.0.1 asynctest-0.13.0 attrs-20.2.0 boto3-1.20.5 botocore-1.23.5 coloredlogs-15.0.1 datasets-1.15.1 docformatter-1.4 docutils-0.15.2 emoji-1.6.1 fairscale-0.4.2 flake8-4.0.1 flake8-bugbear-21.9.2 frozenlist-1.2.0 fsspec-2021.11.0 gitdb-4.0.9 gitdb2-4.0.2 huggingface-hub-0.1.2 humanfriendly-10.0 hydra-core-1.1.1 importlib-metadata-4.2.0 iopath-0.1.9 jmespath-0.10.0 jsonlines-2.0.0 markdown-it-py-0.5.8 mccabe-0.6.1 multidict-5.2.0 myst-parser-0.12.10 omegaconf-2.1.1 parlai-1.5.1 portalocker-2.3.2 py-gfm-1.0.2 py-rouge-1.1 pycodestyle-2.8.0 pyflakes-2.4.0 pytest-datadir-1.3.1 pytest-regressions-2.2.0 pyyaml-6.0 regex-2021.11.10 requests-2.26.0 requests-mock-1.9.3 s3transfer-0.5.0 sh-1.14.2 smmap-5.0.0 sphinx-autodoc-typehints-1.10.3 sphinx-rtd-theme-1.0.0 sphinxcontrib-applehelp-1.0.2 sphinxcontrib-devhelp-1.0.2 sphinxcontrib-htmlhelp-2.0.0 sphinxcontrib-jsmath-1.0.1 sphinxcontrib-qthelp-1.0.3 subword-nmt-0.3.7 tensorboardX-2.4 tokenizers-0.10.3 untokenize-0.1.1 urllib3-1.26.7 websocket-client-1.2.1 websocket-server-0.6.0 xxhash-2.0.2 yarl-1.7.2\n"]},{"output_type":"display_data","data":{"application/vnd.colab-display-data+json":{"pip_warning":{"packages":["pydevd_plugins","sphinxcontrib"]}}},"metadata":{}}]},{"cell_type":"code","metadata":{"id":"XsbNEJbqDK4y"},"source":["import argparse\n","from parlai.utils.io import PathManager\n","from parlai.core.teachers import register_teacher, DialogTeacher\n","from parlai.scripts.display_model import DisplayModel\n","from parlai.scripts.train_model import TrainModel"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"epTNTXiQDYaB"},"source":["data = \"parlai_combined.csv\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"AL80GgMlDkdU"},"source":["@register_teacher(\"yo_teacher\")\n","class YoTeacher(DialogTeacher):\n","    def __init__(self, opt, shared=None):\n","        self.datatype = opt['datatype']\n","        opt['datafile'] = data \n","        self.id = 'yodialog'\n","        super().__init__(opt, shared)\n","\n","\n","    def setup_data(self, path):\n","    # note that path is the value provided by opt['datafile']\n","        prompt = \"\"                 # initialize prompt message\n","        cnter = 0                   # initialize conversation counter\n","        line_skipper = False        # to skip 1st row of curated dailog data with header: line\n","        new_episode = False\n","        print('loading: ' + path)\n","        with PathManager.open(path) as data_file:\n","            for row in data_file.readlines():\n","                if line_skipper:\n","                    row_array = row.rstrip('\\n').split(';')\n","                    if cnter == 0:\n","                        prompt = row_array[1]\n","                        cnter += 1\n","                        new_episode = True if str.lower(row_array[0]) == 'true' else False\n","                    else:\n","                        yield {\"text\": prompt, \"labels\": row_array[1]}, new_episode\n","                        cnter = 0\n","                else:\n","                    line_skipper = True\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"OJaPsY3FDueN"},"source":["class DefaultTeacher(YoTeacher):\n","    pass"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"24WDBGx_Dx24","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1636896939874,"user_tz":-60,"elapsed":4497070,"user":{"displayName":"Tosin Adewumi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gg5Jzk7yK5iD3Qlys_LI6ytxZDPGvDKD_rFgZtE=s64","userId":"13705980824171128224"}},"outputId":"876ab37c-7ae7-47c6-8e14-51f3b5a23b3c"},"source":["TrainModel.main(\n","    # similar to before\n","    task='yo_teacher', \n","    model='transformer/generator',\n","    model_file='from_pretrained/model_yo',\n","    \n","    # initialize with a pretrained model\n","    init_model='zoo:tutorial_transformer_generator/model',\n","\n","    # BlenderBot 90M\n","    n_heads=16, n_layers=8, n_positions=512, text_truncate=512,\n","    label_truncate=128, ffn_size=2048, embedding_size=512,\n","    activation='gelu', variant='xlm',\n","    dict_lower=True, dict_tokenizer='bpe',\n","    dict_file='zoo:tutorial_transformer_generator/model.dict',\n","    learn_positional_embeddings=True,\n","    #dropout=0.1,\n","    #gradient_clip=0.1,\n","    #lr_scheduler='reduceonplateau',\n","\n","    # some training arguments, specific to this fine-tuning\n","    # use a small learning rate with ADAM optimizer 1e-5,\n","    lr=1e-05, optimizer='adam',\n","    warmup_updates=100,\n","    # early stopping on perplexity\n","    validation_metric='ppl',\n","    # train at most 10 minutes, and validate every 0.25 epochs\n","    max_train_time=20 * 60, validation_every_n_epochs=0.25,\n","    \n","    # depend on your gpu. If you have a V100, this is good\n","    batchsize=6, fp16=True, fp16_impl='mem_efficient',\n","    \n","    # speeds up validation\n","    skip_generation=False,\n","    \n","    # helps us cram more examples into our gpu at a time\n","    dynamic_batching='full',\n",")\n"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["12:20:42 | building dictionary first...\n","12:20:42 | No model with opt yet at: from_pretrained/model_yo(.opt)\n","12:20:42 | \u001b[33myour model is being loaded with opts that do not exist in the model you are initializing the weights with: allow_missing_init_opts: False,download_path: None,loglevel: info,dynamic_batching: full,verbose: False,is_debug: False,datapath: /usr/local/lib/python3.7/dist-packages/data,final_extra_opt: ,eval_dynamic_batching: None,num_workers: 0,max_train_steps: -1,log_every_n_steps: 50,validation_every_n_steps: -1,load_from_checkpoint: True,tensorboard_logdir: None,wandb_log: False,wandb_name: None,wandb_project: None,wandb_entity: None,mutators: None,n_encoder_layers: -1,n_decoder_layers: -1,model_parallel: False,checkpoint_activations: False,beam_block_full_context: True,beam_length_penalty: 0.65,topk: 10,topp: 0.9,beam_delay: 30,beam_block_list_filename: None,temperature: 1.0,compute_tokenized_bleu: False,interactive_mode: False,fp16_impl: mem_efficient,force_fp16_tokens: False,adafactor_eps: (1e-30, 0.001),history_reversed: False,history_add_global_end_token: None,special_tok_lst: None,bpe_vocab: None,bpe_merge: None,bpe_add_prefix_space: None,bpe_dropout: None,invsqrt_lr_decay_gamma: -1,parlai_home: /usr/local/lib/python3.7/dist-packages\u001b[0m\n","12:20:42 | \u001b[33myour model is being loaded with opts that differ from the model you are initializing the weights with. Add the following args to your run command to change this: \n","--show-advanced-args False --task internal:new_reddit:presorted --datatype train:stream --numthreads 1 --batchsize 48 --num-epochs 5.0 --max-train-time -1 --validation-every-n-secs 1800.0 --save-after-valid True --validation-every-n-epochs -1 --validation-max-exs 9920 --short-final-eval True --validation-patience 0 --validation-metric-mode min --dict-build-first True --numworkers 4 --pytorch-preprocess False --pytorch-teacher-batch-sort False --batch-sort-cache-type pop --batch-length-range 5 --shuffle False --batch-sort-field text --pytorch-context-length -1 --pytorch-include-labels True --log-every-n-secs 30.0 --distributed-world-size 64 --port 61337 --dropout 0.1 --beam-size 8 --beam-min-n-best 3 --beam-min-length 10 --inference beam --optimizer fused_adam --learningrate 0.0005 --gradient-clip 10.0 --adam-eps 1e-06 --betas 0.9,0.98 --weight-decay 0.01 --lr-scheduler invsqrt --warmup-updates 20000 --gpu 0 --beam-block-ngram 3 --beam-context-block-ngram 3\u001b[0m\n","12:20:42 | Using CUDA\n","12:20:42 | loading dictionary from /usr/local/lib/python3.7/dist-packages/data/models/tutorial_transformer_generator/model.dict\n","12:20:42 | num words = 54944\n","12:20:44 | Total parameters: 87,508,992 (87,508,992 trainable)\n","12:20:44 | Loading existing model params from /usr/local/lib/python3.7/dist-packages/data/models/tutorial_transformer_generator/model\n","12:20:45 | \u001b[33mDetected a fine-tune run. Resetting the optimizer.\u001b[0m\n","12:20:45 | \u001b[33mOptimizer was reset. Also resetting LR scheduler.\u001b[0m\n","12:20:45 | Opt:\n","12:20:45 |     activation: gelu\n","12:20:45 |     adafactor_eps: '(1e-30, 0.001)'\n","12:20:46 |     adam_eps: 1e-08\n","12:20:46 |     add_p1_after_newln: False\n","12:20:46 |     aggregate_micro: False\n","12:20:46 |     allow_missing_init_opts: False\n","12:20:46 |     attention_dropout: 0.0\n","12:20:46 |     batchsize: 6\n","12:20:46 |     beam_block_full_context: True\n","12:20:46 |     beam_block_list_filename: None\n","12:20:46 |     beam_block_ngram: -1\n","12:20:46 |     beam_context_block_ngram: -1\n","12:20:46 |     beam_delay: 30\n","12:20:46 |     beam_length_penalty: 0.65\n","12:20:46 |     beam_min_length: 1\n","12:20:46 |     beam_size: 1\n","12:20:46 |     betas: '(0.9, 0.999)'\n","12:20:46 |     bpe_add_prefix_space: None\n","12:20:46 |     bpe_debug: False\n","12:20:46 |     bpe_dropout: None\n","12:20:46 |     bpe_merge: None\n","12:20:46 |     bpe_vocab: None\n","12:20:46 |     checkpoint_activations: False\n","12:20:46 |     compute_tokenized_bleu: False\n","12:20:46 |     datapath: /usr/local/lib/python3.7/dist-packages/data\n","12:20:46 |     datatype: train\n","12:20:46 |     delimiter: '\\n'\n","12:20:46 |     dict_class: parlai.core.dict:DictionaryAgent\n","12:20:46 |     dict_endtoken: __end__\n","12:20:46 |     dict_file: /usr/local/lib/python3.7/dist-packages/data/models/tutorial_transformer_generator/model.dict\n","12:20:46 |     dict_include_test: False\n","12:20:46 |     dict_include_valid: False\n","12:20:46 |     dict_initpath: None\n","12:20:46 |     dict_language: english\n","12:20:46 |     dict_loaded: True\n","12:20:46 |     dict_lower: True\n","12:20:46 |     dict_max_ngram_size: -1\n","12:20:46 |     dict_maxexs: -1\n","12:20:46 |     dict_maxtokens: -1\n","12:20:46 |     dict_minfreq: 0\n","12:20:46 |     dict_nulltoken: __null__\n","12:20:46 |     dict_starttoken: __start__\n","12:20:46 |     dict_textfields: text,labels\n","12:20:46 |     dict_tokenizer: bpe\n","12:20:46 |     dict_unktoken: __unk__\n","12:20:46 |     display_examples: False\n","12:20:46 |     download_path: None\n","12:20:46 |     dropout: 0.0\n","12:20:46 |     dynamic_batching: full\n","12:20:46 |     embedding_projection: random\n","12:20:46 |     embedding_size: 512\n","12:20:46 |     embedding_type: random\n","12:20:46 |     embeddings_scale: True\n","12:20:46 |     eval_batchsize: None\n","12:20:46 |     eval_dynamic_batching: None\n","12:20:46 |     evaltask: None\n","12:20:46 |     ffn_size: 2048\n","12:20:46 |     final_extra_opt: \n","12:20:46 |     force_fp16_tokens: False\n","12:20:46 |     fp16: True\n","12:20:46 |     fp16_impl: mem_efficient\n","12:20:46 |     gpu: -1\n","12:20:46 |     gradient_clip: 0.1\n","12:20:46 |     hide_labels: False\n","12:20:46 |     history_add_global_end_token: None\n","12:20:46 |     history_reversed: False\n","12:20:46 |     history_size: -1\n","12:20:46 |     image_cropsize: 224\n","12:20:46 |     image_mode: raw\n","12:20:46 |     image_size: 256\n","12:20:46 |     inference: greedy\n","12:20:46 |     init_model: /usr/local/lib/python3.7/dist-packages/data/models/tutorial_transformer_generator/model\n","12:20:46 |     init_opt: None\n","12:20:46 |     interactive_mode: False\n","12:20:46 |     invsqrt_lr_decay_gamma: -1\n","12:20:46 |     is_debug: False\n","12:20:46 |     label_truncate: 128\n","12:20:46 |     learn_positional_embeddings: True\n","12:20:46 |     learningrate: 1e-05\n","12:20:46 |     load_from_checkpoint: True\n","12:20:46 |     log_every_n_secs: -1\n","12:20:46 |     log_every_n_steps: 50\n","12:20:46 |     loglevel: info\n","12:20:46 |     lr_scheduler: reduceonplateau\n","12:20:46 |     lr_scheduler_decay: 0.5\n","12:20:46 |     lr_scheduler_patience: 3\n","12:20:46 |     max_train_steps: -1\n","12:20:46 |     max_train_time: 1200.0\n","12:20:46 |     metrics: default\n","12:20:46 |     model: transformer/generator\n","12:20:46 |     model_file: from_pretrained/model_yo\n","12:20:46 |     model_parallel: False\n","12:20:46 |     momentum: 0\n","12:20:46 |     multitask_weights: [1]\n","12:20:46 |     mutators: None\n","12:20:46 |     n_decoder_layers: -1\n","12:20:46 |     n_encoder_layers: -1\n","12:20:46 |     n_heads: 16\n","12:20:46 |     n_layers: 8\n","12:20:46 |     n_positions: 512\n","12:20:46 |     n_segments: 0\n","12:20:46 |     nesterov: True\n","12:20:46 |     no_cuda: False\n","12:20:46 |     num_epochs: -1\n","12:20:46 |     num_workers: 0\n","12:20:46 |     nus: (0.7,)\n","12:20:46 |     optimizer: mem_eff_adam\n","12:20:46 |     output_scaling: 1.0\n","12:20:46 |     override: \"{'task': 'yo_teacher', 'model': 'transformer/generator', 'model_file': 'from_pretrained/model_yo', 'init_model': 'zoo:tutorial_transformer_generator/model', 'n_heads': 16, 'n_layers': 8, 'n_positions': 512, 'text_truncate': 512, 'label_truncate': 128, 'ffn_size': 2048, 'embedding_size': 512, 'activation': 'gelu', 'variant': 'xlm', 'dict_lower': True, 'dict_tokenizer': 'bpe', 'dict_file': '/usr/local/lib/python3.7/dist-packages/data/models/tutorial_transformer_generator/model.dict', 'learn_positional_embeddings': True, 'learningrate': 1e-05, 'optimizer': 'adam', 'warmup_updates': 100, 'validation_metric': 'ppl', 'max_train_time': 1200.0, 'validation_every_n_epochs': 0.25, 'batchsize': 6, 'fp16': True, 'fp16_impl': 'mem_efficient', 'skip_generation': False, 'dynamic_batching': 'full'}\"\n","12:20:46 |     parlai_home: /usr/local/lib/python3.7/dist-packages\n","12:20:46 |     person_tokens: False\n","12:20:46 |     rank_candidates: False\n","12:20:46 |     relu_dropout: 0.0\n","12:20:46 |     save_after_valid: False\n","12:20:46 |     save_every_n_secs: -1\n","12:20:46 |     share_word_embeddings: True\n","12:20:46 |     short_final_eval: False\n","12:20:46 |     skip_generation: False\n","12:20:46 |     special_tok_lst: None\n","12:20:46 |     split_lines: False\n","12:20:46 |     starttime: Nov14_12-20\n","12:20:46 |     task: yo_teacher\n","12:20:46 |     temperature: 1.0\n","12:20:46 |     tensorboard_log: False\n","12:20:46 |     tensorboard_logdir: None\n","12:20:46 |     text_truncate: 512\n","12:20:46 |     topk: 10\n","12:20:46 |     topp: 0.9\n","12:20:46 |     truncate: -1\n","12:20:46 |     update_freq: 1\n","12:20:46 |     use_reply: label\n","12:20:46 |     validation_cutoff: 1.0\n","12:20:46 |     validation_every_n_epochs: 0.25\n","12:20:46 |     validation_every_n_secs: -1\n","12:20:46 |     validation_every_n_steps: -1\n","12:20:46 |     validation_max_exs: -1\n","12:20:46 |     validation_metric: ppl\n","12:20:46 |     validation_metric_mode: None\n","12:20:46 |     validation_patience: 10\n","12:20:46 |     validation_share_agent: False\n","12:20:46 |     variant: xlm\n","12:20:46 |     verbose: False\n","12:20:46 |     wandb_entity: None\n","12:20:46 |     wandb_log: False\n","12:20:46 |     wandb_name: None\n","12:20:46 |     wandb_project: None\n","12:20:46 |     warmup_rate: 0.0001\n","12:20:46 |     warmup_updates: 100\n","12:20:46 |     weight_decay: None\n","12:20:46 | creating task(s): yo_teacher\n","loading: parlai_combined.csv\n","12:20:48 | training...\n","12:20:54 | Overflow: setting loss scale to 65536.0\n","12:20:59 | Overflow: setting loss scale to 32768.0\n","12:20:59 | time:11s total_exs:240 total_steps:2 epochs:0.32\n","    clen  clip  ctpb  ctps  ctrunc  ctrunclen  exps  exs  fp16_loss_scalar  gnorm  gpu_mem  llen  loss       lr  ltpb  ltps  \\\n","      23     0  2760 558.2       0          0 24.24  240             49152     -1    .2537     4 8.752 2.01e-07   480 97.08   \n","    ltrunc  ltrunclen  ppl  token_acc  token_em  total_train_updates  tpb   tps   ups  \n","         0          0 6322      .2500         0                    2 3240 655.3 .2023\n","\n","12:20:59 | creating task(s): yo_teacher\n","loading: parlai_combined.csv\n","12:21:01 | running eval: valid\n","12:39:32 | eval completed in 1110.97s\n","12:39:32 | \u001b[1mvalid:\n","    accuracy   bleu-4  clen  ctpb  ctps  ctrunc  ctrunclen  exps  exs     f1  gpu_mem  llen  loss       lr  ltpb  ltps  \\\n","           0 .0001246  7174 951.1   339   .9420       6678 .6832  759 .03009    .1428    27 5.041 2.01e-07 51.29 18.28   \n","    ltrunc  ltrunclen   ppl  token_acc  token_em  total_train_updates  tpb   tps  \n","   .003953      .2411 154.6      .2525         0                    2 1002 357.3\n","\u001b[0m\n","12:39:32 | \u001b[1;32mnew best ppl: 154.6\u001b[0m\n","12:39:32 | saving best valid model: from_pretrained/model_yo\n","12:39:32 | Saving dictionary to from_pretrained/model_yo.dict\n","12:39:40 | Overflow: setting loss scale to 16384.0\n","12:39:43 | Overflow: setting loss scale to 8192.0\n","12:39:48 | Overflow: setting loss scale to 4096.0\n","12:39:49 | time:1140s total_exs:472 total_steps:5 epochs:0.62\n","    clen  clip  ctpb  ctps  ctrunc  ctrunclen  exps  exs  fp16_loss_scalar  gnorm  gpu_mem  llen  loss       lr  ltpb  ltps  \\\n","   23.44     0  1812 424.4       0          0 18.11  232              9557     -1    .3450 19.49 5.245 5.01e-07  1507 352.9   \n","    ltrunc  ltrunclen   ppl  token_acc  token_em  total_train_updates  tpb   tps   ups  \n","         0          0 189.5      .2004         0                    5 3319 777.3 .2343\n","\n","12:39:49 | running eval: valid\n","12:58:13 | eval completed in 1104.63s\n","12:58:13 | \u001b[1mvalid:\n","    accuracy   bleu-4  clen  ctpb  ctps  ctrunc  ctrunclen  exps  exs     f1  gpu_mem  llen  loss       lr  ltpb  ltps  \\\n","           0 .0001246  7174 951.1   341   .9420       6678 .6872  759 .03009    .1433    27 5.041 5.01e-07 51.29 18.39   \n","    ltrunc  ltrunclen   ppl  token_acc  token_em  total_train_updates  tpb   tps  \n","   .003953      .2411 154.6      .2525         0                    5 1002 359.4\n","\u001b[0m\n","12:58:13 | \u001b[1mdid not beat best ppl: 154.5705 impatience: 1\u001b[0m\n","12:58:17 | Overflow: setting loss scale to 2048.0\n","12:58:17 | max_train_time elapsed:2249.374839782715s\n","12:58:18 | \u001b[33mOverriding opt[\"init_model\"] to zoo:tutorial_transformer_generator/model (previously: /usr/local/lib/python3.7/dist-packages/data/models/tutorial_transformer_generator/model)\u001b[0m\n","12:58:18 | \u001b[33mOverriding opt[\"optimizer\"] to adam (previously: mem_eff_adam)\u001b[0m\n","12:58:18 | \u001b[33myour model is being loaded with opts that do not exist in the model you are initializing the weights with: allow_missing_init_opts: False,loglevel: info,dynamic_batching: full,is_debug: False,final_extra_opt: ,eval_dynamic_batching: None,num_workers: 0,max_train_steps: -1,log_every_n_steps: 50,validation_every_n_steps: -1,tensorboard_logdir: None,wandb_log: False,wandb_name: None,wandb_project: None,wandb_entity: None,mutators: None,n_encoder_layers: -1,n_decoder_layers: -1,model_parallel: False,checkpoint_activations: False,beam_block_full_context: True,beam_length_penalty: 0.65,topk: 10,topp: 0.9,beam_delay: 30,beam_block_list_filename: None,temperature: 1.0,compute_tokenized_bleu: False,fp16_impl: mem_efficient,force_fp16_tokens: True,adafactor_eps: 1e-30,0.001,history_reversed: False,history_add_global_end_token: None,special_tok_lst: None,bpe_vocab: None,bpe_merge: None,bpe_add_prefix_space: None,bpe_dropout: None,invsqrt_lr_decay_gamma: -1,parlai_home: /usr/local/lib/python3.7/dist-packages,dict_loaded: True,download_path: None,verbose: False,datapath: /usr/local/lib/python3.7/dist-packages/data,load_from_checkpoint: True,interactive_mode: False\u001b[0m\n","12:58:18 | \u001b[33myour model is being loaded with opts that differ from the model you are initializing the weights with. Add the following args to your run command to change this: \n","--show-advanced-args False --task internal:new_reddit:presorted --datatype train:stream --numthreads 1 --batchsize 48 --num-epochs 5.0 --max-train-time -1 --validation-every-n-secs 1800.0 --save-after-valid True --validation-every-n-epochs -1 --validation-max-exs 9920 --short-final-eval True --validation-patience 0 --validation-metric-mode min --dict-build-first True --numworkers 4 --pytorch-preprocess False --pytorch-teacher-batch-sort False --batch-sort-cache-type pop --batch-length-range 5 --shuffle False --batch-sort-field text --pytorch-context-length -1 --pytorch-include-labels True --log-every-n-secs 30.0 --distributed-world-size 64 --port 61337 --dropout 0.1 --beam-size 8 --beam-min-n-best 3 --beam-min-length 10 --inference beam --optimizer fused_adam --learningrate 0.0005 --gradient-clip 10.0 --adam-eps 1e-06 --betas 0.9,0.98 --weight-decay 0.01 --lr-scheduler invsqrt --warmup-updates 20000 --gpu 0 --beam-block-ngram 3 --beam-context-block-ngram 3\u001b[0m\n","12:58:18 | Using CUDA\n","12:58:18 | loading dictionary from from_pretrained/model_yo.dict\n","12:58:18 | num words = 54944\n","12:58:19 | Total parameters: 87,508,992 (87,508,992 trainable)\n","12:58:19 | Loading existing model params from from_pretrained/model_yo\n","12:58:23 | creating task(s): yo_teacher\n","loading: parlai_combined.csv\n","12:58:24 | running eval: valid\n","13:17:02 | eval completed in 1117.71s\n","13:17:02 | \u001b[1mvalid:\n","    accuracy   bleu-4  clen  ctpb  ctps  ctrunc  ctrunclen  exps  exs     f1  gpu_mem  llen  loss       lr  ltpb  ltps  \\\n","           0 .0001246  7174 951.1   337   .9420       6678 .6791  759 .03009    .2449    27 5.041 2.01e-07 51.29 18.17   \n","    ltrunc  ltrunclen   ppl  token_acc  token_em  total_train_updates  tpb   tps  \n","   .003953      .2411 154.6      .2525         0                    2 1002 355.2\n","\u001b[0m\n","13:17:02 | creating task(s): yo_teacher\n","loading: parlai_combined.csv\n","13:17:03 | running eval: test\n","13:35:39 | eval completed in 1115.33s\n","13:35:39 | \u001b[1mtest:\n","    accuracy   bleu-4  clen  ctpb  ctps  ctrunc  ctrunclen  exps  exs     f1  gpu_mem  llen  loss       lr  ltpb  ltps  \\\n","           0 .0001246  7174 951.1 337.7   .9420       6678 .6807  759 .03009    .2450    27 5.041 2.01e-07 51.29 18.21   \n","    ltrunc  ltrunclen   ppl  token_acc  token_em  total_train_updates  tpb  tps  \n","   .003953      .2411 154.6      .2525         0                    2 1002  356\n","\u001b[0m\n"]},{"output_type":"execute_result","data":{"text/plain":["({'accuracy': ExactMatchMetric(0),\n","  'bleu-4': BleuMetric(0.0001246),\n","  'clen': AverageMetric(7174),\n","  'ctpb': GlobalAverageMetric(951.1),\n","  'ctps': GlobalTimerMetric(337),\n","  'ctrunc': AverageMetric(0.942),\n","  'ctrunclen': AverageMetric(6678),\n","  'exps': GlobalTimerMetric(0.6791),\n","  'exs': SumMetric(759),\n","  'f1': F1Metric(0.03009),\n","  'gpu_mem': GlobalAverageMetric(0.2449),\n","  'llen': AverageMetric(27),\n","  'loss': AverageMetric(5.041),\n","  'lr': GlobalAverageMetric(2.01e-07),\n","  'ltpb': GlobalAverageMetric(51.29),\n","  'ltps': GlobalTimerMetric(18.17),\n","  'ltrunc': AverageMetric(0.003953),\n","  'ltrunclen': AverageMetric(0.2411),\n","  'ppl': PPLMetric(154.6),\n","  'token_acc': AverageMetric(0.2525),\n","  'token_em': AverageMetric(0),\n","  'total_train_updates': GlobalFixedMetric(2),\n","  'tpb': GlobalAverageMetric(1002),\n","  'tps': GlobalTimerMetric(355.2)},\n"," {'accuracy': ExactMatchMetric(0),\n","  'bleu-4': BleuMetric(0.0001246),\n","  'clen': AverageMetric(7174),\n","  'ctpb': GlobalAverageMetric(951.1),\n","  'ctps': GlobalTimerMetric(337.7),\n","  'ctrunc': AverageMetric(0.942),\n","  'ctrunclen': AverageMetric(6678),\n","  'exps': GlobalTimerMetric(0.6807),\n","  'exs': SumMetric(759),\n","  'f1': F1Metric(0.03009),\n","  'gpu_mem': GlobalAverageMetric(0.245),\n","  'llen': AverageMetric(27),\n","  'loss': AverageMetric(5.041),\n","  'lr': GlobalAverageMetric(2.01e-07),\n","  'ltpb': GlobalAverageMetric(51.29),\n","  'ltps': GlobalTimerMetric(18.21),\n","  'ltrunc': AverageMetric(0.003953),\n","  'ltrunclen': AverageMetric(0.2411),\n","  'ppl': PPLMetric(154.6),\n","  'token_acc': AverageMetric(0.2525),\n","  'token_em': AverageMetric(0),\n","  'total_train_updates': GlobalFixedMetric(2),\n","  'tpb': GlobalAverageMetric(1002),\n","  'tps': GlobalTimerMetric(356)})"]},"metadata":{},"execution_count":7}]},{"cell_type":"code","metadata":{"id":"YrN6WfZKD3oK","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1636897063771,"user_tz":-60,"elapsed":11749,"user":{"displayName":"Tosin Adewumi","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Gg5Jzk7yK5iD3Qlys_LI6ytxZDPGvDKD_rFgZtE=s64","userId":"13705980824171128224"}},"outputId":"93b4ee11-d628-4625-b629-35e09a370ba1"},"source":["DisplayModel.main(task='yo_teacher', model_file='from_pretrained/model_yo', num_examples=3, skip_generation=False)"],"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["13:37:31 | Using CUDA\n","13:37:31 | loading dictionary from from_pretrained/model_yo.dict\n","13:37:32 | num words = 54944\n","13:37:33 | Total parameters: 87,508,992 (87,508,992 trainable)\n","13:37:33 | Loading existing model params from from_pretrained/model_yo\n","13:37:36 | creating task(s): yo_teacher\n","loading: parlai_combined.csv\n","13:37:36 | Opt:\n","13:37:36 |     activation: gelu\n","13:37:36 |     adafactor_eps: '[1e-30, 0.001]'\n","13:37:36 |     adam_eps: 1e-08\n","13:37:36 |     add_p1_after_newln: False\n","13:37:36 |     aggregate_micro: False\n","13:37:36 |     allow_missing_init_opts: False\n","13:37:36 |     attention_dropout: 0.0\n","13:37:36 |     batchsize: 6\n","13:37:36 |     beam_block_full_context: True\n","13:37:36 |     beam_block_list_filename: None\n","13:37:36 |     beam_block_ngram: -1\n","13:37:36 |     beam_context_block_ngram: -1\n","13:37:36 |     beam_delay: 30\n","13:37:36 |     beam_length_penalty: 0.65\n","13:37:36 |     beam_min_length: 1\n","13:37:36 |     beam_size: 1\n","13:37:36 |     betas: '[0.9, 0.999]'\n","13:37:36 |     bpe_add_prefix_space: None\n","13:37:36 |     bpe_debug: False\n","13:37:36 |     bpe_dropout: None\n","13:37:36 |     bpe_merge: None\n","13:37:36 |     bpe_vocab: None\n","13:37:36 |     checkpoint_activations: False\n","13:37:36 |     compute_tokenized_bleu: False\n","13:37:36 |     datapath: /usr/local/lib/python3.7/dist-packages/data\n","13:37:36 |     datatype: train\n","13:37:36 |     delimiter: '\\n'\n","13:37:36 |     dict_class: parlai.core.dict:DictionaryAgent\n","13:37:36 |     dict_endtoken: __end__\n","13:37:36 |     dict_file: from_pretrained/model_yo.dict\n","13:37:36 |     dict_include_test: False\n","13:37:36 |     dict_include_valid: False\n","13:37:36 |     dict_initpath: None\n","13:37:36 |     dict_language: english\n","13:37:36 |     dict_loaded: True\n","13:37:36 |     dict_lower: True\n","13:37:36 |     dict_max_ngram_size: -1\n","13:37:36 |     dict_maxexs: -1\n","13:37:36 |     dict_maxtokens: -1\n","13:37:36 |     dict_minfreq: 0\n","13:37:36 |     dict_nulltoken: __null__\n","13:37:36 |     dict_starttoken: __start__\n","13:37:36 |     dict_textfields: text,labels\n","13:37:36 |     dict_tokenizer: bpe\n","13:37:36 |     dict_unktoken: __unk__\n","13:37:36 |     display_add_fields: \n","13:37:36 |     display_examples: False\n","13:37:36 |     download_path: None\n","13:37:36 |     dropout: 0.0\n","13:37:36 |     dynamic_batching: full\n","13:37:36 |     embedding_projection: random\n","13:37:36 |     embedding_size: 512\n","13:37:36 |     embedding_type: random\n","13:37:36 |     embeddings_scale: True\n","13:37:36 |     eval_batchsize: None\n","13:37:36 |     eval_dynamic_batching: None\n","13:37:36 |     evaltask: None\n","13:37:36 |     ffn_size: 2048\n","13:37:36 |     final_extra_opt: \n","13:37:36 |     force_fp16_tokens: True\n","13:37:36 |     fp16: True\n","13:37:36 |     fp16_impl: mem_efficient\n","13:37:36 |     gpu: -1\n","13:37:36 |     gradient_clip: 0.1\n","13:37:36 |     hide_labels: False\n","13:37:36 |     history_add_global_end_token: None\n","13:37:36 |     history_reversed: False\n","13:37:36 |     history_size: -1\n","13:37:36 |     image_cropsize: 224\n","13:37:36 |     image_mode: raw\n","13:37:36 |     image_size: 256\n","13:37:36 |     inference: greedy\n","13:37:36 |     init_model: /usr/local/lib/python3.7/dist-packages/data/models/tutorial_transformer_generator/model\n","13:37:36 |     init_opt: None\n","13:37:36 |     interactive_mode: False\n","13:37:36 |     invsqrt_lr_decay_gamma: -1\n","13:37:36 |     is_debug: False\n","13:37:36 |     label_truncate: 128\n","13:37:36 |     learn_positional_embeddings: True\n","13:37:36 |     learningrate: 1e-05\n","13:37:36 |     log_every_n_secs: -1\n","13:37:36 |     log_every_n_steps: 50\n","13:37:36 |     loglevel: info\n","13:37:36 |     lr_scheduler: reduceonplateau\n","13:37:36 |     lr_scheduler_decay: 0.5\n","13:37:36 |     lr_scheduler_patience: 3\n","13:37:36 |     max_train_steps: -1\n","13:37:36 |     max_train_time: 1200.0\n","13:37:36 |     metrics: default\n","13:37:36 |     model: transformer/generator\n","13:37:36 |     model_file: from_pretrained/model_yo\n","13:37:36 |     model_parallel: False\n","13:37:36 |     momentum: 0\n","13:37:36 |     multitask_weights: [1]\n","13:37:36 |     mutators: None\n","13:37:36 |     n_decoder_layers: -1\n","13:37:36 |     n_encoder_layers: -1\n","13:37:36 |     n_heads: 16\n","13:37:36 |     n_layers: 8\n","13:37:36 |     n_positions: 512\n","13:37:36 |     n_segments: 0\n","13:37:36 |     nesterov: True\n","13:37:36 |     no_cuda: False\n","13:37:36 |     num_epochs: -1\n","13:37:36 |     num_examples: 3\n","13:37:36 |     num_workers: 0\n","13:37:36 |     nus: [0.7]\n","13:37:36 |     optimizer: mem_eff_adam\n","13:37:36 |     output_scaling: 1.0\n","13:37:36 |     override: \"{'task': 'yo_teacher', 'model_file': 'from_pretrained/model_yo', 'num_examples': '3', 'skip_generation': False}\"\n","13:37:36 |     parlai_home: /usr/local/lib/python3.7/dist-packages\n","13:37:36 |     person_tokens: False\n","13:37:36 |     rank_candidates: False\n","13:37:36 |     relu_dropout: 0.0\n","13:37:36 |     save_after_valid: False\n","13:37:36 |     save_every_n_secs: -1\n","13:37:36 |     share_word_embeddings: True\n","13:37:36 |     short_final_eval: False\n","13:37:36 |     skip_generation: False\n","13:37:36 |     special_tok_lst: None\n","13:37:36 |     split_lines: False\n","13:37:36 |     starttime: Nov14_12-20\n","13:37:36 |     task: yo_teacher\n","13:37:36 |     temperature: 1.0\n","13:37:36 |     tensorboard_log: False\n","13:37:36 |     tensorboard_logdir: None\n","13:37:36 |     text_truncate: 512\n","13:37:36 |     topk: 10\n","13:37:36 |     topp: 0.9\n","13:37:36 |     truncate: -1\n","13:37:36 |     update_freq: 1\n","13:37:36 |     use_reply: label\n","13:37:36 |     validation_cutoff: 1.0\n","13:37:36 |     validation_every_n_epochs: 0.25\n","13:37:36 |     validation_every_n_secs: -1\n","13:37:36 |     validation_every_n_steps: -1\n","13:37:36 |     validation_max_exs: -1\n","13:37:36 |     validation_metric: ppl\n","13:37:36 |     validation_metric_mode: None\n","13:37:36 |     validation_patience: 10\n","13:37:36 |     validation_share_agent: False\n","13:37:36 |     variant: xlm\n","13:37:36 |     verbose: False\n","13:37:36 |     wandb_entity: None\n","13:37:36 |     wandb_log: False\n","13:37:36 |     wandb_name: None\n","13:37:36 |     wandb_project: None\n","13:37:36 |     warmup_rate: 0.0001\n","13:37:36 |     warmup_updates: 100\n","13:37:36 |     weight_decay: None\n","\u001b[1;31m- - - NEW EPISODE: yodialog- - -\u001b[0;0m\n","\u001b[0mMo nílò ààyè láti jẹun ní ààrín ìlú tí kò gbówólórí \u001b[0;0m\n","\u001b[1;94m    labels: \"Mo ní awọ́n àṣàyàn púpọ̀ fún ọ  ṣé o fẹ́ràn óunjẹ Áfírikà, Ásíà, tàbí ìlú Gẹ̀ẹ́sì?\" \u001b[0;0m\n","\u001b[0;95m     model: i ' m not sure if you ' re joking or not , but i ' m going to go with the latter .\u001b[0;0m\n","\u001b[0mÈyíkèyí iru óúnjẹ yóò dára, níwọ̀n ìgbà tí kò gbówólórí púpò. Ṣé mo lè gbà nọ́mbà èrọ ìbára ẹnisọ̀rọ̀ fún ìṣèdúró rẹ? \u001b[0;0m\n","\u001b[1;94m    labels: Ibi Áfíríkà kan wà tí a ńpè ní Bedouin ní ààrín. Báwo ni ìyẹn? \u001b[0;0m\n","\u001b[0;95m     model: \" mo ni ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a ́ a\u001b[0;0m\n","\u001b[0mÌyẹn dára, ṣé mo lè gba nọ́mbà Fóónù ìbára ẹnisọ̀rọ̀ yẹ́n? Bákannaà, ṣé ò lè sọ hòtẹ́ẹ̀lì tí o gbówólórí fún mi? \u001b[0;0m\n","\u001b[1;94m    labels: Fóónù Bedouin jẹ́ 01223367660. Ní tí àwọn hòtẹ́ẹ̀lì, mo ṣèdúró Ilé-ìwè Arms University ní ààrín ìlú. \u001b[0;0m\n","\u001b[0;95m     model: i ̀ yên da ́ ra , ibn ́ ra nâ ́ ma i ́ ba ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn ́ ra ibn\u001b[0;0m\n"]}]},{"cell_type":"markdown","metadata":{"id":"z9E-8OFydrqS"},"source":["**Please provide a brief assessment of the 3 conversations above here. What do you think about the generated text? Is it repetitive, meaningless, etc?** Example below:\n","\n","We observe from the first example that the model sometimes fails to respond in the target language. And the last two examples show that the model can be repetitively bad."]}]}