Programming_Language_Translator.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Programming Language Translator.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MwQjhzC8D98W",
        "colab_type": "text"
      },
      "source": [
        "## Cloning Facebook Research TransCoder Repository"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TvwGKH3N7Y_d",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "b7a197c8-add0-40ae-9f8d-902279871ba9"
      },
      "source": [
        "!git clone https://github.com/facebookresearch/TransCoder.git"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'TransCoder'...\n",
            "remote: Enumerating objects: 2194, done.\u001b[K\n",
            "remote: Counting objects: 100% (2194/2194), done.\u001b[K\n",
            "remote: Compressing objects: 100% (1106/1106), done.\u001b[K\n",
            "remote: Total 2194 (delta 1088), reused 2191 (delta 1085), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (2194/2194), 4.03 MiB | 15.81 MiB/s, done.\n",
            "Resolving deltas: 100% (1088/1088), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JSYBFdNuEYau",
        "colab_type": "text"
      },
      "source": [
        "## Installing FastBPE"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KH3Wy6BO7nxW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 989
        },
        "outputId": "4bae4fb4-2640-4360-b306-cdcfdcff7201"
      },
      "source": [
        "!git clone https://github.com/glample/fastBPE\n",
        "%cd fastBPE\n",
        "!g++ -std=c++11 -pthread -O3 fastBPE/main.cc -IfastBPE -o fast\n",
        "!python setup.py install\n",
        "%cd .."
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'fastBPE'...\n",
            "remote: Enumerating objects: 5, done.\u001b[K\n",
            "remote: Counting objects:  20% (1/5)\u001b[K\rremote: Counting objects:  40% (2/5)\u001b[K\rremote: Counting objects:  60% (3/5)\u001b[K\rremote: Counting objects:  80% (4/5)\u001b[K\rremote: Counting objects: 100% (5/5)\u001b[K\rremote: Counting objects: 100% (5/5), done.\u001b[K\n",
            "remote: Compressing objects:  20% (1/5)\u001b[K\rremote: Compressing objects:  40% (2/5)\u001b[K\rremote: Compressing objects:  60% (3/5)\u001b[K\rremote: Compressing objects:  80% (4/5)\u001b[K\rremote: Compressing objects: 100% (5/5)\u001b[K\rremote: Compressing objects: 100% (5/5), done.\u001b[K\n",
            "Unpacking objects:   1% (1/59)   \rUnpacking objects:   3% (2/59)   \rUnpacking objects:   5% (3/59)   \rUnpacking objects:   6% (4/59)   \rUnpacking objects:   8% (5/59)   \rUnpacking objects:  10% (6/59)   \rUnpacking objects:  11% (7/59)   \rUnpacking objects:  13% (8/59)   \rUnpacking objects:  15% (9/59)   \rUnpacking objects:  16% (10/59)   \rUnpacking objects:  18% (11/59)   \rUnpacking objects:  20% (12/59)   \rUnpacking objects:  22% (13/59)   \rUnpacking objects:  23% (14/59)   \rUnpacking objects:  25% (15/59)   \rUnpacking objects:  27% (16/59)   \rUnpacking objects:  28% (17/59)   \rUnpacking objects:  30% (18/59)   \rremote: Total 59 (delta 0), reused 2 (delta 0), pack-reused 54\n",
            "Unpacking objects:  32% (19/59)   \rUnpacking objects:  33% (20/59)   \rUnpacking objects:  35% (21/59)   \rUnpacking objects:  37% (22/59)   \rUnpacking objects:  38% (23/59)   \rUnpacking objects:  40% (24/59)   \rUnpacking objects:  42% (25/59)   \rUnpacking objects:  44% (26/59)   \rUnpacking objects:  45% (27/59)   \rUnpacking objects:  47% (28/59)   \rUnpacking objects:  49% (29/59)   \rUnpacking objects:  50% (30/59)   \rUnpacking objects:  52% (31/59)   \rUnpacking objects:  54% (32/59)   \rUnpacking objects:  55% (33/59)   \rUnpacking objects:  57% (34/59)   \rUnpacking objects:  59% (35/59)   \rUnpacking objects:  61% (36/59)   \rUnpacking objects:  62% (37/59)   \rUnpacking objects:  64% (38/59)   \rUnpacking objects:  66% (39/59)   \rUnpacking objects:  67% (40/59)   \rUnpacking objects:  69% (41/59)   \rUnpacking objects:  71% (42/59)   \rUnpacking objects:  72% (43/59)   \rUnpacking objects:  74% (44/59)   \rUnpacking objects:  76% (45/59)   \rUnpacking objects:  77% (46/59)   \rUnpacking objects:  79% (47/59)   \rUnpacking objects:  81% (48/59)   \rUnpacking objects:  83% (49/59)   \rUnpacking objects:  84% (50/59)   \rUnpacking objects:  86% (51/59)   \rUnpacking objects:  88% (52/59)   \rUnpacking objects:  89% (53/59)   \rUnpacking objects:  91% (54/59)   \rUnpacking objects:  93% (55/59)   \rUnpacking objects:  94% (56/59)   \rUnpacking objects:  96% (57/59)   \rUnpacking objects:  98% (58/59)   \rUnpacking objects: 100% (59/59)   \rUnpacking objects: 100% (59/59), done.\n",
            "/content/fastBPE\n",
            "Compiling fastBPE/fastBPE.pyx because it changed.\n",
            "[1/1] Cythonizing fastBPE/fastBPE.pyx\n",
            "running install\n",
            "running bdist_egg\n",
            "running egg_info\n",
            "creating fastBPE.egg-info\n",
            "writing fastBPE.egg-info/PKG-INFO\n",
            "writing dependency_links to fastBPE.egg-info/dependency_links.txt\n",
            "writing top-level names to fastBPE.egg-info/top_level.txt\n",
            "writing manifest file 'fastBPE.egg-info/SOURCES.txt'\n",
            "package init file 'fastBPE/__init__.py' not found (or not a regular file)\n",
            "reading manifest template 'MANIFEST.in'\n",
            "writing manifest file 'fastBPE.egg-info/SOURCES.txt'\n",
            "installing library code to build/bdist.linux-x86_64/egg\n",
            "running install_lib\n",
            "running build_py\n",
            "running build_ext\n",
            "building 'fastBPE' extension\n",
            "creating build\n",
            "creating build/temp.linux-x86_64-3.6\n",
            "creating build/temp.linux-x86_64-3.6/fastBPE\n",
            "x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -IfastBPE -I/usr/include/python3.6m -c fastBPE/fastBPE.cpp -o build/temp.linux-x86_64-3.6/fastBPE/fastBPE.o -std=c++11 -Ofast -pthread\n",
            "creating build/lib.linux-x86_64-3.6\n",
            "x86_64-linux-gnu-g++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 build/temp.linux-x86_64-3.6/fastBPE/fastBPE.o -o build/lib.linux-x86_64-3.6/fastBPE.cpython-36m-x86_64-linux-gnu.so\n",
            "creating build/bdist.linux-x86_64\n",
            "creating build/bdist.linux-x86_64/egg\n",
            "copying build/lib.linux-x86_64-3.6/fastBPE.cpython-36m-x86_64-linux-gnu.so -> build/bdist.linux-x86_64/egg\n",
            "creating stub loader for fastBPE.cpython-36m-x86_64-linux-gnu.so\n",
            "byte-compiling build/bdist.linux-x86_64/egg/fastBPE.py to fastBPE.cpython-36.pyc\n",
            "creating build/bdist.linux-x86_64/egg/EGG-INFO\n",
            "copying fastBPE.egg-info/PKG-INFO -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
            "copying fastBPE.egg-info/SOURCES.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
            "copying fastBPE.egg-info/dependency_links.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
            "copying fastBPE.egg-info/top_level.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
            "writing build/bdist.linux-x86_64/egg/EGG-INFO/native_libs.txt\n",
            "zip_safe flag not set; analyzing archive contents...\n",
            "__pycache__.fastBPE.cpython-36: module references __file__\n",
            "creating dist\n",
            "creating 'dist/fastBPE-0.1.1-py3.6-linux-x86_64.egg' and adding 'build/bdist.linux-x86_64/egg' to it\n",
            "removing 'build/bdist.linux-x86_64/egg' (and everything under it)\n",
            "Processing fastBPE-0.1.1-py3.6-linux-x86_64.egg\n",
            "creating /usr/local/lib/python3.6/dist-packages/fastBPE-0.1.1-py3.6-linux-x86_64.egg\n",
            "Extracting fastBPE-0.1.1-py3.6-linux-x86_64.egg to /usr/local/lib/python3.6/dist-packages\n",
            "Adding fastBPE 0.1.1 to easy-install.pth file\n",
            "\n",
            "Installed /usr/local/lib/python3.6/dist-packages/fastBPE-0.1.1-py3.6-linux-x86_64.egg\n",
            "Processing dependencies for fastBPE==0.1.1\n",
            "Finished processing dependencies for fastBPE==0.1.1\n",
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ijZrdtptEbhT",
        "colab_type": "text"
      },
      "source": [
        "## Installing PyTorch"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZkRom8ulEeF6",
        "colab_type": "text"
      },
      "source": [
        "Checking CUDA version to install the compatible PyTorch version"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nRP8C27A8hzl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "d7dad738-7728-4e31-9ca6-ccc2532c098c"
      },
      "source": [
        "!nvcc --version"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "nvcc: NVIDIA (R) Cuda compiler driver\n",
            "Copyright (c) 2005-2019 NVIDIA Corporation\n",
            "Built on Sun_Jul_28_19:07:16_PDT_2019\n",
            "Cuda compilation tools, release 10.1, V10.1.243\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HHDk8D5IEobD",
        "colab_type": "text"
      },
      "source": [
        "Install the correct PyTorch version"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-u7-13qK8171",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "baf00eee-2381-42f5-8de5-45e229d9e037"
      },
      "source": [
        "%pip install torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Looking in links: https://download.pytorch.org/whl/torch_stable.html\n",
            "Requirement already satisfied: torch==1.6.0+cu101 in /usr/local/lib/python3.6/dist-packages (1.6.0+cu101)\n",
            "Requirement already satisfied: torchvision==0.7.0+cu101 in /usr/local/lib/python3.6/dist-packages (0.7.0+cu101)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch==1.6.0+cu101) (0.16.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torch==1.6.0+cu101) (1.18.5)\n",
            "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision==0.7.0+cu101) (7.0.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8RjQFXkZEuKs",
        "colab_type": "text"
      },
      "source": [
        "##Installing SacreBleu"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KB3Gsar99Gu7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 377
        },
        "outputId": "ed2bdb12-9231-44db-8a8a-fc96be032a51"
      },
      "source": [
        "%pip install sacrebleu==\"1.2.11\""
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sacrebleu==1.2.11\n",
            "  Downloading https://files.pythonhosted.org/packages/0d/fb/ad7d721fbeeba9e2fe459f489f38a792ca1e5f5b61f09e608f22f400ca66/sacrebleu-1.2.11.tar.gz\n",
            "Collecting typing\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/05/d9/6eebe19d46bd05360c9a9aae822e67a80f9242aabbfc58b641b957546607/typing-3.7.4.3.tar.gz (78kB)\n",
            "\u001b[K     |████████████████████████████████| 81kB 5.1MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: sacrebleu, typing\n",
            "  Building wheel for sacrebleu (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacrebleu: filename=sacrebleu-1.2.11-cp36-none-any.whl size=18641 sha256=fab6129dcfbffa8e1fa169c2c0069cbb2071be3df7bd20dec73775063201924b\n",
            "  Stored in directory: /root/.cache/pip/wheels/93/0f/06/e1c5dcbca58e907c17b59be8e1f10ae4e43bb1fb68197a8d7c\n",
            "  Building wheel for typing (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for typing: filename=typing-3.7.4.3-cp36-none-any.whl size=26309 sha256=1f753426f146b7959b9a2d1aa6f8d1438c2794ba348d7ad877fff832e46c81d0\n",
            "  Stored in directory: /root/.cache/pip/wheels/2d/04/41/8e1836e79581989c22eebac3f4e70aaac9af07b0908da173be\n",
            "Successfully built sacrebleu typing\n",
            "Installing collected packages: typing, sacrebleu\n",
            "Successfully installed sacrebleu-1.2.11 typing-3.7.4.3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "typing"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pR8Z174yE63z",
        "colab_type": "text"
      },
      "source": [
        "##Installing Clang"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "URjG2rWE_EOO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "ca1c8240-b9b7-4fc4-d53d-a2acd5808b58"
      },
      "source": [
        "%pip install clang"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting clang\n",
            "  Downloading https://files.pythonhosted.org/packages/6d/d7/40cdcb82d072cd1c5e3f7ce249a9dfbd8d7d2194d3f0885b5eaa8f310f2b/clang-6.0.0.2-py2.py3-none-any.whl\n",
            "Installing collected packages: clang\n",
            "Successfully installed clang-6.0.0.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "doRojFRAE-le",
        "colab_type": "text"
      },
      "source": [
        "Update the code to point to the correct llvm path"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zWBN8HR-BizL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "959238c5-7791-4219-eb3c-55e6ed419482"
      },
      "source": [
        "%cd /usr/lib/llvm-6.0/lib\n",
        "!ln -s libclang.so.1 libclang.so\n",
        "%cd /content\n",
        "!sed -i \"s|clang.cindex.Config.set_library_path('/usr/lib/llvm-7/lib/')|clang.cindex.Config.set_library_path('/usr/lib/llvm-6.0/lib/')|g\" /content/TransCoder/preprocessing/src/code_tokenizer.py"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/lib/llvm-6.0/lib\n",
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0lfHs4R5FKx6",
        "colab_type": "text"
      },
      "source": [
        "## Downloading the model checkpoints"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "68eIvIpxFPGJ",
        "colab_type": "text"
      },
      "source": [
        "First checkpoint for C++ -> Java, Java -> C++ and Java -> Python\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tUQtbN4N9LS6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "outputId": "1787edb7-0a65-4926-f530-1ce12184e9f1"
      },
      "source": [
        "%mkdir checkpoints\n",
        "%cd checkpoints\n",
        "!wget https://dl.fbaipublicfiles.com/transcoder/model_1.pth"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/checkpoints\n",
            "--2020-08-09 22:17:45--  https://dl.fbaipublicfiles.com/transcoder/model_1.pth\n",
            "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 172.67.9.4, 104.22.75.142, 104.22.74.142, ...\n",
            "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|172.67.9.4|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 622089114 (593M) [application/octet-stream]\n",
            "Saving to: ‘model_1.pth’\n",
            "\n",
            "model_1.pth         100%[===================>] 593.27M  66.4MB/s    in 9.3s    \n",
            "\n",
            "2020-08-09 22:17:55 (63.6 MB/s) - ‘model_1.pth’ saved [622089114/622089114]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JIl6nAprFWiH",
        "colab_type": "text"
      },
      "source": [
        "Second checkpoint for C++ -> Python, Python -> C++ and Python -> Java\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a8VlLV_VFWOC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "outputId": "9e6d2fb5-1b1d-4772-9d4c-9579f71d05d8"
      },
      "source": [
        "!wget https://dl.fbaipublicfiles.com/transcoder/model_2.pth\n",
        "%cd .."
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-08-09 22:17:56--  https://dl.fbaipublicfiles.com/transcoder/model_2.pth\n",
            "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.22.74.142, 172.67.9.4, 104.22.75.142, ...\n",
            "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.22.74.142|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 622089114 (593M) [application/octet-stream]\n",
            "Saving to: ‘model_2.pth’\n",
            "\n",
            "model_2.pth         100%[===================>] 593.27M  50.6MB/s    in 11s     \n",
            "\n",
            "2020-08-09 22:18:07 (55.9 MB/s) - ‘model_2.pth’ saved [622089114/622089114]\n",
            "\n",
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bswi8pk3Fy7T",
        "colab_type": "text"
      },
      "source": [
        "##Running the model for translation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NJupG41PF7KV",
        "colab_type": "text"
      },
      "source": [
        "Update the code to point to the correct BPE codes file path"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sOI_7lcrC92p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!sed -i 's|default=\"data/BPE_with_comments_codes\"|default=\"/content/TransCoder/data/BPE_with_comments_codes\"|g' /content/TransCoder/translate.py"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_H-qWbQUGFq6",
        "colab_type": "text"
      },
      "source": [
        "Finally, we reached the interesting part. Let's translate some code files."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "we4qY5TIOeBE",
        "colab_type": "text"
      },
      "source": [
        "First, upload the files you want to translate into another language among (cpp, java and python)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "71JnC-Y6L5YZ",
        "colab_type": "code",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCkgewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwogICAgICBwZXJjZW50LnRleHRDb250ZW50ID0KICAgICAgICAgIGAke01hdGgucm91bmQoKHBvc2l0aW9uIC8gZmlsZURhdGEuYnl0ZUxlbmd0aCkgKiAxMDApfSUgZG9uZWA7CiAgICB9CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 106
        },
        "outputId": "769bfff0-14e1-4cad-f516-01e8476b2930"
      },
      "source": [
        "%mkdir -p examples\n",
        "%cd /content/examples\n",
        "from google.colab import files\n",
        "files.upload()\n",
        "%cd /content"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/examples\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-973a6187-4b9c-4d9f-b45f-e37e3bda53c0\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-973a6187-4b9c-4d9f-b45f-e37e3bda53c0\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving 2.cpp to 2.cpp\n",
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AzebOe8ZOnVC",
        "colab_type": "text"
      },
      "source": [
        "Second, update the following command to reflect your chosen src lang, tgt lang, the model checkpoint path as mentioned in the previous section and the uploaded input file path. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t7NxX-Il-bbI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "37894ebd-4944-4d7b-89b2-12fe47847af1"
      },
      "source": [
        "INPUT_FILE_PATH = '/content/examples/2.cpp'\n",
        "SRC_LANG = 'cpp'\n",
        "TGT_LANG = 'python'\n",
        "MODEL_PATH = '/content/checkpoints/model_1.pth' if (SRC_LANG, TGT_LANG) in [('cpp', 'java'), ('java', 'cpp'), ('java', 'python')] else '/content/checkpoints/model_2.pth'\n",
        "\n",
        "!cat $INPUT_FILE_PATH\n",
        "print('=' * 20)\n",
        "!echo 'Using checkpoint: $MODEL_PATH'\n",
        "!python /content/TransCoder/translate.py --src_lang $SRC_LANG --tgt_lang $TGT_LANG --model_path $MODEL_PATH < $INPUT_FILE_PATH"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "#include <bits/stdc++.h>\r\n",
            "\r\n",
            "using namespace std;\r\n",
            "\r\n",
            "int main()\r\n",
            "{\r\n",
            "    const int MAX_SIZES = 3;\r\n",
            "    map<string,int> size_map = {{\"small\",0}, {\"medium\", 1}, {\"large\", 2}};\r\n",
            "    int T, C, N;\r\n",
            "    cin >> T;\r\n",
            "    assert(T > 0);\r\n",
            "    while(T--){\r\n",
            "        cin >> C >> N;\r\n",
            "        assert(C > 0 && N > 0);\r\n",
            "        map<string, vector<long long> > coffee;\r\n",
            "        for(int i = 0; i < C; i++){\r\n",
            "            string coffee_name;\r\n",
            "            cin >> coffee_name;\r\n",
            "            long long price;\r\n",
            "            for(int j = 0; j < MAX_SIZES; j++){\r\n",
            "                cin >> price;\r\n",
            "                coffee[coffee_name].push_back(price);\r\n",
            "            }\r\n",
            "        }\r\n",
            "        int delivery_cost = 100/N;\r\n",
            "        for(int i = 0; i < N; i++){\r\n",
            "            string person_name, coffee_size, coffee_name;\r\n",
            "            cin >> person_name >> coffee_size >> coffee_name;\r\n",
            "            long long total_cost = coffee[coffee_name][size_map[coffee_size]] + delivery_cost;\r\n",
            "            if(total_cost % 5 == 1) total_cost--;\r\n",
            "            else if(total_cost % 5 == 4) total_cost++;\r\n",
            "            cout << person_name << \" \" << total_cost << \"\\n\";\r\n",
            "        }\r\n",
            "    }\r\n",
            "    return 0;\r\n",
            "}\r\n",
            "====================\n",
            "Using checkpoint: /content/checkpoints/model_2.pth\n",
            "Loading codes from /content/TransCoder/data/BPE_with_comments_codes ...\n",
            "Read 50000 codes from the codes file.\n",
            "====================\n",
            "def __main ( ) :\n",
            "    MAX_SIZES = 3\n",
            "    size_map = { 'small' : 0 , 'medium' : 1 , 'large' : 2 }\n",
            "    T , C , N = sys.argv [ 1 : ]\n",
            "    assert T > 0\n",
            "    while T :\n",
            "        C , N = sys.argv [ 2 : ]\n",
            "        assert C > 0 and N > 0\n",
            "        coffee = { }\n",
            "        for i in range ( C ) :\n",
            "            coffee_name = sys.argv [ 3 ]\n",
            "            price = sys.argv [ 4 ]\n",
            "            for j in range ( MAX_SIZES ) :\n",
            "                price = sys.argv [ 5 ]\n",
            "                coffee [ coffee_name ].append ( price )\n",
            "        delivery_cost = 100 / N\n",
            "        for i in range ( N ) :\n",
            "            person_name , coffee_size , coffee_name = sys.argv [ 3 ]\n",
            "            total_cost = coffee [ coffee_name ] [ size_map [ coffee_size ] ] + delivery_cost\n",
            "            if total_cost % 5 == 1 :\n",
            "                total_cost -= 1\n",
            "            elif total_cost % 5 == 4 :\n",
            "                total_cost += 1\n",
            "            print ( person_name , total_cost , end = ' ' )\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}