metadataGeneration.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "metadataGeneration.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 195,
      "metadata": {
        "id": "_E5HuqY0QT-A"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "def clean_text(column):\n",
        "    for row in column:\n",
        "# Split CamelCase Characters like ConcatenationOperator to Concatenation Operator\n",
        "        row = re.sub('([A-Z][a-z]+)', r' \\1', re.sub('([A-Z]+)', r' \\1',  str(row))).split()\n",
        "        row = ' '.join(row)\n",
        "\n",
        "# Replace tabs and newlines with a single space\n",
        "        row = re.sub(\"(\\\\t)\", \" \", str(row)).lower()\n",
        "        row = re.sub(\"(\\\\r)\", \" \", str(row)).lower()\n",
        "        row = re.sub(\"(\\\\n)\", \" \", str(row)).lower()\n",
        "\n",
        "# Remove the special characters and numbers \n",
        "        row = re.sub(r\"[<>()|&©ø\\[\\]\\'\\\",.\\}`$\\{;@?~*!+=_\\//1234567890]\", \" \", str(row)).lower()\n",
        "\n",
        "# Remove Repeated words\n",
        "        row = re.sub(r\"\\\\b(\\\\w+)(?:\\\\W+\\\\1\\\\b)+\", \"\", str(row)).lower()\n",
        "\n",
        "# Remove punctuation at the end of a word\n",
        "        row = re.sub(\"(\\.\\s+)\", \" \", str(row)).lower()\n",
        "        row = re.sub(\"(\\-\\s+)\", \" \", str(row)).lower()\n",
        "        row = re.sub(\"(\\:\\s+)\", \" \", str(row)).lower()\n",
        "\n",
        "# Remove multiple spaces\n",
        "        row = re.sub(\"(\\s+)\", \" \", str(row)).lower()\n",
        "\n",
        "# Remove the single character (any character) between any two spaces\n",
        "        row = re.sub(\"(\\s+.\\s+)\", \" \", str(row)).lower()\n",
        "        yield row"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount(\"/content/gdrive\")\n",
        "\n",
        "df_code = pd.read_csv('/content/gdrive/MyDrive/metadataGeneration/javascript_Sample_Dataset.csv')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hj3V1OfNQhg9",
        "outputId": "a432b44d-e15f-44d8-a388-fb8d700c9d5b"
      },
      "execution_count": 196,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#extract only the code and docstring columns\n",
        "df_code_p = df_code[[\"code\",\"docstring\"]]\n",
        "\n",
        "print (df_code_p[\"docstring\"][0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l4rxoTG9VWQj",
        "outputId": "204d0ade-726d-4249-d683-8fbb2624d5f6"
      },
      "execution_count": 197,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Find the next separator: space, parens, comma, colon, double-quote, dollar, brackets\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#call clean_text function\n",
        "processed_code= clean_text(df_code_p['code'])\n",
        "processed_summary = clean_text(df_code_p['docstring'])\n",
        "\n",
        "print(processed_code)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cghF3hXTVh6r",
        "outputId": "a51f8fb9-0b62-405f-fbc4-4eae89e1070d"
      },
      "execution_count": 198,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<generator object clean_text at 0x7fb33c5d5f50>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "from time import time\n",
        "\n",
        "nlp = spacy.load('en_core_web_sm', disable=['ner', 'parser']) \n",
        "\n",
        "# Process text as batches and yield Doc objects in order\n",
        "code = [str(doc) for doc in nlp.pipe(processed_code, batch_size=50)]\n",
        "summary = [ str(doc)  for doc in nlp.pipe(processed_summary, batch_size=50)]"
      ],
      "metadata": {
        "id": "vQoCYVukVkfb"
      },
      "execution_count": 199,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print (len(summary))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rSumEBUMVn1q",
        "outputId": "d00c1d37-c4c2-421e-8457-d4af91cb555d"
      },
      "execution_count": 200,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "106\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "code_count=[]\n",
        "for sent in df_code_p['code']:\n",
        "    code_count.append(len(sent.split()))"
      ],
      "metadata": {
        "id": "cIM9cEK1Vsuk"
      },
      "execution_count": 201,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "histogram_df = pd.DataFrame()\n",
        "histogram_df['code'] = code_count\n",
        "histogram_df.hist(bins = 10)\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        },
        "id": "p4-RzncrgcF9",
        "outputId": "145677ef-27c7-4393-a55f-1d036f20d51b"
      },
      "execution_count": 202,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEICAYAAABGaK+TAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAS/klEQVR4nO3df4xl5X3f8ffH/DCUcXeNodPNQrxQkC3kjYkZEVt23RkTpxgnWSIhlwi5S0S1UmtbbutI3cRVlVRNi1slEVWcuFtDs4kcDxQbgYychmyYWpUKzm6MvWDissZL7C1mE7IQj9UmXfLtH/eMMzs7s/fu7r1z57l5v6TRvec5z537/XKWz5557jmzqSokSe151bgLkCSdGQNckhplgEtSowxwSWqUAS5JjTLAJalRBri0iiSHk/zwuOuQTsUAl6RGGeCS1CgDXBMnyeVJPpvkj5O8mORXkrwqyb9M8lySo0l+I8mmZa95f7fvxSQfXfH9XpVkd5Kvd/vvS3Lx+ncmncgA10RJcg7wOeA5YBuwFZgHbu++5oArgSngV7rXXAP8GvB+4PuA1wGXLfu2HwJuBv5et/8Y8PERtyL1FX8XiiZJkrcBDwFbqur4svF9wGeq6le77TcATwIXAj8LXFNVt3b7LqIX0jdV1e8meRr4YFXt6/ZvAf4IuHD5e0jr7dxxFyAN2eXAc6sE6/fROytf8hy9P//T3b5vLu2oqu8meXHZ3NcDDyT5y2Vjr3SvPTLE2qXT4hKKJs03ge9PsvLk5H/TC+Il3w8cB14AnqcX/AAk+Rv0llGWf8/3VNXmZV8XVJXhrbEywDVpvkgvkO9MclGSC5K8Hfg08M+SXJFkCvi3wL3dmfr9wI8meUeS84F/zYn/b3wC+IUkrwdIcmmSHevZlLQaA1wTpapeAX4MuIreOvW3gH8A3AP8JvAF4BvA/6X34SRV9RTwAeC36IX/se51S+6it67+O0m+AzwG/NA6tCOdkh9iSlKjPAOXpEYZ4JLUKANckhplgEtSo9b1Rp5LLrmktm3bdsLYd7/7XS666KL1LGNk7GXjmqR+7GVjGmUvBw4c+JOqunTl+LoG+LZt29i/f/8JYwsLC8zOzq5nGSNjLxvXJPVjLxvTKHtJ8txq4y6hSFKjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSo5r5NzG37X54LO97+M73juV9Jakfz8AlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjRoowJNsTnJ/kj9M8nSStyW5OMkjSZ7pHl876mIlSX9l0DPwu4Dfrqo3Am8GngZ2A/uq6mpgX7ctSVonfQM8ySbgncDdAFX1F1X1ErAD2NtN2wvcPKoiJUknG+QM/Argj4H/kuRLST6Z5CJguqqe7+Z8G5geVZGSpJOlqk49IZkBHgPeXlWPJ7kL+DPgQ1W1edm8Y1V10jp4kl3ALoDp6enr5ufnT9i/uLjI1NRU30IPHnm5fzcjsH3rpoHnDtpLCyapF5isfuxlYxplL3Nzcweqambl+CAB/reBx6pqW7f9d+mtd18FzFbV80m2AAtV9YZTfa+ZmZnav3//CWMLCwvMzs72baCF30Y4aC8tmKReYLL6sZeNaZS9JFk1wPsuoVTVt4FvJlkK5xuArwIPATu7sZ3Ag0OqVZI0gEF/H/iHgE8lOR94FvgpeuF/X5I7gOeA942mREnSagYK8Kp6Ajjp9J3e2bgkaQy8E1OSGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktSocweZlOQw8B3gFeB4Vc0kuRi4F9gGHAbeV1XHRlOmJGml0zkDn6uqa6tqptveDeyrqquBfd22JGmdnM0Syg5gb/d8L3Dz2ZcjSRpUqqr/pOQbwDGggP9UVXuSvFRVm7v9AY4tba947S5gF8D09PR18/PzJ+xfXFxkamqqbw0Hj7zcv5sR2L5108BzB+2lBZPUC0xWP/ayMY2yl7m5uQPLVj++Z6A1cOAdVXUkyd8CHknyh8t3VlUlWfVvgqraA+wBmJmZqdnZ2RP2LywssHJsNbfvfnjAUofr8G2zA88dtJcWTFIvMFn92MvGNI5eBlpCqaoj3eNR4AHgeuCFJFsAusejoypSknSyvgGe5KIkr1l6DvwI8CTwELCzm7YTeHBURUqSTjbIEso08EBvmZtzgd+qqt9O8vvAfUnuAJ4D3je6MiVJK/UN8Kp6FnjzKuMvAjeMoihJUn/eiSlJjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowYO8CTnJPlSks9121ckeTzJoST3Jjl/dGVKklY6nTPwDwNPL9v+GPDLVXUVcAy4Y5iFSZJObaAAT3IZ8F7gk912gHcB93dT9gI3j6JASdLqUlX9JyX3A/8OeA3w08DtwGPd2TdJLgc+X1VvWuW1u4BdANPT09fNz8+fsH9xcZGpqam+NRw88nLfOaOwfeumgecO2ksLJqkXmKx+7GVjGmUvc3NzB6pqZuX4uf1emORHgaNVdSDJ7Om+cVXtAfYAzMzM1Ozsid9iYWGBlWOruX33w6f71kNx+LbZgecO2ksLJqkXmKx+7GVjGkcvfQMceDvw40luAi4A/iZwF7A5yblVdRy4DDgyujIlSSv1XQOvqp+pqsuqahtwK/B7VXUb8ChwSzdtJ/DgyKqUJJ3kbK4D/xfAP09yCHgdcPdwSpIkDWKQJZTvqaoFYKF7/ixw/fBLkiQNwjsxJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5Jjeob4EkuSPLFJF9O8lSSn+/Gr0jyeJJDSe5Ncv7oy5UkLRnkDPzPgXdV1ZuBa4Ebk7wV+Bjwy1V1FXAMuGN0ZUqSVuob4NWz2G2e130V8C7g/m58L3DzSCqUJK0qVdV/UnIOcAC4Cvg48B+Ax7qzb5JcDny+qt60ymt3AbsApqenr5ufnz9h/+LiIlNTU31rOHjk5b5zRmH71k0Dzx20lxZMUi8wWf3Yy8Y0yl7m5uYOVNXMyvFzB3lxVb0CXJtkM/AA8MZB37iq9gB7AGZmZmp2dvaE/QsLC6wcW83tux8e9C2H6vBtswPPHbSXFkxSLzBZ/djLxjSOXk7rKpSqegl4FHgbsDnJ0l8AlwFHhlybJOkUBrkK5dLuzJskFwLvBp6mF+S3dNN2Ag+OqkhJ0skGWULZAuzt1sFfBdxXVZ9L8lVgPsm/Ab4E3D3COiVJK/QN8Kr6CvCDq4w/C1w/iqIkSf15J6YkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5Jjeob4EkuT/Jokq8meSrJh7vxi5M8kuSZ7vG1oy9XkrRkkDPw48BHquoa4K3AB5JcA+wG9lXV1cC+bluStE76BnhVPV9Vf9A9/w7wNLAV2AHs7abtBW4eVZGSpJOlqgafnGwDvgC8CfijqtrcjQc4trS94jW7gF0A09PT183Pz5+wf3Fxkampqb7vffDIywPXOUzbt24aeO6gvbRgknqByerHXjamUfYyNzd3oKpmVo4PHOBJpoD/DvxCVX02yUvLAzvJsao65Tr4zMxM7d+//4SxhYUFZmdn+77/tt0PD1TnsB2+870Dzx20lxZMUi8wWf3Yy8Y0yl6SrBrgA12FkuQ84DPAp6rqs93wC0m2dPu3AEeHVawkqb9BrkIJcDfwdFX90rJdDwE7u+c7gQeHX54kaS3nDjDn7cD7gYNJnujGfha4E7gvyR3Ac8D7RlOiJGk1fQO8qv4HkDV23zDcciRJg/JOTElq1CBLKH+tnc7VLx/Zfpzbh3i1zOlcASPprx/PwCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcp/Um0DO51/zm2Y/KfcpDZ4Bi5JjTLAJalRfQM8yT1JjiZ5ctnYxUkeSfJM9/ja0ZYpSVppkDPwXwduXDG2G9hXVVcD+7ptSdI66hvgVfUF4E9XDO8A9nbP9wI3D7kuSVIfqar+k5JtwOeq6k3d9ktVtbl7HuDY0vYqr90F7AKYnp6+bn5+/oT9i4uLTE1N9a3h4JGX+84Zt+kL4YX/M+4qzt72rZsGPi6tmKR+7GVjGmUvc3NzB6pqZuX4WV9GWFWVZM2/BapqD7AHYGZmpmZnZ0/Yv7CwwMqx1dw+pkvqTsdHth/nFw+2f2Xm4dtmBz4urZikfuxlYxpHL2d6FcoLSbYAdI9Hh1eSJGkQZxrgDwE7u+c7gQeHU44kaVCDXEb4aeB/Am9I8q0kdwB3Au9O8gzww922JGkd9V2wraqfXGPXDUOuRZJ0GrwTU5IaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJalT7v7xaQ7dt98N8ZPvxsfwO9sN3vnfd31NqlWfgktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEZ5I482lG0junmo341J47yB6HR7HtdNVsPkDVvD4Rm4JDXKAJekRhngktSos1oDT3IjcBdwDvDJqrpzKFVJ0giM6jMWOPVnE6Na8z/jM/Ak5wAfB94DXAP8ZJJrhlWYJOnUzmYJ5XrgUFU9W1V/AcwDO4ZTliSpn1TVmb0wuQW4sar+Ubf9fuCHquqDK+btAnZ1m28AvrbiW10C/MkZFbHx2MvGNUn92MvGNMpeXl9Vl64cHPl14FW1B9iz1v4k+6tqZtR1rAd72bgmqR972ZjG0cvZLKEcAS5ftn1ZNyZJWgdnE+C/D1yd5Iok5wO3Ag8NpyxJUj9nvIRSVceTfBD4b/QuI7ynqp46g2+15vJKg+xl45qkfuxlY1r3Xs74Q0xJ0nh5J6YkNcoAl6RGjS3Ak9yY5GtJDiXZPa46zkaSw0kOJnkiyf5u7OIkjyR5pnt87bjrXE2Se5IcTfLksrFVa0/Pf+yO1VeSvGV8lZ9sjV5+LsmR7tg8keSmZft+puvla0n+/niqXl2Sy5M8muSrSZ5K8uFuvLljc4peWj02FyT5YpIvd/38fDd+RZLHu7rv7S7qIMmru+1D3f5tQy+qqtb9i96Hnl8HrgTOB74MXDOOWs6yj8PAJSvG/j2wu3u+G/jYuOtco/Z3Am8BnuxXO3AT8HkgwFuBx8dd/wC9/Bzw06vMvab78/Zq4Iruz+E54+5hWX1bgLd0z18D/K+u5uaOzSl6afXYBJjqnp8HPN79N78PuLUb/wTwj7vn/wT4RPf8VuDeYdc0rjPwSb4Nfwewt3u+F7h5jLWsqaq+APzpiuG1at8B/Eb1PAZsTrJlfSrtb41e1rIDmK+qP6+qbwCH6P153BCq6vmq+oPu+XeAp4GtNHhsTtHLWjb6samqWuw2z+u+CngXcH83vvLYLB2z+4EbkmSYNY0rwLcC31y2/S1OfWA3qgJ+J8mB7lcGAExX1fPd828D0+Mp7YysVXurx+uD3bLCPcuWsprppfuR+wfpnek1fWxW9AKNHpsk5yR5AjgKPELvp4SXqup4N2V5zd/rp9v/MvC6Ydbjh5hn5x1V9RZ6v5HxA0neuXxn9X52avI6zZZr7/wa8HeAa4HngV8cbzmnJ8kU8Bngn1bVny3f19qxWaWXZo9NVb1SVdfSu/P8euCN46xnXAE+EbfhV9WR7vEo8AC9A/rC0o+w3ePR8VV42taqvbnjVVUvdP+z/SXwn/mrH8U3fC9JzqMXeJ+qqs92w00em9V6afnYLKmql4BHgbfRW7Zauilyec3f66fbvwl4cZh1jCvAm78NP8lFSV6z9Bz4EeBJen3s7KbtBB4cT4VnZK3aHwL+YXfFw1uBl5f9OL8hrVgH/gl6xwZ6vdzaXSFwBXA18MX1rm8t3Rrp3cDTVfVLy3Y1d2zW6qXhY3Npks3d8wuBd9Nb138UuKWbtvLYLB2zW4Df6356Gp4xfqJ7E71Ppb8OfHRcdZxF/VfS+8T8y8BTSz3QW+PaBzwD/C5w8bhrXaP+T9P78fX/0Vu3u2Ot2ul9+v7x7lgdBGbGXf8AvfxmV+tXuv+Rtiyb/9Gul68B7xl3/St6eQe95ZGvAE90Xze1eGxO0Uurx+YHgC91dT8J/Ktu/Ep6f9EcAv4r8Opu/IJu+1C3/8ph1+St9JLUKD/ElKRGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUf8f1KzbEwmGVqMAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "max_code_len = 100\n",
        "max_summary_len =25\n",
        "\n",
        "# Extract the codes and summaries within the maximum length\n",
        "import numpy as np\n",
        "\n",
        "cleaned_code = np.array(df_code_p['code'])\n",
        "cleaned_summary= np.array(df_code_p['docstring'])\n",
        "\n",
        "short_text = []\n",
        "short_summary = []\n",
        "\n",
        "for i in range(len(cleaned_code)):\n",
        "  if len(cleaned_summary[i].split()) <= max_summary_len and len(cleaned_code[i].split()) <= max_code_len:\n",
        "    short_text.append(cleaned_code[i])\n",
        "    short_summary.append(cleaned_summary[i])\n",
        "\n",
        "post_code = pd.DataFrame({'code': short_text,'summary': short_summary})\n",
        "post_code.head(100)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "gQa_wl_UVwxt",
        "outputId": "d04052f4-746b-487a-9789-d1c4ed4a1658"
      },
      "execution_count": 203,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                 code  \\\n",
              "0   function show(url) {\\n  const location = get('...   \n",
              "1   function(_url,_href){\\n            if (!_url) ...   \n",
              "2   function(_config){\\n            if (!_config) ...   \n",
              "3   function(_action,_name){\\n            var _am ...   \n",
              "4   function(_node,_message){\\n            var _mo...   \n",
              "5   function(_target,_message){\\n                v...   \n",
              "6   function(_target,_message){\\n                v...   \n",
              "7   function getErrorMessage(errors) {\\n  if (_.is...   \n",
              "8   function dispose() {\\n  if (typeof this.destro...   \n",
              "9   function Traverse (vars) {\\n\\t\\t\\t\\t\\t\\tObject...   \n",
              "10  function form (cols, cllbck, level) {\\n\\t\\t\\t\\...   \n",
              "11  function DoLog (which, what) {\\n\\t\\t\\t\\t\\t\\t\\t...   \n",
              "12  function() {\\n      this.emit('acquiring');\\n ...   \n",
              "13  function (str) {\\n        var arr = str.split(...   \n",
              "14  function (map) {\\n            var key;\\n      ...   \n",
              "15  function (options) {\\n            var key;\\n  ...   \n",
              "16  function (map) {\\n            var key;\\n      ...   \n",
              "17  function (subview) {\\n            subview = (s...   \n",
              "18  function (type) {\\n            var confs = typ...   \n",
              "19  function (viewInstance) {\\n            console...   \n",
              "20  function(callback) {\\n    if (typeof callback ...   \n",
              "21  function(callback) {\\n    if (typeof callback ...   \n",
              "22  function configure_logging() {\\n  if ('log' in...   \n",
              "23  function LocalStorage() {\\n        this.versio...   \n",
              "24  function FileStorage($q, $rootScope) {\\n      ...   \n",
              "25  function(template, delta) {\\n    assert.equal(...   \n",
              "26  function(template, delta, overrideName) {\\n   ...   \n",
              "27  function(desc, f) {\\n    if (typeof desc === '...   \n",
              "28  function (scope, element, attrs) {\\n\\n\\t\\t\\t\\t...   \n",
              "29  function Circle2D( x , y , r ) {\\n\\tthis.cente...   \n",
              "30  function(_event){\\n            var _element = ...   \n",
              "31  function(_list){\\n            var _result = 0;...   \n",
              "32  function injectPaths(paths) {\\n    u.each(path...   \n",
              "33  function mkSrc(p) {\\n    var src = {};\\n    if...   \n",
              "34  function setOptName(opt, optKey) {\\n    var ke...   \n",
              "35  function watchOpts(src) {\\n    if ((src._pkg &...   \n",
              "36  function pkgPath(pkg) {\\n    pkg = pkg.pkg || ...   \n",
              "37  function normalizeOpts(obj, basedir, pkg) {\\n ...   \n",
              "38  function normalizeOptsKey(aval, basedir, pkg) ...   \n",
              "39  function(_id){\\n            var _conf  = _cach...   \n",
              "40  function TimeoutError(message) {\\n  this.messa...   \n",
              "41  function onFunction(data, op, iterStr, context...   \n",
              "42  function onExec(funcStr, parStr, context) {\\n\\...   \n",
              "43  function() {\\n    this._observers.enabled = tr...   \n",
              "\n",
              "                                              summary  \n",
              "0                                                show  \n",
              "1                          check event need delegated  \n",
              "2                                      regist rewrite  \n",
              "3                                      regist actions  \n",
              "4                                        send message  \n",
              "5           send message to every node in target path  \n",
              "6                 broadcast to all target descendants  \n",
              "7                  convert errors given by jsonschema  \n",
              "8                                            /* @flow  \n",
              "9                                         Format Vars  \n",
              "10                                           PRIVATES  \n",
              "11                                             ------  \n",
              "12  Does the work in acquiring a resource and sets...  \n",
              "13  Finds globally dot noted namespaced objects fr...  \n",
              "14  Add instances using an object that\\nmaps subvi...  \n",
              "15  Instantiate all singletons defined in the\\ncon...  \n",
              "16  Create subviews with a map of configured\\nsubv...  \n",
              "17  Get the subviews type or key\\n@memberOf SubVie...  \n",
              "18  Clears the html in all locations specified in ...  \n",
              "19  Marinate only an instance of a view\\n@memberOf...  \n",
              "20  Schedules an observer sync cycle which checks ...  \n",
              "21  Runs the observer sync cycle which checks all ...  \n",
              "22                   Default configuration of logging  \n",
              "23                                        Constructor  \n",
              "24                                        Constructor  \n",
              "25  Merge two environments of matching components....  \n",
              "26  Merge two descriptions with the same name.\\n\\n...  \n",
              "27  Patches every environment in a description.\\n\\...  \n",
              "28  templateUrl: '/templates/directives/njaxCommen...  \n",
              "29  /*\\nThis is the equation:\\n(X - x)² + (Y - y)²...  \n",
              "30                                      enter element  \n",
              "31                         get min value but not zero  \n",
              "32  add injectable staticPaths to opts.injectCss o...  \n",
              "33  computes script src path from p.route + p.path...  \n",
              "34  assign a unique name to opt by indexing opts.[...  \n",
              "35  don't watch inside packages unless opts.watchP...  \n",
              "36  resolve package path given package name or {pk...  \n",
              "37  mutate opts obj by normalizing Key values into...  \n",
              "38                        normalize a single opts key  \n",
              "39                                       input change  \n",
              "40  Timeout Error.\\n\\n@param {String} message\\n@co...  \n",
              "41                         Handle functional requests  \n",
              "42                               Handle exec requests  \n",
              "43         Starts the observers watching their values  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e7060d1f-d8e9-4819-b15d-01c62c6150ea\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>code</th>\n",
              "      <th>summary</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>function show(url) {\\n  const location = get('...</td>\n",
              "      <td>show</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>function(_url,_href){\\n            if (!_url) ...</td>\n",
              "      <td>check event need delegated</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>function(_config){\\n            if (!_config) ...</td>\n",
              "      <td>regist rewrite</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>function(_action,_name){\\n            var _am ...</td>\n",
              "      <td>regist actions</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>function(_node,_message){\\n            var _mo...</td>\n",
              "      <td>send message</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>function(_target,_message){\\n                v...</td>\n",
              "      <td>send message to every node in target path</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>function(_target,_message){\\n                v...</td>\n",
              "      <td>broadcast to all target descendants</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>function getErrorMessage(errors) {\\n  if (_.is...</td>\n",
              "      <td>convert errors given by jsonschema</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>function dispose() {\\n  if (typeof this.destro...</td>\n",
              "      <td>/* @flow</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>function Traverse (vars) {\\n\\t\\t\\t\\t\\t\\tObject...</td>\n",
              "      <td>Format Vars</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>function form (cols, cllbck, level) {\\n\\t\\t\\t\\...</td>\n",
              "      <td>PRIVATES</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>function DoLog (which, what) {\\n\\t\\t\\t\\t\\t\\t\\t...</td>\n",
              "      <td>------</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>function() {\\n      this.emit('acquiring');\\n ...</td>\n",
              "      <td>Does the work in acquiring a resource and sets...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>function (str) {\\n        var arr = str.split(...</td>\n",
              "      <td>Finds globally dot noted namespaced objects fr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>function (map) {\\n            var key;\\n      ...</td>\n",
              "      <td>Add instances using an object that\\nmaps subvi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>function (options) {\\n            var key;\\n  ...</td>\n",
              "      <td>Instantiate all singletons defined in the\\ncon...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>function (map) {\\n            var key;\\n      ...</td>\n",
              "      <td>Create subviews with a map of configured\\nsubv...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>function (subview) {\\n            subview = (s...</td>\n",
              "      <td>Get the subviews type or key\\n@memberOf SubVie...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>function (type) {\\n            var confs = typ...</td>\n",
              "      <td>Clears the html in all locations specified in ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>function (viewInstance) {\\n            console...</td>\n",
              "      <td>Marinate only an instance of a view\\n@memberOf...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>function(callback) {\\n    if (typeof callback ...</td>\n",
              "      <td>Schedules an observer sync cycle which checks ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>function(callback) {\\n    if (typeof callback ...</td>\n",
              "      <td>Runs the observer sync cycle which checks all ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>function configure_logging() {\\n  if ('log' in...</td>\n",
              "      <td>Default configuration of logging</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>function LocalStorage() {\\n        this.versio...</td>\n",
              "      <td>Constructor</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>function FileStorage($q, $rootScope) {\\n      ...</td>\n",
              "      <td>Constructor</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>function(template, delta) {\\n    assert.equal(...</td>\n",
              "      <td>Merge two environments of matching components....</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>function(template, delta, overrideName) {\\n   ...</td>\n",
              "      <td>Merge two descriptions with the same name.\\n\\n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>function(desc, f) {\\n    if (typeof desc === '...</td>\n",
              "      <td>Patches every environment in a description.\\n\\...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>function (scope, element, attrs) {\\n\\n\\t\\t\\t\\t...</td>\n",
              "      <td>templateUrl: '/templates/directives/njaxCommen...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>function Circle2D( x , y , r ) {\\n\\tthis.cente...</td>\n",
              "      <td>/*\\nThis is the equation:\\n(X - x)² + (Y - y)²...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>function(_event){\\n            var _element = ...</td>\n",
              "      <td>enter element</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>function(_list){\\n            var _result = 0;...</td>\n",
              "      <td>get min value but not zero</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>function injectPaths(paths) {\\n    u.each(path...</td>\n",
              "      <td>add injectable staticPaths to opts.injectCss o...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>function mkSrc(p) {\\n    var src = {};\\n    if...</td>\n",
              "      <td>computes script src path from p.route + p.path...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>function setOptName(opt, optKey) {\\n    var ke...</td>\n",
              "      <td>assign a unique name to opt by indexing opts.[...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>function watchOpts(src) {\\n    if ((src._pkg &amp;...</td>\n",
              "      <td>don't watch inside packages unless opts.watchP...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>function pkgPath(pkg) {\\n    pkg = pkg.pkg || ...</td>\n",
              "      <td>resolve package path given package name or {pk...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>function normalizeOpts(obj, basedir, pkg) {\\n ...</td>\n",
              "      <td>mutate opts obj by normalizing Key values into...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>function normalizeOptsKey(aval, basedir, pkg) ...</td>\n",
              "      <td>normalize a single opts key</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>function(_id){\\n            var _conf  = _cach...</td>\n",
              "      <td>input change</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>function TimeoutError(message) {\\n  this.messa...</td>\n",
              "      <td>Timeout Error.\\n\\n@param {String} message\\n@co...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>function onFunction(data, op, iterStr, context...</td>\n",
              "      <td>Handle functional requests</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>function onExec(funcStr, parStr, context) {\\n\\...</td>\n",
              "      <td>Handle exec requests</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>function() {\\n    this._observers.enabled = tr...</td>\n",
              "      <td>Starts the observers watching their values</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e7060d1f-d8e9-4819-b15d-01c62c6150ea')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e7060d1f-d8e9-4819-b15d-01c62c6150ea button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e7060d1f-d8e9-4819-b15d-01c62c6150ea');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 203
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 204,
      "metadata": {
        "id": "ef5fce89",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        },
        "outputId": "55ffc9f7-9023-4a53-8849-a3b0491cf495"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                code  \\\n",
              "0  function show(url) {\\n  const location = get('...   \n",
              "1  function(_url,_href){\\n            if (!_url) ...   \n",
              "\n",
              "                                    summary  \n",
              "0                        sostok show eostok  \n",
              "1  sostok check event need delegated eostok  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-a172d346-fd25-402e-8e00-ef56780595b7\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>code</th>\n",
              "      <th>summary</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>function show(url) {\\n  const location = get('...</td>\n",
              "      <td>sostok show eostok</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>function(_url,_href){\\n            if (!_url) ...</td>\n",
              "      <td>sostok check event need delegated eostok</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a172d346-fd25-402e-8e00-ef56780595b7')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a172d346-fd25-402e-8e00-ef56780595b7 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a172d346-fd25-402e-8e00-ef56780595b7');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 204
        }
      ],
      "source": [
        "post_code['summary'] = post_code['summary'].apply(lambda x: 'sostok ' + x \\\n",
        "        + ' eostok')\n",
        "\n",
        "post_code.head(2)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "t  = Tokenizer()\n",
        "\n",
        "fit_text = 'Machine Learning'\n",
        "\n",
        "t.fit_on_texts(fit_text)\n",
        "\n",
        "print(\"Count of characters:\",t.word_counts)\n",
        "print(\"Length of text:\",t.document_count)\n",
        "print(\"Character index\",t.word_index)\n",
        "print(\"Frequency of characters:\",t.word_docs)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eYtUU3x4YChT",
        "outputId": "8b1cd859-b41c-46b6-f042-784b7e9096dd"
      },
      "execution_count": 205,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Count of characters: OrderedDict([('m', 1), ('a', 2), ('c', 1), ('h', 1), ('i', 2), ('n', 3), ('e', 2), ('l', 1), ('r', 1), ('g', 1)])\n",
            "Length of text: 16\n",
            "Character index {'n': 1, 'a': 2, 'i': 3, 'e': 4, 'm': 5, 'c': 6, 'h': 7, 'l': 8, 'r': 9, 'g': 10}\n",
            "Frequency of characters: defaultdict(<class 'int'>, {'m': 1, 'a': 2, 'c': 1, 'h': 1, 'i': 2, 'n': 3, 'e': 2, 'l': 1, 'r': 1, 'g': 1})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# use the pre-processed codes and summaries that are within the maximum length and divide into train and test sets\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "x_train, x_validation, y_train, y_validation = train_test_split(\n",
        "np.array(post_code[\"code\"]),\n",
        "np.array(post_code[\"summary\"]),\n",
        "test_size=0.15,\n",
        "random_state=0,\n",
        "shuffle=True,\n",
        ")"
      ],
      "metadata": {
        "id": "lcL286eKYGAc"
      },
      "execution_count": 206,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#fit the tokenizer on the code sequences\n",
        "# we will not consider infrequent words in the vocabulary, as it may add noise\n",
        "# Hence we omit words which occur only two times ( you can change the threshold to 5 in case of larger datasets)\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "# Prepare a tokenizer on training data\n",
        "x_tokenizer = Tokenizer()\n",
        "x_tokenizer.fit_on_texts(list(x_train))\n",
        "threshold = 2\n",
        "cnt_infrequent = 0\n",
        "total_cnt = 0\n",
        "for key, value in x_tokenizer.word_counts.items():\n",
        "    total_cnt = total_cnt + 1\n",
        "    if value < threshold:\n",
        "       cnt_infrequent = cnt_infrequent + 1\n",
        "print(\"% of not frequent words in vocabulary: \", (cnt_infrequent / total_cnt) * 100)\n",
        "# Remove the infrequent words\n",
        "x_tokenizer = Tokenizer(num_words = total_cnt - cnt_infrequent)\n",
        "x_tokenizer.fit_on_texts(list(x_train))\n",
        "# Convert the code sequences to integer sequences (integer numbers ranging from 1 to the maximum vocab sizes)\n",
        "x_train_seqs = x_tokenizer.texts_to_sequences(x_train)\n",
        "x_validation_seqs = x_tokenizer.texts_to_sequences(x_validation)\n",
        "# printing the integer sequences\n",
        "print (x_train_seqs)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ptOtOTviYH0u",
        "outputId": "1d5d7979-6d4f-4496-c9e4-10244ceba324"
      },
      "execution_count": 207,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "% of not frequent words in vocabulary:  44.2159383033419\n",
            "[[3, 129, 130, 2, 20, 73, 20, 131, 132, 132, 131, 129, 38, 130, 23, 9, 9], [3, 39, 74, 24, 39, 39, 2, 21, 133, 39, 39, 39, 4, 21, 19, 21, 39, 3, 134, 4, 134, 74, 24], [3, 135, 5, 14, 28, 21, 55, 135, 3, 75, 2, 75, 4, 2, 14, 14, 75, 40, 14, 136, 14, 75, 4, 14], [3, 137, 138, 56, 1, 35, 137, 138, 1, 56, 56], [3, 46, 2, 29, 46, 47, 1, 139, 46, 2, 1, 48, 4, 76, 5, 140, 1, 1, 100, 1, 100, 141, 1, 1, 48, 1, 100, 19, 1, 140, 4, 49], [3, 15, 101, 102, 36, 5, 77, 15, 142, 143, 2, 101, 15, 102, 36, 30, 25, 144, 145, 146, 147, 41, 103, 148, 3, 78, 149, 42, 150, 151, 152, 153, 154, 155, 156, 157, 78, 158, 77, 9, 102, 159, 101, 104, 15, 15, 19, 77, 79, 104, 15, 15, 141, 77, 79, 160, 15, 23, 79, 57, 161, 14, 15, 36, 36], [3, 10, 2, 10, 4, 1, 10, 56, 80, 10, 41, 26, 21, 55, 3, 6, 2, 10, 6, 4, 1, 10, 6, 10, 6, 10, 6, 1], [3, 1, 162, 49, 1, 162, 55, 1, 58, 1, 1, 31, 103, 59, 1, 55, 1, 58, 1, 1], [3, 81, 5, 163, 81, 1, 10, 81, 1, 10, 164, 163, 3, 10, 1, 10, 50, 82, 1, 4, 1], [3, 83, 5, 6, 84, 6, 26, 1, 10, 2, 1, 10, 105, 6, 1, 10, 6, 1, 165, 6, 83, 4, 1], [3, 2, 26, 16, 2, 26, 16, 20, 57, 51, 60, 43, 166, 16, 20, 166, 2, 26, 16, 20, 57, 51, 60, 43, 106, 16, 20, 106, 2, 26, 16, 20, 84, 5, 6, 26, 16, 20, 167, 57, 51, 43, 106, 9, 6, 16, 20, 167, 6], [3, 19, 5, 6, 84, 6, 26, 19, 2, 19, 105, 6, 1, 165, 6, 19, 6, 4, 1], [3, 27, 1, 27, 27, 1, 49, 25, 1, 168], [3, 169, 2, 29, 1, 170, 47, 1, 170, 40, 2, 29, 1, 171, 47, 1, 171, 40, 2, 29, 1, 172, 47, 1, 172, 40, 2, 29, 1, 173, 47, 1, 173, 40, 30, 35, 25, 31, 169], [3, 174, 5, 85, 174, 9, 9, 5, 61, 85, 175, 107, 85, 108, 61, 61, 61, 85, 175, 4, 61], [3, 37, 27, 5, 44, 27, 44, 107, 37, 2, 37, 176, 44, 177, 37, 27, 37, 37], [3, 52, 52, 52, 178, 49, 52, 1, 86, 52, 4, 52], [3, 87, 21, 164, 87, 3, 22, 2, 22, 109, 179, 110, 180, 87, 5, 11, 181, 22, 2, 179, 182, 183, 11, 22, 4, 32, 80, 11, 2, 180, 182, 183, 11, 22, 4, 32, 80, 11, 4, 32, 20, 31, 22, 22], [3, 19, 5, 6, 84, 6, 26, 19, 2, 19, 105, 6, 1, 6, 19, 6, 4, 1], [3, 11, 2, 11, 24, 32, 26, 11, 11, 111, 4, 76, 4, 29, 11, 111, 11, 111], [3, 62, 2, 133, 62, 4, 62, 184, 185, 82, 40, 2, 62, 4, 62, 27], [3, 63, 186, 5, 6, 186, 9, 9, 32, 6, 32, 6, 5, 8, 63, 8, 112, 187, 63, 22, 107, 32, 6, 8, 8, 9, 9, 8, 32, 6, 8, 63, 63, 8, 8], [3, 64, 88, 2, 64, 4, 5, 53, 50, 41, 64, 1, 53, 5, 89, 1, 53, 22, 88, 53, 88, 4, 1, 1, 89], [3, 33, 17, 188, 2, 33, 8, 17, 8, 2, 188, 5, 12, 35, 25, 65, 12, 33, 12, 17, 30, 12, 81, 5, 14, 8, 17, 8, 33, 8, 66, 17, 66, 17, 66, 33, 66, 65, 17, 65, 17, 65, 33, 65, 51, 33, 51, 17, 51, 2, 33, 67, 17, 67, 14, 67, 33, 67, 17, 67, 4, 14], [3, 1, 68, 90, 189, 12, 43, 190, 91, 191, 113, 92, 192, 12, 1, 68, 69, 12, 1, 193, 69, 90, 194, 195, 1, 34, 195, 1, 28, 2, 1, 34, 196, 1, 1, 34, 197, 12, 43, 53, 91, 12, 1, 68, 1, 114, 1, 34, 197, 12, 43, 53, 91, 192, 12, 1, 114, 69, 1, 34, 196, 198, 1, 68, 198, 1, 114, 90, 199, 115, 43, 190, 91, 191, 113, 92, 115, 1, 68, 69, 115, 1, 193, 69, 194, 189, 200, 199], [3, 181, 45, 5, 11, 2, 29, 45, 109, 11, 45, 109, 11, 22, 112, 45, 116, 45, 116, 184, 45, 116, 9, 9, 112, 187, 45, 22, 4, 11], [3, 117, 118, 1, 201, 28, 119, 1, 117, 117, 1, 118, 118, 1, 28, 1, 23, 1, 82, 1, 23, 1, 76, 1, 1, 23], [3, 37, 27, 5, 44, 27, 44, 37, 3, 60, 2, 60, 176, 44, 177, 60, 27], [3, 70, 1, 201, 28, 119, 2, 13, 54, 30, 35, 25, 13, 54, 93, 31, 42, 94, 95, 13, 70, 2, 13, 38, 30, 35, 25, 13, 38, 93, 31, 42, 94, 95, 13, 70, 2, 13, 54, 202, 13, 54, 203, 13, 54, 204, 30, 35, 25, 13, 54, 92, 202, 203, 110, 204, 93, 31, 42, 94, 95, 13, 70, 2, 13, 38, 205, 13, 38, 206, 30, 35, 25, 13, 38, 92, 205, 110, 206, 93, 31, 42, 94, 95, 13, 70], [3, 96, 24, 24, 24, 24, 24, 5, 96, 24, 74, 32, 74, 87, 2, 96, 30, 35, 25, 66, 9, 24, 4, 96], [3, 46, 2, 29, 46, 47, 1, 139, 46, 2, 1, 48, 1, 48, 1, 48, 55, 1, 1, 48, 23, 2, 1, 1, 49, 4, 76, 1, 4, 49], [3, 120, 73, 120, 178, 207, 42, 185, 207, 5, 208, 168, 119, 120, 208], [3, 209, 97, 121, 97, 19, 122, 56, 5, 34, 97, 122, 159, 29, 34, 104, 97, 122, 34, 79, 160, 209, 34], [3, 123, 5, 16, 210, 123, 124, 98, 86, 123, 211, 98, 86, 16, 2, 124, 16, 4, 5, 7, 99, 124, 59, 7, 108, 16, 7, 99, 7, 17, 16, 212, 7, 108, 16, 7, 211, 7, 59, 136, 212, 28, 7, 17], [3, 125, 126, 36, 5, 127, 128, 15, 12, 23, 142, 143, 2, 125, 36, 30, 25, 144, 145, 146, 147, 128, 126, 38, 41, 126, 23, 41, 103, 148, 3, 78, 149, 42, 150, 151, 152, 153, 154, 155, 156, 157, 78, 158, 127, 9, 125, 15, 127, 128, 73, 20, 9, 9, 15, 200, 98, 15, 23, 12, 98, 57, 161, 14, 15, 12, 12, 36, 36], [3, 71, 8, 5, 72, 1, 10, 72, 41, 7, 18, 44, 121, 5, 7, 18, 71, 8, 8, 82, 2, 21, 71, 7, 71, 7, 7, 18, 71, 59, 41, 18, 2, 21, 213, 18, 18, 3, 89, 83, 1, 214, 89, 215, 28, 99, 83, 58, 1, 18, 40, 2, 21, 216, 18, 18, 18, 23, 3, 7, 5, 14, 7, 59, 2, 21, 213, 14, 1, 214, 14, 215, 28, 99, 7, 28, 58, 1, 80, 18, 31, 210, 2, 21, 216, 18, 7, 2, 72, 7, 1, 7, 1, 58, 1, 72, 7, 72, 7, 8, 18], [3, 64, 90, 50, 86, 2, 50, 73, 113, 31, 50, 121, 4, 50, 88, 64]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Pad zero upto maximum length\n",
        "x_train = pad_sequences(x_train_seqs,  maxlen=max_code_len, padding='post')\n",
        "x_validation = pad_sequences(x_validation_seqs, maxlen=max_code_len, padding='post')\n",
        "\n",
        "# Size of vocabulary (+1 for padding token)\n",
        "x_voc = x_tokenizer.num_words + 1\n",
        "\n",
        "print(\"Size of vocabulary in X = {}\".format(x_voc))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RvXB5eKaiFyn",
        "outputId": "ce3b1e64-a15a-4cd4-e348-c89d24af34d7"
      },
      "execution_count": 208,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Size of vocabulary in X = 218\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#fit the tokenizer on the summary sequences\n",
        "y_tokenizer = Tokenizer()\n",
        "y_tokenizer.fit_on_texts(list(y_train))\n",
        "\n",
        "threshold = 2\n",
        "cnt_infrequent = 0\n",
        "total_cnt = 0\n",
        "\n",
        "for key, value in y_tokenizer.word_counts.items():\n",
        "    total_cnt = total_cnt + 1\n",
        "    if value < threshold:\n",
        "        cnt_infrequent = cnt_infrequent + 1\n",
        "\n",
        "y_tokenizer = Tokenizer(num_words = total_cnt - cnt_infrequent)\n",
        "y_tokenizer.fit_on_texts(list(y_train))\n",
        "\n",
        "y_train_seqs = y_tokenizer.texts_to_sequences(y_train)\n",
        "y_validation_seqs = y_tokenizer.texts_to_sequences(y_validation)\n",
        "\n",
        "y_train = pad_sequences(y_train_seqs,  maxlen=max_summary_len, padding='post')\n",
        "y_validation = pad_sequences(y_validation_seqs, maxlen=max_summary_len, padding='post')\n",
        "\n",
        "y_voc = y_tokenizer.num_words + 1\n",
        "print(\"Size of vocabulary in Y = {}\".format(y_voc))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RQ9cro_6YJq6",
        "outputId": "096fb74f-0a7e-4928-d413-9941e2cea783"
      },
      "execution_count": 209,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Size of vocabulary in Y = 61\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(x_train))\n",
        "print(len(y_train))\n",
        "print(len(x_validation))\n",
        "print(len(y_validation))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "usrKCJpkYLyH",
        "outputId": "ec7e318b-add3-4bc2-d035-39b15893c1a6"
      },
      "execution_count": 210,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "37\n",
            "37\n",
            "7\n",
            "7\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# vocab sizes for x and y are 356 ans 105, hence highest integer value would be less than that. Also the maximum lengths of x and y are different\n",
        "print((x_train[0]))\n",
        "print((y_train[0]))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oc3cPv85YM3x",
        "outputId": "a9605e34-61a6-4aa3-ecf3-f4fccafd22d7"
      },
      "execution_count": 211,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[  3 129 130   2  20  73  20 131 132 132 131 129  38 130  23   9   9   0\n",
            "   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "   0   0   0   0   0   0   0   0   0   0]\n",
            "[1 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from tensorflow.keras.layers import Input, LSTM, Embedding, Dense, \\\n",
        "Concatenate, TimeDistributed\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.callbacks import EarlyStopping"
      ],
      "metadata": {
        "id": "tNRq5SyOYOW_"
      },
      "execution_count": 212,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "latent_dim = 300\n",
        "embedding_dim = 200\n",
        "# Encoder\n",
        "encoder_inputs = Input(shape=(max_code_len, ))\n",
        "\n",
        "# Embedding layer\n",
        "enc_emb = Embedding(x_voc, embedding_dim,\n",
        "trainable=True)(encoder_inputs)\n",
        "\n",
        "# Encoder LSTM 1\n",
        "encoder_lstm1 = LSTM(latent_dim, return_sequences=True,\n",
        "return_state=True, dropout=0.4,\n",
        "recurrent_dropout=0.4)\n",
        "(encoder_output1, state_h1, state_c1) = encoder_lstm1(enc_emb)\n",
        "\n",
        "# Encoder LSTM 2\n",
        "encoder_lstm2 = LSTM(latent_dim, return_sequences=True,\n",
        "return_state=True, dropout=0.4,\n",
        "recurrent_dropout=0.4)\n",
        "(encoder_output2, state_h2, state_c2) = encoder_lstm2(encoder_output1)\n",
        "\n",
        "# Encoder LSTM 3\n",
        "encoder_lstm3 = LSTM(latent_dim, return_state=True,\n",
        "return_sequences=True, dropout=0.4,\n",
        "recurrent_dropout=0.4)\n",
        "(encoder_outputs, state_h, state_c) = encoder_lstm3(encoder_output2)\n",
        "\n",
        "# Set up the decoder, using encoder_states as the initial state\n",
        "decoder_inputs = Input(shape=(None, ))\n",
        "\n",
        "# Embedding layer\n",
        "dec_emb_layer = Embedding(y_voc, embedding_dim, trainable=True)\n",
        "dec_emb = dec_emb_layer(decoder_inputs)\n",
        "\n",
        "# Decoder LSTM\n",
        "decoder_lstm = LSTM(latent_dim, return_sequences=True,\n",
        "return_state=True, dropout=0.4,\n",
        "recurrent_dropout=0.2)\n",
        "(decoder_outputs, decoder_fwd_state, decoder_back_state) = \\\n",
        "decoder_lstm(dec_emb, initial_state=[state_h, state_c])\n",
        "\n",
        "# Dense layer\n",
        "decoder_dense = TimeDistributed(Dense(y_voc, activation='softmax'))\n",
        "decoder_outputs = decoder_dense(decoder_outputs)\n",
        "\n",
        "# Define the model\n",
        "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)\n",
        "model.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5CP2NcT2YP37",
        "outputId": "2cbfa3ab-94d2-45a7-c119-07a8c16b29c6"
      },
      "execution_count": 213,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_6 (InputLayer)           [(None, 100)]        0           []                               \n",
            "                                                                                                  \n",
            " embedding_2 (Embedding)        (None, 100, 200)     43600       ['input_6[0][0]']                \n",
            "                                                                                                  \n",
            " lstm_4 (LSTM)                  [(None, 100, 300),   601200      ['embedding_2[0][0]']            \n",
            "                                 (None, 300),                                                     \n",
            "                                 (None, 300)]                                                     \n",
            "                                                                                                  \n",
            " input_7 (InputLayer)           [(None, None)]       0           []                               \n",
            "                                                                                                  \n",
            " lstm_5 (LSTM)                  [(None, 100, 300),   721200      ['lstm_4[0][0]']                 \n",
            "                                 (None, 300),                                                     \n",
            "                                 (None, 300)]                                                     \n",
            "                                                                                                  \n",
            " embedding_3 (Embedding)        (None, None, 200)    12200       ['input_7[0][0]']                \n",
            "                                                                                                  \n",
            " lstm_6 (LSTM)                  [(None, 100, 300),   721200      ['lstm_5[0][0]']                 \n",
            "                                 (None, 300),                                                     \n",
            "                                 (None, 300)]                                                     \n",
            "                                                                                                  \n",
            " lstm_7 (LSTM)                  [(None, None, 300),  601200      ['embedding_3[0][0]',            \n",
            "                                 (None, 300),                     'lstm_6[0][1]',                 \n",
            "                                 (None, 300)]                     'lstm_6[0][2]']                 \n",
            "                                                                                                  \n",
            " time_distributed_1 (TimeDistri  (None, None, 61)    18361       ['lstm_7[0][0]']                 \n",
            " buted)                                                                                           \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 2,718,961\n",
            "Trainable params: 2,718,961\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy')\n",
        "es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=4)\n",
        "\n",
        "history = model.fit(\n",
        "[x_train, y_train[:, :-1]],\n",
        "y_train.reshape(y_train.shape[0], y_train.shape[1], 1)[:, 1:],\n",
        "epochs=50,\n",
        "callbacks=[es],\n",
        "batch_size=128,\n",
        "validation_data=([x_validation, y_validation[:, :-1]],\n",
        "y_validation.reshape(y_validation.shape[0], y_validation.shape[1], 1)[:, 1:]),\n",
        ")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OciIeDIDYRmj",
        "outputId": "1a8aa72b-d0e0-448a-84bc-1a5309fa890b"
      },
      "execution_count": 228,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/50\n",
            "1/1 [==============================] - 17s 17s/step - loss: 0.7932 - val_loss: 0.7374\n",
            "Epoch 2/50\n",
            "1/1 [==============================] - 4s 4s/step - loss: 0.8282 - val_loss: 1.1480\n",
            "Epoch 3/50\n",
            "1/1 [==============================] - 4s 4s/step - loss: 1.1651 - val_loss: 0.8191\n",
            "Epoch 4/50\n",
            "1/1 [==============================] - 4s 4s/step - loss: 1.0136 - val_loss: 0.8922\n",
            "Epoch 5/50\n",
            "1/1 [==============================] - 6s 6s/step - loss: 1.0618 - val_loss: 0.7079\n",
            "Epoch 6/50\n",
            "1/1 [==============================] - 4s 4s/step - loss: 0.9009 - val_loss: 0.6922\n",
            "Epoch 7/50\n",
            "1/1 [==============================] - 4s 4s/step - loss: 0.7931 - val_loss: 0.6856\n",
            "Epoch 8/50\n",
            "1/1 [==============================] - 4s 4s/step - loss: 0.7589 - val_loss: 0.6973\n",
            "Epoch 9/50\n",
            "1/1 [==============================] - 4s 4s/step - loss: 0.7394 - val_loss: 0.6996\n",
            "Epoch 10/50\n",
            "1/1 [==============================] - 4s 4s/step - loss: 0.7392 - val_loss: 0.7049\n",
            "Epoch 11/50\n",
            "1/1 [==============================] - 4s 4s/step - loss: 0.7413 - val_loss: 0.7376\n",
            "Epoch 11: early stopping\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# for prediction, define the decoding sequence\n",
        "\n",
        "reverse_target_word_index = y_tokenizer.index_word\n",
        "reverse_source_word_index = x_tokenizer.index_word\n",
        "target_word_index = y_tokenizer.word_index\n",
        "encoder_model = Model(inputs=encoder_inputs, outputs=[encoder_outputs,\n",
        "state_h, state_c])\n",
        "\n",
        "# Decoder setup\n",
        "\n",
        "# Below tensors will hold the states of the previous time step\n",
        "decoder_state_input_h = Input(shape=(latent_dim, ))\n",
        "decoder_state_input_c = Input(shape=(latent_dim, ))\n",
        "decoder_hidden_state_input = Input(shape=(max_code_len, latent_dim))\n",
        "\n",
        "# Get the embeddings of the decoder sequence\n",
        "dec_emb2 = dec_emb_layer(decoder_inputs)\n",
        "\n",
        "# To predict the next word in the sequence, set the initial states to the states from the previous time step\n",
        "(decoder_outputs2, state_h2, state_c2) = decoder_lstm(dec_emb2,\n",
        "initial_state=[decoder_state_input_h, decoder_state_input_c])\n",
        "\n",
        "# A dense softmax layer to generate prob dist. over the target vocabulary\n",
        "decoder_outputs2 = decoder_dense(decoder_outputs2)\n",
        "\n",
        "# Final decoder model\n",
        "decoder_model = Model([decoder_inputs] + [decoder_hidden_state_input,\n",
        "decoder_state_input_h, decoder_state_input_c],\n",
        "[decoder_outputs2] + [state_h2, state_c2])"
      ],
      "metadata": {
        "id": "BgmGwkBzYWE_"
      },
      "execution_count": 229,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# generating the decoded sequence\n",
        "\n",
        "def decode_sequence(input_seq):\n",
        "\n",
        "    # Encode the input as state vectors.\n",
        "    (e_out, e_h, e_c) = encoder_model.predict(input_seq)\n",
        "\n",
        "    # Generate empty target sequence of length 1\n",
        "    target_seq = np.zeros((1, 1))\n",
        "\n",
        "    # Populate the first word of target sequence with the start word.\n",
        "    target_seq[0, 0] = target_word_index['sostok']\n",
        "\n",
        "    stop_condition = False\n",
        "    decoded_sentence = ''\n",
        "\n",
        "    while not stop_condition:\n",
        "        (output_tokens, h, c) = decoder_model.predict([target_seq]\n",
        "                + [e_out, e_h, e_c])\n",
        "\n",
        "        # Sample a token\n",
        "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
        "        sampled_token = reverse_target_word_index[sampled_token_index]\n",
        "\n",
        "        if sampled_token != 'eostok':\n",
        "            decoded_sentence += ' ' + sampled_token\n",
        "\n",
        "        # Exit condition: either hit max length or find the stop word.\n",
        "        if sampled_token == 'eostok' or len(decoded_sentence.split()) \\\n",
        "            >= max_summary_len - 1:\n",
        "            stop_condition = True\n",
        "\n",
        "        # Update the target sequence (of length 1)\n",
        "        target_seq = np.zeros((1, 1))\n",
        "        target_seq[0, 0] = sampled_token_index\n",
        "\n",
        "        # Update internal states\n",
        "        (e_h, e_c) = (h, c)\n",
        "\n",
        "    return decoded_sentence"
      ],
      "metadata": {
        "id": "xZe6M0DfYZrV"
      },
      "execution_count": 230,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def seq2summary(input_seq):\n",
        "    newString = ''\n",
        "    for i in input_seq:\n",
        "        if i != 0 and i != target_word_index['sostok'] and i \\\n",
        "        != target_word_index['eostok']:\n",
        "            newString = newString + reverse_target_word_index[i] + ' '\n",
        "\n",
        "    return newString\n",
        "\n",
        "\n",
        "# To convert sequence to text\n",
        "def seq2text(input_seq):\n",
        "    newString = ''\n",
        "    for i in input_seq:\n",
        "        if i != 0:\n",
        "            newString = newString + reverse_source_word_index[i] + ' '\n",
        "\n",
        "    return newString"
      ],
      "metadata": {
        "id": "IPuAQHXDYbro"
      },
      "execution_count": 231,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in range(10, 15):\n",
        "    print ('Code:', seq2text(x_train[i]))\n",
        "    print ('Original summary:', seq2summary(y_train[i]))\n",
        "    print ('Predicted summary:', decode_sequence(x_train[i].reshape(1,\n",
        "    max_code_len)))\n",
        "    print ('\\n')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ICQkIPdGYdwB",
        "outputId": "44e2e148-b840-4ce5-d68c-8308a6dd7700"
      },
      "execution_count": 232,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Code: function if in conf if in conf log process env node logger plugin conf log plugin if in conf log process env node logger level conf log level if in conf log for var key in conf log customlevels process env logger level ' key conf log customlevels key \n",
            "Original summary: of \n",
            "Predicted summary:  the a in in opts or param param param\n",
            "\n",
            "\n",
            "Code: function map var key for key in map if map hasownproperty key this init key map key return this \n",
            "Original summary: subviews with a map of subview keys to additional options param object map return subviewmanager \n",
            "Predicted summary:  the a in in opts or param param param\n",
            "\n",
            "\n",
            "Code: function message this message message this true error this arguments \n",
            "Original summary: param string message constructor \n",
            "Predicted summary:  the a in in opts or param param param\n",
            "\n",
            "\n",
            "Code: function dispose if typeof this destroy 'function' this destroy else if typeof this stop 'function' this stop else if typeof this kill 'function' this kill else if typeof this end 'function' this end else throw new error to dispose \n",
            "Original summary: \n",
            "Predicted summary:  the a in in opts or param param param\n",
            "\n",
            "\n",
            "Code: function str var arr str ' ' var obj arr shift while arr length obj obj obj arr shift return obj \n",
            "Original summary: from a string \n",
            "Predicted summary:  the a in in opts or param param param\n",
            "\n",
            "\n"
          ]
        }
      ]
    }
  ]
}