Machine_Translation_Model.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Exm8GljwuJt8",
        "outputId": "1b7926fd-8d1e-4a2d-cea6-4bb806fa6902"
      },
      "outputs": [],
      "source": [
        "import numpy as np \n",
        "import pandas as pd \n",
        "import os\n",
        "import string\n",
        "from string import digits\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "import re\n",
        "import seaborn as sns\n",
        "from sklearn.utils import shuffle\n",
        "from sklearn.model_selection import train_test_split\n",
        "from keras.layers import Input, LSTM, Embedding, Dense\n",
        "from keras.models import Model\n",
        "\n",
        "pd.set_option('display.max_rows', 500)\n",
        "pd.set_option('display.max_columns', 500)\n",
        "pd.set_option('display.width', 1000)\n",
        "pd.set_option('display.max_colwidth', None)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "id": "PdM7Fim_sPFa"
      },
      "outputs": [],
      "source": [
        "#ADDING THE DATASET\n",
        "\n",
        "path =(\"D:\\project\\Dataset_1.csv\")\n",
        "data=pd.read_csv(path)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UaYPZwpxsNs7",
        "outputId": "e48816f7-13cb-4430-ce57-3ff06ce59a60"
      },
      "outputs": [],
      "source": [
        "#\n",
        "#data['source'].value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 669
        },
        "id": "yNYtJlgl1ElB",
        "outputId": "a9ce03b4-0158-441a-b2be-1205ade6b26f"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>source</th>\n",
              "      <th>english_sentence</th>\n",
              "      <th>hindi_sentence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>This percentage is even greater than the percentage in India.</td>\n",
              "      <td>यह प्रतिशत भारत में हिन्दुओं प्रतिशत से अधिक है।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>.The ending portion of these Vedas is called Upanishad.</td>\n",
              "      <td>इन्हीं वेदों का अंतिम भाग उपनिषद कहलाता है।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>In this lies the circumstances of people before you.</td>\n",
              "      <td>इसमें तुमसे पूर्व गुज़रे हुए लोगों के हालात हैं।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>“”Global Warming“” refer to warming caused in recent decades and probability of its continual presence and its indirect effect on human being.</td>\n",
              "      <td>ग्लोबल वॉर्मिंग से आशय हाल ही के दशकों में हुई वार्मिंग और इसके निरंतर बने रहने के अनुमान और इसके अप्रत्यक्ष रूप से मानव पर पड़ने वाले प्रभाव से है।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>Category: Religious Text</td>\n",
              "      <td>श्रेणी:धर्मग्रन्थ</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>This period summarily is pepped up with devotion.</td>\n",
              "      <td>यह काल समग्रतः भक्ति भावना से ओतप्रोत काल है।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>And now at present the naturecure, Ayurvedic and modern treatments are taking place through the government in Nepal.</td>\n",
              "      <td>हाल में नेपाल के हस्पताल सामन्यतया आयुर्वेद, प्राकृतिक चिकित्सा तथा आधुनिक चिकीत्सा करके सरकारी सेवा विद्यमान हे ।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>Parliament time frame is 5 years and this will be dissolved before that.</td>\n",
              "      <td>लोकसभा की कार्यावधि 5 वर्ष है पर्ंतु इसे समय से पूर्व भंग किया जा सकता है</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>Extreme weather due to increased mortality; displacements and economic loss will be compounded through growing population. Although, temperate climate has some benefits out of it such as decreased mortality due to cold weather.</td>\n",
              "      <td>बढ़ती हुई मौतों displacements और आर्थिक नुकसान जो की अतिवादी मौसम (extreme weather)के कारण संभावित हैं बढती हुई जनसँख्या (growing population)के कारण और भी बदतर हो सकते हैं . हालांकि शीतोष्ण क्षेत्र में इसके कुछ फैदे भी हो सकते हैं जैसे की ठंड की वजह से कम मौतें होना .</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>Islam is the world's second-largest religion, after Christianity.</td>\n",
              "      <td>इस्लाम धर्म (الإسلام) ईसाई धर्म के बाद अनुयाइयों के आधार पर दुनिया का दूसरा सब से बड़ा धर्म है।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>Maine</td>\n",
              "      <td>मेन</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>IN farsi philosophy firdaus garden is presented as ideal one in Mugal literatures.</td>\n",
              "      <td>फारसी रहस्यवाद में मुगल कालीन इस्लामी पाठ्य में फिरदौस को एक आदर्श पूर्णता का बाग बताया गया है।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>But after his demise, his bother Sardar Vallabh Bhai Patel did not accept this will and filed a case in the court.</td>\n",
              "      <td>मगर उनके निधन के पश्चात उनके भाई सरदार वल्लभ भाई पटेल ने इस वसीयत को स्वीकार नहीं किया और उसपर अदालत में मुकदमा चलाया।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>This position is similar to armchair. In this the male puts its legs slightly up near the knees. and his hands are helpful to support the hips of the female. The female spreads her leg touching his back.and after letting her penis inside her vagina takes support on his arms by her hands.In this,he male and female remains quite closer and this provides quite closure.</td>\n",
              "      <td>-झूला यह हत्थाकुर्सी से मिलती जुलती पोजीशन है इसमें पुरुष अपने पैरों को घुटनों के पास से थोड़ा उपर उठा लेता है और उसके हाथ महिला को कूल्हों के पास से सहारा देने का काम करते हैं तथा महिला अपने दोनों पैर उसकी कमर से सटाते हुए सीधे फैला लेती है तथा योनि को लिंग में प्रवेश कराने के बाद अपने हाथ उसके कंधे पर रख सहारा लेती है. इसमें महिला पुरुष काफी निकट रहते हैं इससे यह काफी निकटता प्रदान करती है.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>category:information technology</td>\n",
              "      <td>श्रेणी:सूचना प्रौद्योगिकी</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>57</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>Aryans did not make any statues or temples for deities.</td>\n",
              "      <td>आर्य देवताओं की कोई मूर्ति या मन्दिर नहीं बनाते थे।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>58</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>It has three small domes and has been constructed using white marble.</td>\n",
              "      <td>यह एक छोटी तीन गुम्बद वाली तराशे हुए श्वेत संगमर्मर से निर्मित है।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>.Sarojini Naidu with Mahatma Gandhi</td>\n",
              "      <td>महात्मा गांधी के साथ सरोजिनी नायडू</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>61</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>External links</td>\n",
              "      <td>बाहरी कड़ियाँ</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>Buddha and danceroom (1958)</td>\n",
              "      <td>बुद्ध और नाचघर (1958)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       source                                                                                                                                                                                                                                                                                                                                                                  english_sentence                                                                                                                                                                                                                                                                                                                                                                                               hindi_sentence\n",
              "2   indic2012                                                                                                                                                                                                                                                                                                                     This percentage is even greater than the percentage in India.                                                                                                                                                                                                                                                                                                                                                             यह प्रतिशत भारत में हिन्दुओं प्रतिशत से अधिक है।\n",
              "4   indic2012                                                                                                                                                                                                                                                                                                                           .The ending portion of these Vedas is called Upanishad.                                                                                                                                                                                                                                                                                                                                                                  इन्हीं वेदों का अंतिम भाग उपनिषद कहलाता है।\n",
              "6   indic2012                                                                                                                                                                                                                                                                                                                              In this lies the circumstances of people before you.                                                                                                                                                                                                                                                                                                                                                             इसमें तुमसे पूर्व गुज़रे हुए लोगों के हालात हैं।\n",
              "8   indic2012                                                                                                                                                                                                                                    “”Global Warming“” refer to warming caused in recent decades and probability of its continual presence and its indirect effect on human being.                                                                                                                                                                                                                                                         ग्लोबल वॉर्मिंग से आशय हाल ही के दशकों में हुई वार्मिंग और इसके निरंतर बने रहने के अनुमान और इसके अप्रत्यक्ष रूप से मानव पर पड़ने वाले प्रभाव से है।\n",
              "11  indic2012                                                                                                                                                                                                                                                                                                                                                          Category: Religious Text                                                                                                                                                                                                                                                                                                                                                                                            श्रेणी:धर्मग्रन्थ\n",
              "12  indic2012                                                                                                                                                                                                                                                                                                                                 This period summarily is pepped up with devotion.                                                                                                                                                                                                                                                                                                                                                                यह काल समग्रतः भक्ति भावना से ओतप्रोत काल है।\n",
              "16  indic2012                                                                                                                                                                                                                                                              And now at present the naturecure, Ayurvedic and modern treatments are taking place through the government in Nepal.                                                                                                                                                                                                                                                                                           हाल में नेपाल के हस्पताल सामन्यतया आयुर्वेद, प्राकृतिक चिकित्सा तथा आधुनिक चिकीत्सा करके सरकारी सेवा विद्यमान हे ।\n",
              "17  indic2012                                                                                                                                                                                                                                                                                                          Parliament time frame is 5 years and this will be dissolved before that.                                                                                                                                                                                                                                                                                                                                    लोकसभा की कार्यावधि 5 वर्ष है पर्ंतु इसे समय से पूर्व भंग किया जा सकता है\n",
              "19  indic2012                                                                                                                                               Extreme weather due to increased mortality; displacements and economic loss will be compounded through growing population. Although, temperate climate has some benefits out of it such as decreased mortality due to cold weather.                                                                                                                                 बढ़ती हुई मौतों displacements और आर्थिक नुकसान जो की अतिवादी मौसम (extreme weather)के कारण संभावित हैं बढती हुई जनसँख्या (growing population)के कारण और भी बदतर हो सकते हैं . हालांकि शीतोष्ण क्षेत्र में इसके कुछ फैदे भी हो सकते हैं जैसे की ठंड की वजह से कम मौतें होना .\n",
              "22  indic2012                                                                                                                                                                                                                                                                                                                 Islam is the world's second-largest religion, after Christianity.                                                                                                                                                                                                                                                                                                             इस्लाम धर्म (الإسلام) ईसाई धर्म के बाद अनुयाइयों के आधार पर दुनिया का दूसरा सब से बड़ा धर्म है।\n",
              "33  indic2012                                                                                                                                                                                                                                                                                                                                                                             Maine                                                                                                                                                                                                                                                                                                                                                                                                          मेन\n",
              "34  indic2012                                                                                                                                                                                                                                                                                                IN farsi philosophy firdaus garden is presented as ideal one in Mugal literatures.                                                                                                                                                                                                                                                                                                              फारसी रहस्यवाद में मुगल कालीन इस्लामी पाठ्य में फिरदौस को एक आदर्श पूर्णता का बाग बताया गया है।\n",
              "40  indic2012                                                                                                                                                                                                                                                                But after his demise, his bother Sardar Vallabh Bhai Patel did not accept this will and filed a case in the court.                                                                                                                                                                                                                                                                                       मगर उनके निधन के पश्चात उनके भाई सरदार वल्लभ भाई पटेल ने इस वसीयत को स्वीकार नहीं किया और उसपर अदालत में मुकदमा चलाया।\n",
              "48  indic2012  This position is similar to armchair. In this the male puts its legs slightly up near the knees. and his hands are helpful to support the hips of the female. The female spreads her leg touching his back.and after letting her penis inside her vagina takes support on his arms by her hands.In this,he male and female remains quite closer and this provides quite closure.  -झूला यह हत्थाकुर्सी से मिलती जुलती पोजीशन है इसमें पुरुष अपने पैरों को घुटनों के पास से थोड़ा उपर उठा लेता है और उसके हाथ महिला को कूल्हों के पास से सहारा देने का काम करते हैं तथा महिला अपने दोनों पैर उसकी कमर से सटाते हुए सीधे फैला लेती है तथा योनि को लिंग में प्रवेश कराने के बाद अपने हाथ उसके कंधे पर रख सहारा लेती है. इसमें महिला पुरुष काफी निकट रहते हैं इससे यह काफी निकटता प्रदान करती है.\n",
              "50  indic2012                                                                                                                                                                                                                                                                                                                                                   category:information technology                                                                                                                                                                                                                                                                                                                                                                                    श्रेणी:सूचना प्रौद्योगिकी\n",
              "57  indic2012                                                                                                                                                                                                                                                                                                                           Aryans did not make any statues or temples for deities.                                                                                                                                                                                                                                                                                                                                                          आर्य देवताओं की कोई मूर्ति या मन्दिर नहीं बनाते थे।\n",
              "58  indic2012                                                                                                                                                                                                                                                                                                             It has three small domes and has been constructed using white marble.                                                                                                                                                                                                                                                                                                                                           यह एक छोटी तीन गुम्बद वाली तराशे हुए श्वेत संगमर्मर से निर्मित है।\n",
              "59  indic2012                                                                                                                                                                                                                                                                                                                                               .Sarojini Naidu with Mahatma Gandhi                                                                                                                                                                                                                                                                                                                                                                           महात्मा गांधी के साथ सरोजिनी नायडू\n",
              "61  indic2012                                                                                                                                                                                                                                                                                                                                                                    External links                                                                                                                                                                                                                                                                                                                                                                                                बाहरी कड़ियाँ\n",
              "69  indic2012                                                                                                                                                                                                                                                                                                                                                       Buddha and danceroom (1958)                                                                                                                                                                                                                                                                                                                                                                                        बुद्ध और नाचघर (1958)"
            ]
          },
          "execution_count": 32,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "lines=data[data['source']=='indic2012']\n",
        "lines.head(20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "L3nuADrq27HC"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X0lhzOWX2cVN",
        "outputId": "63ca89d9-653c-4157-b215-c50332c94e9b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "source              0\n",
              "english_sentence    2\n",
              "hindi_sentence      0\n",
              "dtype: int64"
            ]
          },
          "execution_count": 33,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Checking for null values\n",
        "pd.isnull(lines).sum() "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "id": "9_-Sggo02eaN"
      },
      "outputs": [],
      "source": [
        "#Dropping null values if any\n",
        "lines=lines[~pd.isnull(lines['english_sentence'])]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "id": "sxEk3lwX2q_X"
      },
      "outputs": [],
      "source": [
        "#Dropping duplicates values if any\n",
        "lines.drop_duplicates(inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8SdyiGXo8NHO",
        "outputId": "70844e57-f6c1-4a88-ed4d-fcac4616e083"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>source</th>\n",
              "      <th>english_sentence</th>\n",
              "      <th>hindi_sentence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>43137</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>Occurring in the environment recently increased carbon di Oxide (CO2) monthly CO2 measurements Dershat that if you saw all year get to see the seasonal changes are small; every year Paryaverton Northern Hemisphere (Northern Hemisphere) seasons in the Aswh At the end of the Northern Hemisphere crops have become more Bjne is the time get at the change of environment because the plants remove some CO2 charge</td>\n",
              "      <td>वातावरण में कार्बन डाइऑक्साइड में हाल ही में होने वाली बढोतरी ( CO2 ) .मासिक CO2 मापन यह दर्शाते हैं कि अगर सारे वर्ष को देखा जाए तो छोटे-छोटे मौसमी परिवर्तन देखने को मिलते हैं ; हर साल यह परिवर्तन उत्तरी गोलार्ध (Northern Hemisphere) में वसव्त मौसम के आख़िर में अधिक हो जाते हैं और जब उत्तरी गोलार्ध में फसलें बीजने का समय होता है तो यह परिवर्तन कम हो जाते हैं क्यूंकि पौधे वातावरण में से कुछ CO2 हटा लेते हैंI</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>107772</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>Her father Aghoranath Chattopadhyay was a well-known scholar and mother was a poet who wrote in Bengali.</td>\n",
              "      <td>इनके पिता अघोरनाथ चट्टोपाध्याय एक नामी विद्वान तथा माँ कवयित्री थीं और बांग्ला में लिखती थीं ।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>120507</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>Nishadraj Guh served those three very well in Ringverpur.</td>\n",
              "      <td>ऋंगवेरपुर में निषादराज गुह ने तीनों की बहुत सेवा की।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28755</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>2. The ordinance that was rejected by Loksabha, can be passed again before six weeks period.</td>\n",
              "      <td>2. लोकसभा एक अध्यादेश को अस्वीकृत करने वाला प्रस्ताव 6 सप्ताह की अवधि समाप्त होने से पूर्व पास कर सकती है</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86303</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>Number of 18</td>\n",
              "      <td>अठारह की संख्या</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>117657</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>There are 6 political parties in India, But the Indian national congress is the most powerful in India</td>\n",
              "      <td>बहुदलिय राजनीतिक व्यवस्था वाले इस देश में ६ राष्ट्रीय स्तर के राजनीतिक दल हैं किंतु यहां की राजनीतिक व्यवस्था पर सर्वाधीक प्रभावी भारतीय राष्ट्रीय कांग्रेस रही है।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>89633</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>They had the experience of ruling.</td>\n",
              "      <td>उन्हें शासन करने का अनुभव है ।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>127215</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>Ram Kabir comprehensive nature of the concept and wanted to give a different</td>\n",
              "      <td>कबीर राम की अवधारणा को एक भिन्न और व्यापक स्वरूप देना चाहते थे।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>67194</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>this country has many famous places like Agra, Aligad,Ayodhya,Berle,Me rut,Varanasi, (Banaras)Gorakhpur,Ghaziabad,Muradabad,Saharanpur, Faisalabad,Kanpur,</td>\n",
              "      <td>प्रदेश के अन्य महत्त्वपूर्ण शहर हैं आगरा अलीगढ अयोध्या बरेली मेरठ वाराणसी( बनारस) गोरखपुर गाज़ियाबाद मुरादाबाद सहारनपुर फ़ैज़ाबाद कानपुर।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>71489</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>This is the reason why the vedas are also called “”Shrutis“”. (that which is heard)</td>\n",
              "      <td>इसी कारण वेद को ‘श्रुति' भी कहते हैं।</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>25000 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "           source                                                                                                                                                                                                                                                                                                                                                                                                          english_sentence                                                                                                                                                                                                                                                                                                                                                                                                               hindi_sentence\n",
              "43137   indic2012  Occurring in the environment recently increased carbon di Oxide (CO2) monthly CO2 measurements Dershat that if you saw all year get to see the seasonal changes are small; every year Paryaverton Northern Hemisphere (Northern Hemisphere) seasons in the Aswh At the end of the Northern Hemisphere crops have become more Bjne is the time get at the change of environment because the plants remove some CO2 charge  वातावरण में कार्बन डाइऑक्साइड में हाल ही में होने वाली बढोतरी ( CO2 ) .मासिक CO2 मापन यह दर्शाते हैं कि अगर सारे वर्ष को देखा जाए तो छोटे-छोटे मौसमी परिवर्तन देखने को मिलते हैं ; हर साल यह परिवर्तन उत्तरी गोलार्ध (Northern Hemisphere) में वसव्त मौसम के आख़िर में अधिक हो जाते हैं और जब उत्तरी गोलार्ध में फसलें बीजने का समय होता है तो यह परिवर्तन कम हो जाते हैं क्यूंकि पौधे वातावरण में से कुछ CO2 हटा लेते हैंI\n",
              "107772  indic2012                                                                                                                                                                                                                                                                                                                  Her father Aghoranath Chattopadhyay was a well-known scholar and mother was a poet who wrote in Bengali.                                                                                                                                                                                                                                                                                                                               इनके पिता अघोरनाथ चट्टोपाध्याय एक नामी विद्वान तथा माँ कवयित्री थीं और बांग्ला में लिखती थीं ।\n",
              "120507  indic2012                                                                                                                                                                                                                                                                                                                                                                 Nishadraj Guh served those three very well in Ringverpur.                                                                                                                                                                                                                                                                                                                                                                         ऋंगवेरपुर में निषादराज गुह ने तीनों की बहुत सेवा की।\n",
              "28755   indic2012                                                                                                                                                                                                                                                                                                                              2. The ordinance that was rejected by Loksabha, can be passed again before six weeks period.                                                                                                                                                                                                                                                                                                                    2. लोकसभा एक अध्यादेश को अस्वीकृत करने वाला प्रस्ताव 6 सप्ताह की अवधि समाप्त होने से पूर्व पास कर सकती है\n",
              "86303   indic2012                                                                                                                                                                                                                                                                                                                                                                                                              Number of 18                                                                                                                                                                                                                                                                                                                                                                                                              अठारह की संख्या\n",
              "...           ...                                                                                                                                                                                                                                                                                                                                                                                                                       ...                                                                                                                                                                                                                                                                                                                                                                                                                          ...\n",
              "117657  indic2012                                                                                                                                                                                                                                                                                                                    There are 6 political parties in India, But the Indian national congress is the most powerful in India                                                                                                                                                                                                                                                          बहुदलिय राजनीतिक व्यवस्था वाले इस देश में ६ राष्ट्रीय स्तर के राजनीतिक दल हैं किंतु यहां की राजनीतिक व्यवस्था पर सर्वाधीक प्रभावी भारतीय राष्ट्रीय कांग्रेस रही है।\n",
              "89633   indic2012                                                                                                                                                                                                                                                                                                                                                                                        They had the experience of ruling.                                                                                                                                                                                                                                                                                                                                                                                               उन्हें शासन करने का अनुभव है ।\n",
              "127215  indic2012                                                                                                                                                                                                                                                                                                                                              Ram Kabir comprehensive nature of the concept and wanted to give a different                                                                                                                                                                                                                                                                                                                                                              कबीर राम की अवधारणा को एक भिन्न और व्यापक स्वरूप देना चाहते थे।\n",
              "67194   indic2012                                                                                                                                                                                                                                                                this country has many famous places like Agra, Aligad,Ayodhya,Berle,Me rut,Varanasi, (Banaras)Gorakhpur,Ghaziabad,Muradabad,Saharanpur, Faisalabad,Kanpur,                                                                                                                                                                                                                                                                                    प्रदेश के अन्य महत्त्वपूर्ण शहर हैं आगरा अलीगढ अयोध्या बरेली मेरठ वाराणसी( बनारस) गोरखपुर गाज़ियाबाद मुरादाबाद सहारनपुर फ़ैज़ाबाद कानपुर।\n",
              "71489   indic2012                                                                                                                                                                                                                                                                                                                                       This is the reason why the vedas are also called “”Shrutis“”. (that which is heard)                                                                                                                                                                                                                                                                                                                                                                                        इसी कारण वेद को ‘श्रुति' भी कहते हैं।\n",
              "\n",
              "[25000 rows x 3 columns]"
            ]
          },
          "execution_count": 36,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Extracting Small Part of Dataset to cross check if any null values are present\n",
        "\n",
        "lines=lines.sample(n=25000,random_state=42)\n",
        "lines\n",
        "#lines.shape\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {
        "id": "QgMzq7GK2_Dx"
      },
      "outputs": [],
      "source": [
        "#USING LAMBDA FUNCTIIONS"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "id": "8jnagUvF8VHX"
      },
      "outputs": [],
      "source": [
        "# Lowercase all characters\n",
        "lines['english_sentence']=lines['english_sentence'].apply(lambda x: x.lower())\n",
        "lines['hindi_sentence']=lines['hindi_sentence'].apply(lambda x: x.lower())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "id": "OD5WyIGZ8b1x"
      },
      "outputs": [],
      "source": [
        "# Remove quotes\n",
        "lines['english_sentence']=lines['english_sentence'].apply(lambda x: re.sub(\"'\", '', x))\n",
        "lines['hindi_sentence']=lines['hindi_sentence'].apply(lambda x: re.sub(\"'\", '', x))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "id": "g2ulkBsF8d5p"
      },
      "outputs": [],
      "source": [
        "# Set of all special characters\n",
        "exclude = set(string.punctuation) \n",
        "#print(exclude)\n",
        "\n",
        "# Remove all the special characters\n",
        "lines['english_sentence']=lines['english_sentence'].apply(lambda x: ''.join(ch for ch in x if ch not in exclude))\n",
        "lines['hindi_sentence']=lines['hindi_sentence'].apply(lambda x: ''.join(ch for ch in x if ch not in exclude))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {
        "id": "HZH-snBG87KX"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{48: None, 49: None, 50: None, 51: None, 52: None, 53: None, 54: None, 55: None, 56: None, 57: None}\n"
          ]
        }
      ],
      "source": [
        "# Remove all numbers from text\n",
        "remove_digits = str.maketrans('', '', digits)\n",
        "print(remove_digits)\n",
        "lines['english_sentence']=lines['english_sentence'].apply(lambda x: x.translate(remove_digits))\n",
        "lines['hindi_sentence']=lines['hindi_sentence'].apply(lambda x: x.translate(remove_digits))\n",
        "\n",
        "lines['hindi_sentence'] = lines['hindi_sentence'].apply(lambda x: re.sub(\"[२३०८१५७९४६]\", \"\", x))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {
        "id": "_rFMYzk99Asd"
      },
      "outputs": [],
      "source": [
        "# Remove extra spaces\n",
        "lines['english_sentence']=lines['english_sentence'].apply(lambda x: x.strip())\n",
        "lines['hindi_sentence']=lines['hindi_sentence'].apply(lambda x: x.strip())\n",
        "\n",
        "\n",
        "lines['english_sentence']=lines['english_sentence'].apply(lambda x: re.sub(\" +\", \" \", x))\n",
        "lines['hindi_sentence']=lines['hindi_sentence'].apply(lambda x: re.sub(\" +\", \" \", x))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {
        "id": "D3XBwfCy9Fzi"
      },
      "outputs": [],
      "source": [
        "# Add start and end tokens to target sequences\n",
        "lines['hindi_sentence'] = lines['hindi_sentence'].apply(lambda x : 'START_ '+ x + ' _END')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "id": "6SXqq4-79Lss",
        "outputId": "9dac08ee-9d57-4151-844e-bd52edb89d50"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>source</th>\n",
              "      <th>english_sentence</th>\n",
              "      <th>hindi_sentence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>43137</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>occurring in the environment recently increased carbon di oxide co monthly co measurements dershat that if you saw all year get to see the seasonal changes are small every year paryaverton northern hemisphere northern hemisphere seasons in the aswh at the end of the northern hemisphere crops have become more bjne is the time get at the change of environment because the plants remove some co charge</td>\n",
              "      <td>START_ वातावरण में कार्बन डाइऑक्साइड में हाल ही में होने वाली बढोतरी co मासिक co मापन यह दर्शाते हैं कि अगर सारे वर्ष को देखा जाए तो छोटेछोटे मौसमी परिवर्तन देखने को मिलते हैं हर साल यह परिवर्तन उत्तरी गोलार्ध northern hemisphere में वसव्त मौसम के आख़िर में अधिक हो जाते हैं और जब उत्तरी गोलार्ध में फसलें बीजने का समय होता है तो यह परिवर्तन कम हो जाते हैं क्यूंकि पौधे वातावरण में से कुछ co हटा लेते हैंi _END</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>107772</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>her father aghoranath chattopadhyay was a wellknown scholar and mother was a poet who wrote in bengali</td>\n",
              "      <td>START_ इनके पिता अघोरनाथ चट्टोपाध्याय एक नामी विद्वान तथा माँ कवयित्री थीं और बांग्ला में लिखती थीं । _END</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>120507</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>nishadraj guh served those three very well in ringverpur</td>\n",
              "      <td>START_ ऋंगवेरपुर में निषादराज गुह ने तीनों की बहुत सेवा की। _END</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28755</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>the ordinance that was rejected by loksabha can be passed again before six weeks period</td>\n",
              "      <td>START_ लोकसभा एक अध्यादेश को अस्वीकृत करने वाला प्रस्ताव सप्ताह की अवधि समाप्त होने से पूर्व पास कर सकती है _END</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86303</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>number of</td>\n",
              "      <td>START_ अठारह की संख्या _END</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           source                                                                                                                                                                                                                                                                                                                                                                                                  english_sentence                                                                                                                                                                                                                                                                                                                                                                                                              hindi_sentence\n",
              "43137   indic2012  occurring in the environment recently increased carbon di oxide co monthly co measurements dershat that if you saw all year get to see the seasonal changes are small every year paryaverton northern hemisphere northern hemisphere seasons in the aswh at the end of the northern hemisphere crops have become more bjne is the time get at the change of environment because the plants remove some co charge  START_ वातावरण में कार्बन डाइऑक्साइड में हाल ही में होने वाली बढोतरी co मासिक co मापन यह दर्शाते हैं कि अगर सारे वर्ष को देखा जाए तो छोटेछोटे मौसमी परिवर्तन देखने को मिलते हैं हर साल यह परिवर्तन उत्तरी गोलार्ध northern hemisphere में वसव्त मौसम के आख़िर में अधिक हो जाते हैं और जब उत्तरी गोलार्ध में फसलें बीजने का समय होता है तो यह परिवर्तन कम हो जाते हैं क्यूंकि पौधे वातावरण में से कुछ co हटा लेते हैंi _END\n",
              "107772  indic2012                                                                                                                                                                                                                                                                                                            her father aghoranath chattopadhyay was a wellknown scholar and mother was a poet who wrote in bengali                                                                                                                                                                                                                                                                                                                  START_ इनके पिता अघोरनाथ चट्टोपाध्याय एक नामी विद्वान तथा माँ कवयित्री थीं और बांग्ला में लिखती थीं । _END\n",
              "120507  indic2012                                                                                                                                                                                                                                                                                                                                                          nishadraj guh served those three very well in ringverpur                                                                                                                                                                                                                                                                                                                                                            START_ ऋंगवेरपुर में निषादराज गुह ने तीनों की बहुत सेवा की। _END\n",
              "28755   indic2012                                                                                                                                                                                                                                                                                                                           the ordinance that was rejected by loksabha can be passed again before six weeks period                                                                                                                                                                                                                                                                                                            START_ लोकसभा एक अध्यादेश को अस्वीकृत करने वाला प्रस्ताव सप्ताह की अवधि समाप्त होने से पूर्व पास कर सकती है _END\n",
              "86303   indic2012                                                                                                                                                                                                                                                                                                                                                                                                         number of                                                                                                                                                                                                                                                                                                                                                                                                 START_ अठारह की संख्या _END"
            ]
          },
          "execution_count": 44,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "lines.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {
        "id": "9YK-gzKV9O7W"
      },
      "outputs": [],
      "source": [
        "### Get English and Hindi Vocabulary\n",
        "all_eng_words=set()\n",
        "for eng in lines['english_sentence']:\n",
        "    for word in eng.split():\n",
        "        if word not in all_eng_words:\n",
        "            all_eng_words.add(word)\n",
        "\n",
        "all_hindi_words=set()\n",
        "for hin in lines['hindi_sentence']:\n",
        "    for word in hin.split():\n",
        "        if word not in all_hindi_words:\n",
        "            all_hindi_words.add(word)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 57,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yjUeA5dS9dEt",
        "outputId": "da1a3a10-98b7-4766-ba39-3f5277314933"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "30447"
            ]
          },
          "execution_count": 57,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(all_eng_words)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 58,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CbuATncQ9hNo",
        "outputId": "048c7cbf-168e-47e2-8f8f-d2a38dae585b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "20506"
            ]
          },
          "execution_count": 58,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(all_hindi_words)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {
        "id": "0FPzhzQB9l1a"
      },
      "outputs": [],
      "source": [
        "#Calculating length of each sentence\n",
        "lines['length_eng_sentence']=lines['english_sentence'].apply(lambda x:len(x.split(\" \")))\n",
        "lines['length_hin_sentence']=lines['hindi_sentence'].apply(lambda x:len(x.split(\" \")))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "id": "Yg6nPKJ39n_H",
        "outputId": "54a5c43d-5747-4ba1-ff47-710e93c19e71"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>source</th>\n",
              "      <th>english_sentence</th>\n",
              "      <th>hindi_sentence</th>\n",
              "      <th>length_eng_sentence</th>\n",
              "      <th>length_hin_sentence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>43137</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>occurring in the environment recently increased carbon di oxide co monthly co measurements dershat that if you saw all year get to see the seasonal changes are small every year paryaverton northern hemisphere northern hemisphere seasons in the aswh at the end of the northern hemisphere crops have become more bjne is the time get at the change of environment because the plants remove some co charge</td>\n",
              "      <td>START_ वातावरण में कार्बन डाइऑक्साइड में हाल ही में होने वाली बढोतरी co मासिक co मापन यह दर्शाते हैं कि अगर सारे वर्ष को देखा जाए तो छोटेछोटे मौसमी परिवर्तन देखने को मिलते हैं हर साल यह परिवर्तन उत्तरी गोलार्ध northern hemisphere में वसव्त मौसम के आख़िर में अधिक हो जाते हैं और जब उत्तरी गोलार्ध में फसलें बीजने का समय होता है तो यह परिवर्तन कम हो जाते हैं क्यूंकि पौधे वातावरण में से कुछ co हटा लेते हैंi _END</td>\n",
              "      <td>67</td>\n",
              "      <td>81</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>107772</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>her father aghoranath chattopadhyay was a wellknown scholar and mother was a poet who wrote in bengali</td>\n",
              "      <td>START_ इनके पिता अघोरनाथ चट्टोपाध्याय एक नामी विद्वान तथा माँ कवयित्री थीं और बांग्ला में लिखती थीं । _END</td>\n",
              "      <td>17</td>\n",
              "      <td>19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>120507</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>nishadraj guh served those three very well in ringverpur</td>\n",
              "      <td>START_ ऋंगवेरपुर में निषादराज गुह ने तीनों की बहुत सेवा की। _END</td>\n",
              "      <td>9</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28755</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>the ordinance that was rejected by loksabha can be passed again before six weeks period</td>\n",
              "      <td>START_ लोकसभा एक अध्यादेश को अस्वीकृत करने वाला प्रस्ताव सप्ताह की अवधि समाप्त होने से पूर्व पास कर सकती है _END</td>\n",
              "      <td>15</td>\n",
              "      <td>21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86303</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>number of</td>\n",
              "      <td>START_ अठारह की संख्या _END</td>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           source                                                                                                                                                                                                                                                                                                                                                                                                  english_sentence                                                                                                                                                                                                                                                                                                                                                                                                              hindi_sentence  length_eng_sentence  length_hin_sentence\n",
              "43137   indic2012  occurring in the environment recently increased carbon di oxide co monthly co measurements dershat that if you saw all year get to see the seasonal changes are small every year paryaverton northern hemisphere northern hemisphere seasons in the aswh at the end of the northern hemisphere crops have become more bjne is the time get at the change of environment because the plants remove some co charge  START_ वातावरण में कार्बन डाइऑक्साइड में हाल ही में होने वाली बढोतरी co मासिक co मापन यह दर्शाते हैं कि अगर सारे वर्ष को देखा जाए तो छोटेछोटे मौसमी परिवर्तन देखने को मिलते हैं हर साल यह परिवर्तन उत्तरी गोलार्ध northern hemisphere में वसव्त मौसम के आख़िर में अधिक हो जाते हैं और जब उत्तरी गोलार्ध में फसलें बीजने का समय होता है तो यह परिवर्तन कम हो जाते हैं क्यूंकि पौधे वातावरण में से कुछ co हटा लेते हैंi _END                   67                   81\n",
              "107772  indic2012                                                                                                                                                                                                                                                                                                            her father aghoranath chattopadhyay was a wellknown scholar and mother was a poet who wrote in bengali                                                                                                                                                                                                                                                                                                                  START_ इनके पिता अघोरनाथ चट्टोपाध्याय एक नामी विद्वान तथा माँ कवयित्री थीं और बांग्ला में लिखती थीं । _END                   17                   19\n",
              "120507  indic2012                                                                                                                                                                                                                                                                                                                                                          nishadraj guh served those three very well in ringverpur                                                                                                                                                                                                                                                                                                                                                            START_ ऋंगवेरपुर में निषादराज गुह ने तीनों की बहुत सेवा की। _END                    9                   12\n",
              "28755   indic2012                                                                                                                                                                                                                                                                                                                           the ordinance that was rejected by loksabha can be passed again before six weeks period                                                                                                                                                                                                                                                                                                            START_ लोकसभा एक अध्यादेश को अस्वीकृत करने वाला प्रस्ताव सप्ताह की अवधि समाप्त होने से पूर्व पास कर सकती है _END                   15                   21\n",
              "86303   indic2012                                                                                                                                                                                                                                                                                                                                                                                                         number of                                                                                                                                                                                                                                                                                                                                                                                                 START_ अठारह की संख्या _END                    2                    5"
            ]
          },
          "execution_count": 49,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "lines.head()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XQYdGjUv99SN",
        "outputId": "307f204b-d15a-4550-f5cd-03b4ed892ddf"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(2030, 5)"
            ]
          },
          "execution_count": 50,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Lines having length greater than 30.\n",
        "lines[lines['length_eng_sentence']>30].shape\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 59,
      "metadata": {
        "id": "XtynQX1P-HPy"
      },
      "outputs": [],
      "source": [
        "lines=lines[lines['length_eng_sentence']<=20]\n",
        "lines=lines[lines['length_hin_sentence']<=20]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QEluWZCw-Lwg",
        "outputId": "6024f72e-5342-4deb-fd45-1a5d25709dd8"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(16970, 5)"
            ]
          },
          "execution_count": 52,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "lines.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 53,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1wRgRnLH-SLv",
        "outputId": "fc9f5dc1-675e-481c-f221-48510432e55c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "maximum length of Hindi Sentence  20\n",
            "maximum length of English Sentence  20\n"
          ]
        }
      ],
      "source": [
        "print(\"maximum length of Hindi Sentence \",max(lines['length_hin_sentence']))\n",
        "print(\"maximum length of English Sentence \",max(lines['length_eng_sentence']))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 62,
      "metadata": {
        "id": "pF8tUQOq-YGV"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "20\n",
            "20\n"
          ]
        }
      ],
      "source": [
        "max_length_src=max(lines['length_hin_sentence'])\n",
        "max_length_tar=max(lines['length_eng_sentence']) #Target sentence having length 20\n",
        "print(max_length_src)\n",
        "print(max_length_tar)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 64,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vChnd2lN-gmv",
        "outputId": "091c6fd2-8e07-4f5b-ac81-0da63f57037c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(30447, 20506)"
            ]
          },
          "execution_count": 64,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "input_words = sorted(list(all_eng_words))\n",
        "#print(input_words)\n",
        "target_words = sorted(list(all_hindi_words))\n",
        "#print(target_words)\n",
        "num_encoder_tokens = len(all_eng_words)\n",
        "num_decoder_tokens = len(all_hindi_words)\n",
        "num_encoder_tokens, num_decoder_tokens #Prints the total number of words\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 73,
      "metadata": {
        "id": "5KYcueYD-qDn"
      },
      "outputs": [],
      "source": [
        "#For zero padding\n",
        "num_decoder_tokens = num_decoder_tokens + 1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 107,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "30447\n",
            "20506\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>source</th>\n",
              "      <th>english_sentence</th>\n",
              "      <th>hindi_sentence</th>\n",
              "      <th>length_eng_sentence</th>\n",
              "      <th>length_hin_sentence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>90504</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>meri kavitai ki aadhi sadi half century of my being poet</td>\n",
              "      <td>START_ मेरी कविताई की आधी सदी _END</td>\n",
              "      <td>11</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50833</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>ames research center</td>\n",
              "      <td>START_ एम्स अनुसंधान केंद्र _END</td>\n",
              "      <td>3</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>77153</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>then drona became the commanderinchief of the army</td>\n",
              "      <td>START_ तब आचार्य द्रोण ने सेनापतित्व का भार ग्रहण किया। _END</td>\n",
              "      <td>8</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>73502</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>then it was divided into north india secret and persia of sasaani empire</td>\n",
              "      <td>START_ इसके बाद यह उत्तर भारत के गुप्त और फारस के सासानी साम्राज्य के बीच बँटा रहा । _END</td>\n",
              "      <td>13</td>\n",
              "      <td>19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59955</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>terrorism reasons of terrorisma special report</td>\n",
              "      <td>START_ आतंकवाद आतंकवाद विरोधी कारणों और आतंकवाद का उपयोग करता है पर एक विश्लेषण _END</td>\n",
              "      <td>6</td>\n",
              "      <td>15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56563</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>sawai jaisingh ii built sisodia rani ka bagsisodia queen garden for his sisodia queen</td>\n",
              "      <td>START_ सवाई जय सिंह ii ने अपनी सिसोदिया रानी के लिए सिसोदिया रानी का बाग बनवाया। _END</td>\n",
              "      <td>14</td>\n",
              "      <td>17</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>95036</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>khayam ki madhushala khayams bar</td>\n",
              "      <td>START_ खय्याम की मधुशाला _END</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96771</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>part five chapter parliament</td>\n",
              "      <td>START_ भाग पाँच अध्याय संसद _END</td>\n",
              "      <td>4</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>72424</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>washington dc capital of columbia</td>\n",
              "      <td>START_ वाशिंगटन डीसी डिस्ट्रिक्ट अव कोलंबिया _END</td>\n",
              "      <td>5</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96705</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>pure ganga on goumukh ganga river is known by its capacity of purification in the whole world</td>\n",
              "      <td>START_ गोमुख पर शुद्ध गंगागंगा नदी विश्व भर में अपनी शुद्धीकरण क्षमता के कारण जानी जाती है। _END</td>\n",
              "      <td>17</td>\n",
              "      <td>18</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>16970 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "          source                                                                               english_sentence                                                                                    hindi_sentence  length_eng_sentence  length_hin_sentence\n",
              "90504  indic2012                                       meri kavitai ki aadhi sadi half century of my being poet                                                                START_ मेरी कविताई की आधी सदी _END                   11                    7\n",
              "50833  indic2012                                                                           ames research center                                                                  START_ एम्स अनुसंधान केंद्र _END                    3                    5\n",
              "77153  indic2012                                             then drona became the commanderinchief of the army                                      START_ तब आचार्य द्रोण ने सेनापतित्व का भार ग्रहण किया। _END                    8                   11\n",
              "73502  indic2012                       then it was divided into north india secret and persia of sasaani empire         START_ इसके बाद यह उत्तर भारत के गुप्त और फारस के सासानी साम्राज्य के बीच बँटा रहा । _END                   13                   19\n",
              "59955  indic2012                                                 terrorism reasons of terrorisma special report              START_ आतंकवाद आतंकवाद विरोधी कारणों और आतंकवाद का उपयोग करता है पर एक विश्लेषण _END                    6                   15\n",
              "...          ...                                                                                            ...                                                                                               ...                  ...                  ...\n",
              "56563  indic2012          sawai jaisingh ii built sisodia rani ka bagsisodia queen garden for his sisodia queen             START_ सवाई जय सिंह ii ने अपनी सिसोदिया रानी के लिए सिसोदिया रानी का बाग बनवाया। _END                   14                   17\n",
              "95036  indic2012                                                               khayam ki madhushala khayams bar                                                                     START_ खय्याम की मधुशाला _END                    5                    5\n",
              "96771  indic2012                                                                   part five chapter parliament                                                                  START_ भाग पाँच अध्याय संसद _END                    4                    6\n",
              "72424  indic2012                                                              washington dc capital of columbia                                                 START_ वाशिंगटन डीसी डिस्ट्रिक्ट अव कोलंबिया _END                    5                    7\n",
              "96705  indic2012  pure ganga on goumukh ganga river is known by its capacity of purification in the whole world  START_ गोमुख पर शुद्ध गंगागंगा नदी विश्व भर में अपनी शुद्धीकरण क्षमता के कारण जानी जाती है। _END                   17                   18\n",
              "\n",
              "[16970 rows x 5 columns]"
            ]
          },
          "execution_count": 107,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Converting the list of words into dictionary using enumerate( ) function for automatic indexing.\n",
        "\n",
        "input_token_index = dict([(word, i+1) for i, word in enumerate(input_words)])\n",
        "target_token_index = dict([(word, i+1) for i, word in enumerate(target_words)])\n",
        "#print(input_token_index)\n",
        "#print(target_token_index)\n",
        "print(len(input_token_index.keys()))\n",
        "print(len(target_token_index))\n",
        "lines"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 99,
      "metadata": {},
      "outputs": [],
      "source": [
        "#Converting the same dataset in reverse order to keep index as numeric and value as the word.\n",
        "\n",
        "reverse_input_char_index = dict((i, word) for word, i in input_token_index.items())\n",
        "#print(reverse_input_char_index)\n",
        "reverse_target_char_index = dict((i, word) for word, i in target_token_index.items())\n",
        "#print(reverse_target_char_index)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 93,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>source</th>\n",
              "      <th>english_sentence</th>\n",
              "      <th>hindi_sentence</th>\n",
              "      <th>length_eng_sentence</th>\n",
              "      <th>length_hin_sentence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>90504</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>meri kavitai ki aadhi sadi half century of my being poet</td>\n",
              "      <td>START_ मेरी कविताई की आधी सदी _END</td>\n",
              "      <td>11</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50833</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>ames research center</td>\n",
              "      <td>START_ एम्स अनुसंधान केंद्र _END</td>\n",
              "      <td>3</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>77153</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>then drona became the commanderinchief of the army</td>\n",
              "      <td>START_ तब आचार्य द्रोण ने सेनापतित्व का भार ग्रहण किया। _END</td>\n",
              "      <td>8</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>73502</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>then it was divided into north india secret and persia of sasaani empire</td>\n",
              "      <td>START_ इसके बाद यह उत्तर भारत के गुप्त और फारस के सासानी साम्राज्य के बीच बँटा रहा । _END</td>\n",
              "      <td>13</td>\n",
              "      <td>19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59955</th>\n",
              "      <td>indic2012</td>\n",
              "      <td>terrorism reasons of terrorisma special report</td>\n",
              "      <td>START_ आतंकवाद आतंकवाद विरोधी कारणों और आतंकवाद का उपयोग करता है पर एक विश्लेषण _END</td>\n",
              "      <td>6</td>\n",
              "      <td>15</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          source                                                          english_sentence                                                                             hindi_sentence  length_eng_sentence  length_hin_sentence\n",
              "90504  indic2012                  meri kavitai ki aadhi sadi half century of my being poet                                                         START_ मेरी कविताई की आधी सदी _END                   11                    7\n",
              "50833  indic2012                                                      ames research center                                                           START_ एम्स अनुसंधान केंद्र _END                    3                    5\n",
              "77153  indic2012                        then drona became the commanderinchief of the army                               START_ तब आचार्य द्रोण ने सेनापतित्व का भार ग्रहण किया। _END                    8                   11\n",
              "73502  indic2012  then it was divided into north india secret and persia of sasaani empire  START_ इसके बाद यह उत्तर भारत के गुप्त और फारस के सासानी साम्राज्य के बीच बँटा रहा । _END                   13                   19\n",
              "59955  indic2012                            terrorism reasons of terrorisma special report       START_ आतंकवाद आतंकवाद विरोधी कारणों और आतंकवाद का उपयोग करता है पर एक विश्लेषण _END                    6                   15"
            ]
          },
          "execution_count": 93,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "lines = shuffle(lines)\n",
        "lines.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 94,
      "metadata": {},
      "outputs": [],
      "source": [
        "#SPLITIING THE DATASET IN TRAINING AND TESTING SETS."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 104,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "((11369,), (5601,))"
            ]
          },
          "execution_count": 104,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "X, y = lines['english_sentence'], lines['hindi_sentence']\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.33,random_state=42) #Using keras to split arrays into random train and test subsets.\n",
        "X_train.shape, X_test.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 106,
      "metadata": {},
      "outputs": [],
      "source": [
        "X_train.to_pickle('X_train.pkl')\n",
        "X_test.to_pickle('X_test.pkl')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 109,
      "metadata": {},
      "outputs": [],
      "source": [
        "def generate_batch(X = X_train, y = y_train, batch_size = 128):\n",
        "    ''' Generate a batch of data '''\n",
        "    while True:\n",
        "        for j in range(0, len(X), batch_size):\n",
        "            encoder_input_data = np.zeros((batch_size, max_length_src),dtype='float32')\n",
        "            decoder_input_data = np.zeros((batch_size, max_length_tar),dtype='float32')\n",
        "            decoder_target_data = np.zeros((batch_size, max_length_tar, num_decoder_tokens),dtype='float32')\n",
        "            for i, (input_text, target_text) in enumerate(zip(X[j:j+batch_size], y[j:j+batch_size])):\n",
        "                for t, word in enumerate(input_text.split()):\n",
        "                    encoder_input_data[i, t] = input_token_index[word] # encoder input seq\n",
        "                for t, word in enumerate(target_text.split()):\n",
        "                    if t<len(target_text.split())-1:\n",
        "                        decoder_input_data[i, t] = target_token_index[word] # decoder input seq\n",
        "                    if t>0:\n",
        "                        # decoder target sequence (one hot encoded)\n",
        "                        # does not include the START_ token\n",
        "                        # Offset by one timestep\n",
        "                        decoder_target_data[i, t - 1, target_token_index[word]] = 1.\n",
        "            yield([encoder_input_data, decoder_input_data], decoder_target_data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 110,
      "metadata": {},
      "outputs": [],
      "source": [
        "latent_dim=300"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 111,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Encoder\n",
        "encoder_inputs = Input(shape=(None,))\n",
        "enc_emb =  Embedding(num_encoder_tokens, latent_dim, mask_zero = True)(encoder_inputs)\n",
        "encoder_lstm = LSTM(latent_dim, return_state=True)\n",
        "encoder_outputs, state_h, state_c = encoder_lstm(enc_emb)\n",
        "# We discard `encoder_outputs` and only keep the states.\n",
        "encoder_states = [state_h, state_c]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 113,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Set up the decoder, using `encoder_states` as initial state.\n",
        "decoder_inputs = Input(shape=(None,))\n",
        "dec_emb_layer = Embedding(num_decoder_tokens, latent_dim, mask_zero = True)\n",
        "dec_emb = dec_emb_layer(decoder_inputs)\n",
        "# We set up our decoder to return full output sequences,\n",
        "# and to return internal states as well. We don't use the\n",
        "# return states in the training model, but we will use them in inference.\n",
        "decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True)\n",
        "decoder_outputs, _, _ = decoder_lstm(dec_emb,\n",
        "                                     initial_state=encoder_states)\n",
        "decoder_dense = Dense(num_decoder_tokens, activation='softmax')\n",
        "decoder_outputs = decoder_dense(decoder_outputs)\n",
        "\n",
        "# Define the model that will turn\n",
        "# `encoder_input_data` & `decoder_input_data` into `decoder_target_data`\n",
        "model = Model([encoder_inputs, decoder_inputs], decoder_outputs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 114,
      "metadata": {},
      "outputs": [],
      "source": [
        "model.compile(optimizer='rmsprop', loss='categorical_crossentropy')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 115,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"model_1\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_1 (InputLayer)           [(None, None)]       0           []                               \n",
            "                                                                                                  \n",
            " input_3 (InputLayer)           [(None, None)]       0           []                               \n",
            "                                                                                                  \n",
            " embedding (Embedding)          (None, None, 300)    9134100     ['input_1[0][0]']                \n",
            "                                                                                                  \n",
            " embedding_2 (Embedding)        (None, None, 300)    6152100     ['input_3[0][0]']                \n",
            "                                                                                                  \n",
            " lstm (LSTM)                    [(None, 300),        721200      ['embedding[0][0]']              \n",
            "                                 (None, 300),                                                     \n",
            "                                 (None, 300)]                                                     \n",
            "                                                                                                  \n",
            " lstm_2 (LSTM)                  [(None, None, 300),  721200      ['embedding_2[0][0]',            \n",
            "                                 (None, 300),                     'lstm[0][1]',                   \n",
            "                                 (None, 300)]                     'lstm[0][2]']                   \n",
            "                                                                                                  \n",
            " dense_1 (Dense)                (None, None, 20507)  6172607     ['lstm_2[0][0]']                 \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 22,901,207\n",
            "Trainable params: 22,901,207\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 116,
      "metadata": {},
      "outputs": [],
      "source": [
        "\n",
        "train_samples = len(X_train)\n",
        "val_samples = len(X_test)\n",
        "batch_size = 128\n",
        "epochs = 100"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 117,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\kpriy\\AppData\\Local\\Temp/ipykernel_3480/2108290721.py:1: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.\n",
            "  model.fit_generator(generator = generate_batch(X_train, y_train, batch_size = batch_size),\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/100\n",
            "88/88 [==============================] - 138s 1s/step - loss: 3.5518 - val_loss: 3.3327\n",
            "Epoch 2/100\n",
            "88/88 [==============================] - 115s 1s/step - loss: 3.1991 - val_loss: 3.2315\n",
            "Epoch 3/100\n",
            "88/88 [==============================] - 162s 2s/step - loss: 3.0697 - val_loss: 3.1399\n",
            "Epoch 4/100\n",
            "88/88 [==============================] - 521s 6s/step - loss: 2.9453 - val_loss: 3.0531\n",
            "Epoch 5/100\n",
            "88/88 [==============================] - 161s 2s/step - loss: 2.8220 - val_loss: 2.9670\n",
            "Epoch 6/100\n",
            "88/88 [==============================] - 1555s 18s/step - loss: 2.6897 - val_loss: 2.8755\n",
            "Epoch 7/100\n",
            "88/88 [==============================] - 105s 1s/step - loss: 2.5567 - val_loss: 2.7913\n",
            "Epoch 8/100\n",
            "88/88 [==============================] - 104s 1s/step - loss: 2.4214 - val_loss: 2.7030\n",
            "Epoch 9/100\n",
            "88/88 [==============================] - 106s 1s/step - loss: 2.2954 - val_loss: 2.6228\n",
            "Epoch 10/100\n",
            "88/88 [==============================] - 107s 1s/step - loss: 2.1683 - val_loss: 2.5540\n",
            "Epoch 11/100\n",
            "88/88 [==============================] - 108s 1s/step - loss: 2.0512 - val_loss: 2.4790\n",
            "Epoch 12/100\n",
            "88/88 [==============================] - 110s 1s/step - loss: 1.9275 - val_loss: 2.4222\n",
            "Epoch 13/100\n",
            "88/88 [==============================] - 115s 1s/step - loss: 1.8088 - val_loss: 2.3532\n",
            "Epoch 14/100\n",
            "88/88 [==============================] - 120s 1s/step - loss: 1.6962 - val_loss: 2.2948\n",
            "Epoch 15/100\n",
            "88/88 [==============================] - 116s 1s/step - loss: 1.5874 - val_loss: 2.2445\n",
            "Epoch 16/100\n",
            "88/88 [==============================] - 123s 1s/step - loss: 1.4780 - val_loss: 2.1839\n",
            "Epoch 17/100\n",
            "88/88 [==============================] - 115s 1s/step - loss: 1.3737 - val_loss: 2.1448\n",
            "Epoch 18/100\n",
            "88/88 [==============================] - 117s 1s/step - loss: 1.2708 - val_loss: 2.0759\n",
            "Epoch 19/100\n",
            "88/88 [==============================] - 117s 1s/step - loss: 1.1736 - val_loss: 2.0304\n",
            "Epoch 20/100\n",
            "88/88 [==============================] - 118s 1s/step - loss: 1.0813 - val_loss: 1.9805\n",
            "Epoch 21/100\n",
            "88/88 [==============================] - 119s 1s/step - loss: 0.9928 - val_loss: 1.9438\n",
            "Epoch 22/100\n",
            "88/88 [==============================] - 121s 1s/step - loss: 0.9116 - val_loss: 1.8889\n",
            "Epoch 23/100\n",
            "88/88 [==============================] - 115s 1s/step - loss: 0.8318 - val_loss: 1.8579\n",
            "Epoch 24/100\n",
            "88/88 [==============================] - 119s 1s/step - loss: 0.7574 - val_loss: 1.8187\n",
            "Epoch 25/100\n",
            "88/88 [==============================] - 127s 1s/step - loss: 0.6894 - val_loss: 1.7786\n",
            "Epoch 26/100\n",
            "88/88 [==============================] - 128s 1s/step - loss: 0.6264 - val_loss: 1.7546\n",
            "Epoch 27/100\n",
            "88/88 [==============================] - 127s 1s/step - loss: 0.5658 - val_loss: 1.7170\n",
            "Epoch 28/100\n",
            "88/88 [==============================] - 135s 2s/step - loss: 0.5075 - val_loss: 1.6973\n",
            "Epoch 29/100\n",
            "88/88 [==============================] - ETA: 0s - loss: 0.4561"
          ]
        }
      ],
      "source": [
        "model.fit_generator(generator = generate_batch(X_train, y_train, batch_size = batch_size),\n",
        "                    steps_per_epoch = train_samples//batch_size,\n",
        "                    epochs=epochs,\n",
        "                    validation_data = generate_batch(X_test, y_test, batch_size = batch_size),\n",
        "                    validation_steps = val_samples//batch_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "model.save_weights('nmt_weights.h5')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Encode the input sequence to get the \"thought vectors\"\n",
        "encoder_model = Model(encoder_inputs, encoder_states)\n",
        "\n",
        "# Decoder setup\n",
        "# Below tensors will hold the states of the previous time step\n",
        "decoder_state_input_h = Input(shape=(latent_dim,))\n",
        "decoder_state_input_c = Input(shape=(latent_dim,))\n",
        "decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
        "\n",
        "dec_emb2= dec_emb_layer(decoder_inputs) # Get the embeddings of the decoder sequence\n",
        "\n",
        "# To predict the next word in the sequence, set the initial states to the states from the previous time step\n",
        "decoder_outputs2, state_h2, state_c2 = decoder_lstm(dec_emb2, initial_state=decoder_states_inputs)\n",
        "decoder_states2 = [state_h2, state_c2]\n",
        "decoder_outputs2 = decoder_dense(decoder_outputs2) # A dense softmax layer to generate prob dist. over the target vocabulary\n",
        "\n",
        "# Final decoder model\n",
        "decoder_model = Model(\n",
        "    [decoder_inputs] + decoder_states_inputs,\n",
        "    [decoder_outputs2] + decoder_states2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "def decode_sequence(input_seq):\n",
        "    # Encode the input as state vectors.\n",
        "    states_value = encoder_model.predict(input_seq)\n",
        "    # Generate empty target sequence of length 1.\n",
        "    target_seq = np.zeros((1,1))\n",
        "    # Populate the first character of target sequence with the start character.\n",
        "    target_seq[0, 0] = target_token_index['START_']\n",
        "\n",
        "    # Sampling loop for a batch of sequences\n",
        "    # (to simplify, here we assume a batch of size 1).\n",
        "    stop_condition = False\n",
        "    decoded_sentence = ''\n",
        "    while not stop_condition:\n",
        "        output_tokens, h, c = decoder_model.predict([target_seq] + states_value)\n",
        "\n",
        "        # Sample a token\n",
        "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
        "        sampled_char = reverse_target_char_index[sampled_token_index]\n",
        "        decoded_sentence += ' '+sampled_char\n",
        "\n",
        "        # Exit condition: either hit max length\n",
        "        # or find stop character.\n",
        "        if (sampled_char == '_END' or\n",
        "           len(decoded_sentence) > 50):\n",
        "            stop_condition = True\n",
        "\n",
        "        # Update the target sequence (of length 1).\n",
        "        target_seq = np.zeros((1,1))\n",
        "        target_seq[0, 0] = sampled_token_index\n",
        "\n",
        "        # Update states\n",
        "        states_value = [h, c]\n",
        "\n",
        "    return decoded_sentence"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "train_gen = generate_batch(X_train, y_train, batch_size = 1)\n",
        "k=-1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "k+=1\n",
        "(input_seq, actual_output), _ = next(train_gen)\n",
        "decoded_sentence = decode_sequence(input_seq)\n",
        "print('Input English sentence:', X_train[k:k+1].values[0])\n",
        "print('Actual Hindi Translation:', y_train[k:k+1].values[0][6:-4])\n",
        "print('Predicted Hindi Translation:', decoded_sentence[:-4])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "k+=1\n",
        "(input_seq, actual_output), _ = next(train_gen)\n",
        "decoded_sentence = decode_sequence(input_seq)\n",
        "print('Input English sentence:', X_train[k:k+1].values[0])\n",
        "print('Actual Hindi Translation:', y_train[k:k+1].values[0][6:-4])\n",
        "print('Predicted Hindi Translation:', decoded_sentence[:-4])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "k+=1\n",
        "(input_seq, actual_output), _ = next(train_gen)\n",
        "decoded_sentence = decode_sequence(input_seq)\n",
        "print('Input English sentence:', X_train[k:k+1].values[0])\n",
        "print('Actual Hindi Translation:', y_train[k:k+1].values[0][6:-4])\n",
        "print('Predicted Hindi Translation:', decoded_sentence[:-4])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "k+=1\n",
        "(input_seq, actual_output), _ = next(train_gen)\n",
        "decoded_sentence = decode_sequence(input_seq)\n",
        "print('Input English sentence:', X_train[k:k+1].values[0])\n",
        "print('Actual Hindi Translation:', y_train[k:k+1].values[0][6:-4])\n",
        "print('Predicted Hindi Translation:', decoded_sentence[:-4])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "k+=1\n",
        "(input_seq, actual_output), _ = next(train_gen)\n",
        "decoded_sentence = decode_sequence(input_seq)\n",
        "print('Input English sentence:', X_train[k:k+1].values[0])\n",
        "print('Actual Hindi Translation:', y_train[k:k+1].values[0][6:-4])\n",
        "print('Predicted Hindi Translation:', decoded_sentence[:-4])"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "Machine Translation Model.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.7"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}