English_to_hindi_Translation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "English_to_hindi_Translation.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m9JJ7FBw84tG",
        "colab_type": "text"
      },
      "source": [
        "# Importing dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZbcvtPlp3YWu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import math\n",
        "import re\n",
        "import time"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P6o_cpZz3y_-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras import layers\n",
        "import tensorflow_datasets as tfds"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BQN8jwx48_yU",
        "colab_type": "text"
      },
      "source": [
        "# Data preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q8Or0sLV5b8t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# with open('data/parallel/IITB.en-hi.en',mode='r',encoding='utf-8') as f:\n",
        "#   eng = f.read()\n",
        "# with open('data/parallel/IITB.en-hi.hi',mode='r',encoding='utf-8') as f:\n",
        "#   hindi = f.read()\n",
        "with open('data/pruned_train.en',mode='r',encoding='utf-8') as f:\n",
        "  eng = f.read()\n",
        "with open('data/pruned_train.hi',mode='r',encoding='utf-8') as f:\n",
        "  hindi = f.read()\n",
        "with open(\"data/nonbreaking_prefix.en\",\n",
        "          mode='r',\n",
        "          encoding='utf-8') as f:\n",
        "    non_breaking_prefix_en = f.read()\n",
        "with open(\"data/nonbreaking_prefix.hi\",\n",
        "          mode='r',\n",
        "          encoding='utf-8') as f:\n",
        "    non_breaking_prefix_hi = f.read()"
      ],
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L_TeuktU40Cb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "non_breaking_prefix_en = non_breaking_prefix_en.split(\"\\n\")\n",
        "non_breaking_prefix_en = [' ' + pref.lower() + '.' for pref in non_breaking_prefix_en]\n",
        "non_breaking_prefix_hi = non_breaking_prefix_hi.split(\"\\n\")\n",
        "non_breaking_prefix_hi = [' ' + pref + '.' for pref in non_breaking_prefix_hi]"
      ],
      "execution_count": 69,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qg-8LLK-WdFp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "corpus_en = eng\n",
        "# Add $$$ after non ending sentence points\n",
        "for prefix in non_breaking_prefix_en:\n",
        "    corpus_en = corpus_en.replace(prefix, prefix + '$$$')\n",
        "corpus_en = re.sub(r\"\\.(?=[0-9]|[a-z]|[A-Z])\", \".$$$\", corpus_en)\n",
        "# Remove $$$ markers\n",
        "corpus_en = re.sub(r\".\\$\\$\\$\", '', corpus_en)\n",
        "# Clear multiple spaces\n",
        "corpus_en = re.sub(r\"  +\", \" \", corpus_en)\n",
        "corpus_en = corpus_en.split('\\n')\n",
        "corpus_en = [x.lower() for x in corpus_en]\n",
        "    \n",
        "corpus_hi = hindi\n",
        "for prefix in non_breaking_prefix_hi:\n",
        "    corpus_hi = corpus_hi.replace(prefix, prefix + '$$$')\n",
        "corpus_hi = re.sub(r\"\\.(?=[0-9]|[a-z]|[A-Z])\", \".$$$\", corpus_hi)\n",
        "corpus_hi = re.sub(r\".\\$\\$\\$\", '', corpus_hi)\n",
        "corpus_hi = re.sub(r\"  +\", \" \", corpus_hi)\n",
        "corpus_hi = corpus_hi.split('\\n')"
      ],
      "execution_count": 70,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s-Y9v8-Tozl2",
        "colab_type": "text"
      },
      "source": [
        "## Tokenizing text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pGKbzv7eVrw-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vocab_en_path = './vocab/tokenizer_en' \n",
        "vocab_hi_path = './vocab/tokenizer_hi'"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p5YXanmOd_xK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer_en = tfds.features.text.SubwordTextEncoder.build_from_corpus(\n",
        "    corpus_en, target_vocab_size=2**13)\n",
        "tokenizer_hi = tfds.features.text.SubwordTextEncoder.build_from_corpus(\n",
        "    corpus_hi, target_vocab_size=2**13)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kKx3TaRvXk8D",
        "colab_type": "text"
      },
      "source": [
        "## Saving locally"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "95Vi5iIAVaKl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer_en.save_to_file(vocab_en_path)\n",
        "tokenizer_hi.save_to_file(vocab_hi_path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RE_VHGpFVmB4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "MAX_LENGTH = 50\n",
        "tokenizer_en = tfds.features.text.SubwordTextEncoder.load_from_file(vocab_en_path)\n",
        "tokenizer_hi = tfds.features.text.SubwordTextEncoder.load_from_file(vocab_hi_path)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ftIbPzIwCtwL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "VOCAB_SIZE_EN = tokenizer_en.vocab_size + 2 \n",
        "VOCAB_SIZE_HI = tokenizer_hi.vocab_size + 2 "
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oPFe2YJDC9jw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "inputs = [[VOCAB_SIZE_EN-2] + tokenizer_en.encode(sentence) + [VOCAB_SIZE_EN-1]\n",
        "          for sentence in corpus_en]\n",
        "outputs = [[VOCAB_SIZE_HI-2] + tokenizer_hi.encode(sentence) + [VOCAB_SIZE_HI-1]\n",
        "           for sentence in corpus_hi]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bG6AlcFMpC5C",
        "colab_type": "text"
      },
      "source": [
        "## Remove too long sentences"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F6CD6PLGyQWy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "MAX_LENGTH = 50\n",
        "idx_to_remove = [count for count, sent in enumerate(inputs)\n",
        "                 if len(sent) > MAX_LENGTH]\n",
        "for idx in reversed(idx_to_remove):\n",
        "    del inputs[idx]\n",
        "    del outputs[idx]\n",
        "idx_to_remove = [count for count, sent in enumerate(outputs)\n",
        "                 if len(sent) > MAX_LENGTH]\n",
        "for idx in reversed(idx_to_remove):\n",
        "    del inputs[idx]\n",
        "    del outputs[idx]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ypm8h5aZQTZ1",
        "colab_type": "text"
      },
      "source": [
        "## Final data(Padding to ensure that all sequences in a list have the same length)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nvDfLDWUONlE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "inputs = tf.keras.preprocessing.sequence.pad_sequences(inputs,\n",
        "                                                       value=0,\n",
        "                                                       padding='post',\n",
        "                                                       maxlen=MAX_LENGTH)\n",
        "outputs = tf.keras.preprocessing.sequence.pad_sequences(outputs,\n",
        "                                                        value=0,\n",
        "                                                        padding='post',\n",
        "                                                        maxlen=MAX_LENGTH)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "juf4TWiUYZrS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "np.save('final_data/inputs', a) \n",
        "np.save('final_data/outputs', a) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jbIOdEOyJaKD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "inputs = np.load('final_data/inputs.npy') \n",
        "outputs = np.load('final_data/outputs.npy') "
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wFxMp3TOIYff",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 64\n",
        "BUFFER_SIZE = 20000\n",
        "\n",
        "dataset = tf.data.Dataset.from_tensor_slices((inputs, outputs))\n",
        "\n",
        "dataset = dataset.cache()\n",
        "dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n",
        "dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ycT0YqydRcUd",
        "colab_type": "text"
      },
      "source": [
        "# Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-SBoH8G4XyR9",
        "colab_type": "text"
      },
      "source": [
        "## Embedding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e2wc6sYlX0dr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PositionalEncoding(layers.Layer):\n",
        "\n",
        "    def __init__(self):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "    \n",
        "    def get_angles(self, pos, i, d_model):\n",
        "        angles = 1 / np.power(10000., (2*(i//2)) / np.float32(d_model))\n",
        "        return pos * angles\n",
        "\n",
        "    def call(self, inputs):\n",
        "        seq_length = inputs.shape.as_list()[-2]\n",
        "        d_model = inputs.shape.as_list()[-1]\n",
        "        angles = self.get_angles(np.arange(seq_length)[:, np.newaxis],\n",
        "                                 np.arange(d_model)[np.newaxis, :],\n",
        "                                 d_model)\n",
        "        angles[:, 0::2] = np.sin(angles[:, 0::2])\n",
        "        angles[:, 1::2] = np.cos(angles[:, 1::2])\n",
        "        pos_encoding = angles[np.newaxis, ...]\n",
        "        return inputs + tf.cast(pos_encoding, tf.float32)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lcw8YIQqRhOJ",
        "colab_type": "text"
      },
      "source": [
        "## Attention"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2rEoCNJURbrT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def scaled_dot_product_attention(queries, keys, values, mask):\n",
        "    product = tf.matmul(queries, keys, transpose_b=True)\n",
        "    \n",
        "    keys_dim = tf.cast(tf.shape(keys)[-1], tf.float32)\n",
        "    scaled_product = product / tf.math.sqrt(keys_dim)\n",
        "    \n",
        "    if mask is not None:\n",
        "        scaled_product += (mask * -1e9)\n",
        "    \n",
        "    attention = tf.matmul(tf.nn.softmax(scaled_product, axis=-1), values)\n",
        "    \n",
        "    return attention"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-MjtvXrfYEx7",
        "colab_type": "text"
      },
      "source": [
        "### Multi-head attention sublayer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lvq4I9uTX5p7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MultiHeadAttention(layers.Layer):\n",
        "    \n",
        "    def __init__(self, num_heads):\n",
        "        super(MultiHeadAttention, self).__init__()\n",
        "        self.num_heads = num_heads\n",
        "        \n",
        "    def build(self, input_shape):\n",
        "        self.d_model = input_shape[-1]\n",
        "        assert self.d_model % self.num_heads == 0\n",
        "        \n",
        "        self.d_proj = self.d_model // self.num_heads\n",
        "        \n",
        "        self.query_lin = layers.Dense(units=self.d_model)\n",
        "        self.key_lin = layers.Dense(units=self.d_model)\n",
        "        self.value_lin = layers.Dense(units=self.d_model)\n",
        "        \n",
        "        self.final_lin = layers.Dense(units=self.d_model)\n",
        "        \n",
        "    def split_proj(self, inputs, batch_size): # inputs: (batch_size, seq_length, d_model)\n",
        "        shape = (batch_size,\n",
        "                 -1,\n",
        "                 self.num_heads,\n",
        "                 self.d_proj)\n",
        "        splited_inputs = tf.reshape(inputs, shape=shape) # (batch_size, seq_length, num_heads, d_proj)\n",
        "        return tf.transpose(splited_inputs, perm=[0, 2, 1, 3]) # (batch_size, num_heads, seq_length, d_proj)\n",
        "    \n",
        "    def call(self, queries, keys, values, mask):\n",
        "        batch_size = tf.shape(queries)[0]\n",
        "        \n",
        "        queries = self.query_lin(queries)\n",
        "        keys = self.key_lin(keys)\n",
        "        values = self.value_lin(values)\n",
        "        \n",
        "        queries = self.split_proj(queries, batch_size)\n",
        "        keys = self.split_proj(keys, batch_size)\n",
        "        values = self.split_proj(values, batch_size)\n",
        "        \n",
        "        attention = scaled_dot_product_attention(queries, keys, values, mask)\n",
        "        \n",
        "        attention = tf.transpose(attention, perm=[0, 2, 1, 3])\n",
        "        \n",
        "        concat_attention = tf.reshape(attention,\n",
        "                                      shape=(batch_size, -1, self.d_model))\n",
        "        \n",
        "        outputs = self.final_lin(concat_attention)\n",
        "        \n",
        "        return outputs"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yiyuHe1OeT5N",
        "colab_type": "text"
      },
      "source": [
        "## Encoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UV0ZMH7KT_KZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class EncoderLayer(layers.Layer):\n",
        "    \n",
        "    def __init__(self, FFN_units, num_heads, dropout_rate):\n",
        "        super(EncoderLayer, self).__init__()\n",
        "        self.FFN_units = FFN_units\n",
        "        self.num_heads = num_heads\n",
        "        self.dropout_rate = dropout_rate\n",
        "    \n",
        "    def build(self, input_shape):\n",
        "        self.d_model = input_shape[-1]\n",
        "        \n",
        "        self.multi_head_attention = MultiHeadAttention(self.num_heads)\n",
        "        self.dropout_1 = layers.Dropout(rate=self.dropout_rate)\n",
        "        self.norm_1 = layers.LayerNormalization(epsilon=1e-6)\n",
        "        \n",
        "        self.dense_1 = layers.Dense(units=self.FFN_units, activation=\"relu\")\n",
        "        self.dense_2 = layers.Dense(units=self.d_model)\n",
        "        self.dropout_2 = layers.Dropout(rate=self.dropout_rate)\n",
        "        self.norm_2 = layers.LayerNormalization(epsilon=1e-6)\n",
        "        \n",
        "    def call(self, inputs, mask, training):\n",
        "        attention = self.multi_head_attention(inputs,\n",
        "                                              inputs,\n",
        "                                              inputs,\n",
        "                                              mask)\n",
        "        attention = self.dropout_1(attention, training=training)\n",
        "        attention = self.norm_1(attention + inputs)\n",
        "        \n",
        "        outputs = self.dense_1(attention)\n",
        "        outputs = self.dense_2(outputs)\n",
        "        outputs = self.dropout_2(outputs, training=training)\n",
        "        outputs = self.norm_2(outputs + attention)\n",
        "        \n",
        "        return outputs"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P-P92KeZih60",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Encoder(layers.Layer):\n",
        "    \n",
        "    def __init__(self,\n",
        "                 num_layers,\n",
        "                 FFN_units,\n",
        "                 num_heads,\n",
        "                 dropout_rate,\n",
        "                 vocab_size,\n",
        "                 d_model,\n",
        "                 name=\"encoder\"):\n",
        "        super(Encoder, self).__init__(name=name)\n",
        "        self.num_layers = num_layers\n",
        "        self.d_model = d_model\n",
        "        \n",
        "        self.embedding = layers.Embedding(vocab_size, d_model)\n",
        "        self.pos_encoding = PositionalEncoding()\n",
        "        self.dropout = layers.Dropout(rate=dropout_rate)\n",
        "        self.enc_layers = [EncoderLayer(FFN_units,\n",
        "                                        num_heads,\n",
        "                                        dropout_rate) \n",
        "                           for _ in range(num_layers)]\n",
        "    \n",
        "    def call(self, inputs, mask, training):\n",
        "        outputs = self.embedding(inputs)\n",
        "        outputs *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
        "        outputs = self.pos_encoding(outputs)\n",
        "        outputs = self.dropout(outputs, training)\n",
        "        \n",
        "        for i in range(self.num_layers):\n",
        "            outputs = self.enc_layers[i](outputs, mask, training)\n",
        "\n",
        "        return outputs"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7DthraBEwuvl",
        "colab_type": "text"
      },
      "source": [
        "## Decoder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7ZWZyFBnwy8u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class DecoderLayer(layers.Layer):\n",
        "    \n",
        "    def __init__(self, FFN_units, num_heads, dropout_rate):\n",
        "        super(DecoderLayer, self).__init__()\n",
        "        self.FFN_units = FFN_units\n",
        "        self.num_heads = num_heads\n",
        "        self.dropout_rate = dropout_rate\n",
        "    \n",
        "    def build(self, input_shape):\n",
        "        self.d_model = input_shape[-1]\n",
        "        \n",
        "        # Self multi head attention\n",
        "        self.multi_head_attention_1 = MultiHeadAttention(self.num_heads)\n",
        "        self.dropout_1 = layers.Dropout(rate=self.dropout_rate)\n",
        "        self.norm_1 = layers.LayerNormalization(epsilon=1e-6)\n",
        "        \n",
        "        # Multi head attention combined with encoder output\n",
        "        self.multi_head_attention_2 = MultiHeadAttention(self.num_heads)\n",
        "        self.dropout_2 = layers.Dropout(rate=self.dropout_rate)\n",
        "        self.norm_2 = layers.LayerNormalization(epsilon=1e-6)\n",
        "        \n",
        "        # Feed foward\n",
        "        self.dense_1 = layers.Dense(units=self.FFN_units,\n",
        "                                    activation=\"relu\")\n",
        "        self.dense_2 = layers.Dense(units=self.d_model)\n",
        "        self.dropout_3 = layers.Dropout(rate=self.dropout_rate)\n",
        "        self.norm_3 = layers.LayerNormalization(epsilon=1e-6)\n",
        "        \n",
        "    def call(self, inputs, enc_outputs, mask_1, mask_2, training):\n",
        "        attention = self.multi_head_attention_1(inputs,\n",
        "                                                inputs,\n",
        "                                                inputs,\n",
        "                                                mask_1)\n",
        "        attention = self.dropout_1(attention, training)\n",
        "        attention = self.norm_1(attention + inputs)\n",
        "        \n",
        "        attention_2 = self.multi_head_attention_2(attention,\n",
        "                                                  enc_outputs,\n",
        "                                                  enc_outputs,\n",
        "                                                  mask_2)\n",
        "        attention_2 = self.dropout_2(attention_2, training)\n",
        "        attention_2 = self.norm_2(attention_2 + attention)\n",
        "        \n",
        "        outputs = self.dense_1(attention_2)\n",
        "        outputs = self.dense_2(outputs)\n",
        "        outputs = self.dropout_3(outputs, training)\n",
        "        outputs = self.norm_3(outputs + attention_2)\n",
        "        \n",
        "        return outputs"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kpzdiWHiwywF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Decoder(layers.Layer):\n",
        "    \n",
        "    def __init__(self,\n",
        "                 num_layers,\n",
        "                 FFN_units,\n",
        "                 num_heads,\n",
        "                 dropout_rate,\n",
        "                 vocab_size,\n",
        "                 d_model,\n",
        "                 name=\"decoder\"):\n",
        "        super(Decoder, self).__init__(name=name)\n",
        "        self.d_model = d_model\n",
        "        self.num_layers = num_layers\n",
        "        \n",
        "        self.embedding = layers.Embedding(vocab_size, d_model)\n",
        "        self.pos_encoding = PositionalEncoding()\n",
        "        self.dropout = layers.Dropout(rate=dropout_rate)\n",
        "        \n",
        "        self.dec_layers = [DecoderLayer(FFN_units,\n",
        "                                        num_heads,\n",
        "                                        dropout_rate) \n",
        "                           for i in range(num_layers)]\n",
        "    \n",
        "    def call(self, inputs, enc_outputs, mask_1, mask_2, training):\n",
        "        outputs = self.embedding(inputs)\n",
        "        outputs *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
        "        outputs = self.pos_encoding(outputs)\n",
        "        outputs = self.dropout(outputs, training)\n",
        "        \n",
        "        for i in range(self.num_layers):\n",
        "            outputs = self.dec_layers[i](outputs,\n",
        "                                         enc_outputs,\n",
        "                                         mask_1,\n",
        "                                         mask_2,\n",
        "                                         training)\n",
        "\n",
        "        return outputs"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x5sJYkjbz5DD",
        "colab_type": "text"
      },
      "source": [
        "## Transformer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GqvqNjJPwyh-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Transformer(tf.keras.Model):\n",
        "    \n",
        "    def __init__(self,\n",
        "                 vocab_size_enc,\n",
        "                 vocab_size_dec,\n",
        "                 d_model,\n",
        "                 num_layers,\n",
        "                 FFN_units,\n",
        "                 num_heads,\n",
        "                 dropout_rate,\n",
        "                 name=\"transformer\"):\n",
        "        super(Transformer, self).__init__(name=name)\n",
        "        \n",
        "        self.encoder = Encoder(num_layers,\n",
        "                               FFN_units,\n",
        "                               num_heads,\n",
        "                               dropout_rate,\n",
        "                               vocab_size_enc,\n",
        "                               d_model)\n",
        "        self.decoder = Decoder(num_layers,\n",
        "                               FFN_units,\n",
        "                               num_heads,\n",
        "                               dropout_rate,\n",
        "                               vocab_size_dec,\n",
        "                               d_model)\n",
        "        self.last_linear = layers.Dense(units=vocab_size_dec, name=\"lin_ouput\")\n",
        "    \n",
        "    def create_padding_mask(self, seq):\n",
        "        mask = tf.cast(tf.math.equal(seq, 0), tf.float32)\n",
        "        return mask[:, tf.newaxis, tf.newaxis, :]\n",
        "\n",
        "    def create_look_ahead_mask(self, seq):\n",
        "        seq_len = tf.shape(seq)[1]\n",
        "        look_ahead_mask = 1 - tf.linalg.band_part(tf.ones((seq_len, seq_len)), -1, 0)\n",
        "        return look_ahead_mask\n",
        "    \n",
        "    def call(self, enc_inputs, dec_inputs, training):\n",
        "        enc_mask = self.create_padding_mask(enc_inputs)\n",
        "        dec_mask_1 = tf.maximum(\n",
        "            self.create_padding_mask(dec_inputs),\n",
        "            self.create_look_ahead_mask(dec_inputs)\n",
        "        )\n",
        "        dec_mask_2 = self.create_padding_mask(enc_inputs)\n",
        "        \n",
        "        enc_outputs = self.encoder(enc_inputs, enc_mask, training)\n",
        "        dec_outputs = self.decoder(dec_inputs,\n",
        "                                   enc_outputs,\n",
        "                                   dec_mask_1,\n",
        "                                   dec_mask_2,\n",
        "                                   training)\n",
        "        \n",
        "        outputs = self.last_linear(dec_outputs)\n",
        "        \n",
        "        return outputs"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-c-LRThUPrso",
        "colab_type": "text"
      },
      "source": [
        "# Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qiOdqQ5qPs8z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tf.keras.backend.clear_session()\n",
        "\n",
        "# Hyper-parameters\n",
        "D_MODEL = 128 # 512\n",
        "NUM_LAYERS = 4 # 6\n",
        "FFN_UNITS = 512 # 2048\n",
        "NUM_HEADS = 8 # 8\n",
        "DROPOUT_RATE = 0.1 # 0.1\n",
        "\n",
        "transformer = Transformer(vocab_size_enc=VOCAB_SIZE_EN,\n",
        "                          vocab_size_dec=VOCAB_SIZE_HI,\n",
        "                          d_model=D_MODEL,\n",
        "                          num_layers=NUM_LAYERS,\n",
        "                          FFN_units=FFN_UNITS,\n",
        "                          num_heads=NUM_HEADS,\n",
        "                          dropout_rate=DROPOUT_RATE)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "46xg4Wrg1Wgl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True,\n",
        "                                                            reduction=\"none\")\n",
        "\n",
        "def loss_function(target, pred):\n",
        "    mask = tf.math.logical_not(tf.math.equal(target, 0))\n",
        "    loss_ = loss_object(target, pred)\n",
        "    \n",
        "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
        "    loss_ *= mask\n",
        "    \n",
        "    return tf.reduce_mean(loss_)\n",
        "\n",
        "train_loss = tf.keras.metrics.Mean(name=\"train_loss\")\n",
        "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name=\"train_accuracy\")"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Goque362343",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
        "    \n",
        "    def __init__(self, d_model, warmup_steps=4000):\n",
        "        super(CustomSchedule, self).__init__()\n",
        "        \n",
        "        self.d_model = tf.cast(d_model, tf.float32)\n",
        "        self.warmup_steps = warmup_steps\n",
        "    \n",
        "    def __call__(self, step):\n",
        "        arg1 = tf.math.rsqrt(step)\n",
        "        arg2 = step * (self.warmup_steps**-1.5)\n",
        "        \n",
        "        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)\n",
        "\n",
        "leaning_rate = CustomSchedule(D_MODEL)\n",
        "\n",
        "optimizer = tf.keras.optimizers.Adam(leaning_rate,\n",
        "                                     beta_1=0.9,\n",
        "                                     beta_2=0.98,\n",
        "                                     epsilon=1e-9)\n",
        "        "
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nb_32PIU5Zkh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "8017f6ed-e0cd-4152-fd61-e0be8af3afab"
      },
      "source": [
        "checkpoint_path = \"./checkpoints/\"\n",
        "\n",
        "ckpt = tf.train.Checkpoint(transformer=transformer,\n",
        "                           optimizer=optimizer)\n",
        "\n",
        "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n",
        "\n",
        "if ckpt_manager.latest_checkpoint:\n",
        "    ckpt.restore(ckpt_manager.latest_checkpoint)\n",
        "    print(\"Latest checkpoint restored!!\")"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Latest checkpoint restored!!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lhFK5kUx602K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "EPOCHS = 10\n",
        "for epoch in range(EPOCHS):\n",
        "    print(\"Start of epoch {}\".format(epoch+1))\n",
        "    start = time.time()\n",
        "    \n",
        "    train_loss.reset_states()\n",
        "    train_accuracy.reset_states()\n",
        "    \n",
        "    for (batch, (enc_inputs, targets)) in enumerate(dataset):\n",
        "        dec_inputs = targets[:, :-1]\n",
        "        dec_outputs_real = targets[:, 1:]\n",
        "        with tf.GradientTape() as tape:\n",
        "            predictions = transformer(enc_inputs, dec_inputs, True)\n",
        "            loss = loss_function(dec_outputs_real, predictions)\n",
        "        \n",
        "        gradients = tape.gradient(loss, transformer.trainable_variables)\n",
        "        optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n",
        "        \n",
        "        train_loss(loss)\n",
        "        train_accuracy(dec_outputs_real, predictions)\n",
        "        \n",
        "        if batch % 50 == 0:\n",
        "            print(\"Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}\".format(\n",
        "                epoch+1, batch, train_loss.result(), train_accuracy.result()))\n",
        "            \n",
        "    ckpt_save_path = ckpt_manager.save()\n",
        "    print(\"Saving checkpoint for epoch {} at {}\".format(epoch+1,\n",
        "                                                        ckpt_save_path))\n",
        "    print(\"Time taken for 1 epoch: {} secs\\n\".format(time.time() - start))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nmzyRwDrRGdq",
        "colab_type": "text"
      },
      "source": [
        "# Evaluating"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cNHwJJrz3lPB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def evaluate(inp_sentence):\n",
        "    inp_sentence = \\\n",
        "        [VOCAB_SIZE_EN-2] + tokenizer_en.encode(inp_sentence) + [VOCAB_SIZE_EN-1]\n",
        "    enc_input = tf.expand_dims(inp_sentence, axis=0)\n",
        "    \n",
        "    output = tf.expand_dims([VOCAB_SIZE_HI-2], axis=0)\n",
        "    \n",
        "    for _ in range(MAX_LENGTH):\n",
        "        predictions = transformer(enc_input, output, False)\n",
        "        \n",
        "        prediction = predictions[:, -1:, :]\n",
        "        \n",
        "        predicted_id = tf.cast(tf.argmax(prediction, axis=-1), tf.int32)\n",
        "        \n",
        "        if predicted_id == VOCAB_SIZE_HI-1:\n",
        "            return tf.squeeze(output, axis=0)\n",
        "        \n",
        "        output = tf.concat([output, predicted_id], axis=-1)\n",
        "        \n",
        "    return tf.squeeze(output, axis=0)"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s6VeFKrE6Kdx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def translate(sentence):\n",
        "    output = evaluate(sentence.lower()).numpy()\n",
        "    \n",
        "    predicted_sentence = tokenizer_hi.decode(\n",
        "        [i for i in output if i < VOCAB_SIZE_HI-2]\n",
        "    )\n",
        "    \n",
        "    print(\"Input: {}\".format(sentence))\n",
        "    print(\"Predicted translation: {}\".format(predicted_sentence))"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kc4PNwSM0dpO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "dd4f0d89-fb1e-4563-d123-7ea640f16c7b"
      },
      "source": [
        "translate(\"Trade has stopped \")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: Trade has stopped \n",
            "Predicted translation: व्यापार का रोक दिया गया है । \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XMDSpOp7L33X",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "3631d7ce-e66e-466a-d60d-a1f35d178d56"
      },
      "source": [
        "translate(\"This is the problem we have to solve .\")"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: This is the problem we have to solve .\n",
            "Predicted translation: इस समस्या के लिए एक साथ समस्या है जिससे हम किसी समस्या को हल करना चाहते हैं । \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZdoWKbCP7Czs",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "44da226d-22f4-4cbc-c5b4-014b9bc71c1d"
      },
      "source": [
        "translate(\"Online Education has been adopted by all institutes .\")"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: Online Education has been adopted by all institutes .\n",
            "Predicted translation: सभी प्रमुख विद्यालय के साथ संचालन प्रदान किया गया है । \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3uueuIp9rAOZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "58f1d743-ebd1-45f1-c1de-93fe91df515a"
      },
      "source": [
        "translate(\"union is strength\")"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: union is strength\n",
            "Predicted translation: संयुक्त संयोजन\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aAiS6BPK1Z8B",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "5b453f86-9e16-4be3-d141-86b38259722a"
      },
      "source": [
        "translate(\"very Less employment opportunities are produced\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: very Less employment opportunities are produced\n",
            "Predicted translation: रोजगार की प्रतिभूति के लिए बहुत कम होती है । \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OxGUdwaa14e_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "1239186f-9b8e-455f-e817-356bb5f93179"
      },
      "source": [
        "translate(\"population growth in india is a huge concern\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: population growth in india is a huge concern\n",
            "Predicted translation: भारत में विश्व की एक संख्या है । \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zk9VBIFc3uaQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "1f7db1ff-e073-42f8-8b93-fdc25ae24948"
      },
      "source": [
        "translate(\"union of india includes several states\")"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: union of india includes several states\n",
            "Predicted translation: भारत में संयुक्त राज्य सभा के विभिन्न राज्यों में से एक है । \n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}