Transformer_architecture.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "mf3HT1Z_E8uD"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import torch\n",
        "import math\n",
        "from torch import nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "def get_device():\n",
        "    return torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "def scaled_dot_product(q, k, v, mask=None):\n",
        "    d_k = q.size()[-1]\n",
        "    scaled = torch.matmul(q, k.transpose(-1, -2)) / math.sqrt(d_k)\n",
        "    if mask is not None:\n",
        "        scaled = scaled.permute(1, 0, 2, 3) + mask\n",
        "        scaled = scaled.permute(1, 0, 2, 3)\n",
        "    attention = F.softmax(scaled, dim=-1)\n",
        "    values = torch.matmul(attention, v)\n",
        "    return values, attention"
      ],
      "metadata": {
        "id": "wMKu8JQzFOef"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class PositionalEncoding(nn.Module):\n",
        "    def __init__(self, d_model, max_sequence_length):\n",
        "        super().__init__()\n",
        "        self.max_sequence_length = max_sequence_length\n",
        "        self.d_model = d_model\n",
        "\n",
        "    def forward(self):\n",
        "        even_i = torch.arange(0, self.d_model, 2).float()\n",
        "        denominator = torch.pow(10000, even_i/self.d_model)\n",
        "        position = (torch.arange(self.max_sequence_length)\n",
        "                          .reshape(self.max_sequence_length, 1))\n",
        "        even_PE = torch.sin(position / denominator)\n",
        "        odd_PE = torch.cos(position / denominator)\n",
        "        stacked = torch.stack([even_PE, odd_PE], dim=2)\n",
        "        PE = torch.flatten(stacked, start_dim=1, end_dim=2)\n",
        "        return PE\n"
      ],
      "metadata": {
        "id": "B3Tirh6eFfzx"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class SentenceEmbedding(nn.Module):\n",
        "    \"For a given sentence, create an embedding\"\n",
        "    def __init__(self, max_sequence_length, d_model, string_to_index, START_TOKEN, END_TOKEN, PADDING_TOKEN):\n",
        "        super().__init__()\n",
        "        self.vocab_size = len(string_to_index)\n",
        "        self.max_sequence_length = max_sequence_length\n",
        "        self.embedding = nn.Embedding(self.vocab_size, d_model)\n",
        "        self.string_to_index = string_to_index\n",
        "        self.position_encoder = PositionalEncoding(d_model, max_sequence_length)\n",
        "        self.dropout = nn.Dropout(p=0.1)\n",
        "        self.START_TOKEN = START_TOKEN\n",
        "        self.END_TOKEN = END_TOKEN\n",
        "        self.PADDING_TOKEN = PADDING_TOKEN\n",
        "\n",
        "    def batch_tokenize(self, batch, start_token, end_token):\n",
        "\n",
        "        def tokenize(sentence, start_token, end_token):\n",
        "            sentence_word_indicies = [self.string_to_index[token] for token in list(sentence)]\n",
        "            if start_token:\n",
        "                sentence_word_indicies.insert(0, self.string_to_index[self.START_TOKEN])\n",
        "            if end_token:\n",
        "                sentence_word_indicies.append(self.string_to_index[self.END_TOKEN])\n",
        "            for _ in range(len(sentence_word_indicies), self.max_sequence_length):\n",
        "                sentence_word_indicies.append(self.string_to_index[self.PADDING_TOKEN])\n",
        "            return torch.tensor(sentence_word_indicies)\n",
        "\n",
        "        tokenized = []\n",
        "        for mol in range(len(batch)):\n",
        "           tokenized.append( tokenize(batch[mol], start_token, end_token) )\n",
        "        tokenized = torch.stack(tokenized)\n",
        "        return tokenized.to(get_device())\n",
        "\n",
        "    def forward(self, x, start_token, end_token): # sentence\n",
        "        x = self.batch_tokenize(x, start_token, end_token)\n",
        "        x = self.embedding(x)\n",
        "        pos = self.position_encoder().to(get_device())\n",
        "        x = self.dropout(x + pos)\n",
        "        return x"
      ],
      "metadata": {
        "id": "6VTqv5AdFiwA"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class MultiHeadAttention(nn.Module):\n",
        "    def __init__(self, d_model, num_heads):\n",
        "        super().__init__()\n",
        "        self.d_model = d_model\n",
        "        self.num_heads = num_heads\n",
        "        self.head_dim = d_model // num_heads\n",
        "        self.qkv_layer = nn.Linear(d_model , 3 * d_model)\n",
        "        self.linear_layer = nn.Linear(d_model, d_model)\n",
        "\n",
        "    def forward(self, x, mask):\n",
        "        batch_size, sequence_length, d_model = x.size()\n",
        "        qkv = self.qkv_layer(x)\n",
        "        qkv = qkv.reshape(batch_size, sequence_length, self.num_heads, 3 * self.head_dim)\n",
        "        qkv = qkv.permute(0, 2, 1, 3)\n",
        "        q, k, v = qkv.chunk(3, dim=-1)\n",
        "        values, attention = scaled_dot_product(q, k, v, mask)\n",
        "        values = values.permute(0, 2, 1, 3).reshape(batch_size, sequence_length, self.num_heads * self.head_dim)\n",
        "        out = self.linear_layer(values)\n",
        "        return out\n"
      ],
      "metadata": {
        "id": "qgpE-GcmFrJJ"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class LayerNormalization(nn.Module):\n",
        "    def __init__(self, parameters_shape, eps=1e-5):\n",
        "        super().__init__()\n",
        "        self.parameters_shape=parameters_shape\n",
        "        self.eps=eps\n",
        "        self.gamma = nn.Parameter(torch.ones(parameters_shape))\n",
        "        self.beta =  nn.Parameter(torch.zeros(parameters_shape))\n",
        "\n",
        "    def forward(self, inputs):\n",
        "        dims = [-(i + 1) for i in range(len(self.parameters_shape))]\n",
        "        mean = inputs.mean(dim=dims, keepdim=True)\n",
        "        var = ((inputs - mean) ** 2).mean(dim=dims, keepdim=True)\n",
        "        std = (var + self.eps).sqrt()\n",
        "        y = (inputs - mean) / std\n",
        "        out = self.gamma * y + self.beta\n",
        "        return out\n"
      ],
      "metadata": {
        "id": "K0ZL3vFbFt4a"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class PositionwiseFeedForward(nn.Module):\n",
        "    def __init__(self, d_model, hidden, drop_prob=0.1):\n",
        "        super(PositionwiseFeedForward, self).__init__()\n",
        "        self.linear1 = nn.Linear(d_model, hidden)\n",
        "        self.linear2 = nn.Linear(hidden, d_model)\n",
        "        self.relu = nn.ReLU()\n",
        "        self.dropout = nn.Dropout(p=drop_prob)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.linear1(x)\n",
        "        x = self.relu(x)\n",
        "        x = self.dropout(x)\n",
        "        x = self.linear2(x)\n",
        "        return x\n"
      ],
      "metadata": {
        "id": "pB8nXZehGKQ-"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class EncoderLayer(nn.Module):\n",
        "    def __init__(self, d_model, ffn_hidden, num_heads, drop_prob):\n",
        "        super(EncoderLayer, self).__init__()\n",
        "        self.attention = MultiHeadAttention(d_model=d_model, num_heads=num_heads)\n",
        "        self.norm1 = LayerNormalization(parameters_shape=[d_model])\n",
        "        self.dropout1 = nn.Dropout(p=drop_prob)\n",
        "        self.ffn = PositionwiseFeedForward(d_model=d_model, hidden=ffn_hidden, drop_prob=drop_prob)\n",
        "        self.norm2 = LayerNormalization(parameters_shape=[d_model])\n",
        "        self.dropout2 = nn.Dropout(p=drop_prob)\n",
        "\n",
        "    def forward(self, x, self_attention_mask):\n",
        "        residual_x = x.clone()\n",
        "        x = self.attention(x, mask=self_attention_mask)\n",
        "        x = self.dropout1(x)\n",
        "        x = self.norm1(x + residual_x)\n",
        "        residual_x = x.clone()\n",
        "        x = self.ffn(x)\n",
        "        x = self.dropout2(x)\n",
        "        x = self.norm2(x + residual_x)\n",
        "        return x"
      ],
      "metadata": {
        "id": "PduxKMN5GN77"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class SequentialEncoder(nn.Sequential):\n",
        "    def forward(self, *inputs):\n",
        "        x, self_attention_mask  = inputs\n",
        "        for module in self._modules.values():\n",
        "            x = module(x, self_attention_mask)\n",
        "        return x"
      ],
      "metadata": {
        "id": "C33XwZJVGRMW"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Encoder(nn.Module):\n",
        "    def __init__(self,\n",
        "                 d_model,\n",
        "                 ffn_hidden,\n",
        "                 num_heads,\n",
        "                 drop_prob,\n",
        "                 num_layers,\n",
        "                 max_sequence_length,\n",
        "                 string_to_index,\n",
        "                 START_TOKEN,\n",
        "                 END_TOKEN,\n",
        "                 PADDING_TOKEN):\n",
        "        super().__init__()\n",
        "        self.sentence_embedding = SentenceEmbedding(max_sequence_length, d_model, string_to_index, START_TOKEN, END_TOKEN, PADDING_TOKEN)\n",
        "        self.layers = SequentialEncoder(*[EncoderLayer(d_model, ffn_hidden, num_heads, drop_prob)\n",
        "                                      for _ in range(num_layers)])\n",
        "\n",
        "    def forward(self, x, self_attention_mask, start_token, end_token):\n",
        "        x = self.sentence_embedding(x, start_token, end_token)\n",
        "        x = self.layers(x, self_attention_mask)\n",
        "        return x"
      ],
      "metadata": {
        "id": "A6q15ztXGTxH"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class MultiHeadCrossAttention(nn.Module):\n",
        "    def __init__(self, d_model, num_heads):\n",
        "        super().__init__()\n",
        "        self.d_model = d_model\n",
        "        self.num_heads = num_heads\n",
        "        self.head_dim = d_model // num_heads\n",
        "        self.kv_layer = nn.Linear(d_model , 2 * d_model)\n",
        "        self.q_layer = nn.Linear(d_model , d_model)\n",
        "        self.linear_layer = nn.Linear(d_model, d_model)\n",
        "\n",
        "    def forward(self, x, y, mask):\n",
        "        batch_size, sequence_length, d_model = x.size() # in practice, this is the same for both languages...so we can technically combine with normal attention\n",
        "        kv = self.kv_layer(x)\n",
        "        q = self.q_layer(y)\n",
        "        kv = kv.reshape(batch_size, sequence_length, self.num_heads, 2 * self.head_dim)\n",
        "        q = q.reshape(batch_size, sequence_length, self.num_heads, self.head_dim)\n",
        "        kv = kv.permute(0, 2, 1, 3)\n",
        "        q = q.permute(0, 2, 1, 3)\n",
        "        k, v = kv.chunk(2, dim=-1)\n",
        "        values, attention = scaled_dot_product(q, k, v, mask) # We don't need the mask for cross attention, removing in outer function!\n",
        "        values = values.permute(0, 2, 1, 3).reshape(batch_size, sequence_length, d_model)\n",
        "        out = self.linear_layer(values)\n",
        "        return out"
      ],
      "metadata": {
        "id": "2msYNE2rGZWY"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class DecoderLayer(nn.Module):\n",
        "    def __init__(self, d_model, ffn_hidden, num_heads, drop_prob):\n",
        "        super(DecoderLayer, self).__init__()\n",
        "        self.self_attention = MultiHeadAttention(d_model=d_model, num_heads=num_heads)\n",
        "        self.layer_norm1 = LayerNormalization(parameters_shape=[d_model])\n",
        "        self.dropout1 = nn.Dropout(p=drop_prob)\n",
        "\n",
        "        self.encoder_decoder_attention = MultiHeadCrossAttention(d_model=d_model, num_heads=num_heads)\n",
        "        self.layer_norm2 = LayerNormalization(parameters_shape=[d_model])\n",
        "        self.dropout2 = nn.Dropout(p=drop_prob)\n",
        "\n",
        "        self.ffn = PositionwiseFeedForward(d_model=d_model, hidden=ffn_hidden, drop_prob=drop_prob)\n",
        "        self.layer_norm3 = LayerNormalization(parameters_shape=[d_model])\n",
        "        self.dropout3 = nn.Dropout(p=drop_prob)\n",
        "\n",
        "    def forward(self, x, y, self_attention_mask, cross_attention_mask):\n",
        "        _y = y.clone()\n",
        "        y = self.self_attention(y, mask=self_attention_mask)\n",
        "        y = self.dropout1(y)\n",
        "        y = self.layer_norm1(y + _y)\n",
        "\n",
        "        _y = y.clone()\n",
        "        y = self.encoder_decoder_attention(x, y, mask=cross_attention_mask)\n",
        "        y = self.dropout2(y)\n",
        "        y = self.layer_norm2(y + _y)\n",
        "\n",
        "        _y = y.clone()\n",
        "        y = self.ffn(y)\n",
        "        y = self.dropout3(y)\n",
        "        y = self.layer_norm3(y + _y)\n",
        "        return y\n"
      ],
      "metadata": {
        "id": "LdSFFqn9GcFs"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class SequentialDecoder(nn.Sequential):\n",
        "    def forward(self, *inputs):\n",
        "        x, y, self_attention_mask, cross_attention_mask = inputs\n",
        "        for module in self._modules.values():\n",
        "            y = module(x, y, self_attention_mask, cross_attention_mask)\n",
        "        return y\n"
      ],
      "metadata": {
        "id": "GzEIXBI0GhUw"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Decoder(nn.Module):\n",
        "    def __init__(self,\n",
        "                 d_model,\n",
        "                 ffn_hidden,\n",
        "                 num_heads,\n",
        "                 drop_prob,\n",
        "                 num_layers,\n",
        "                 max_sequence_length,\n",
        "                 string_to_index,\n",
        "                 START_TOKEN,\n",
        "                 END_TOKEN,\n",
        "                 PADDING_TOKEN):\n",
        "        super().__init__()\n",
        "        self.sentence_embedding = SentenceEmbedding(max_sequence_length, d_model, string_to_index, START_TOKEN, END_TOKEN, PADDING_TOKEN)\n",
        "        self.layers = SequentialDecoder(*[DecoderLayer(d_model, ffn_hidden, num_heads, drop_prob) for _ in range(num_layers)])\n",
        "\n",
        "    def forward(self, x, y, self_attention_mask, cross_attention_mask, start_token, end_token):\n",
        "        y = self.sentence_embedding(y, start_token, end_token)\n",
        "        y = self.layers(x, y, self_attention_mask, cross_attention_mask)\n",
        "        return y\n"
      ],
      "metadata": {
        "id": "9L-sX0dAGkRH"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Transformer(nn.Module):\n",
        "    def __init__(self,\n",
        "                d_model,\n",
        "                ffn_hidden,\n",
        "                num_heads,\n",
        "                drop_prob,\n",
        "                num_layers,\n",
        "                max_sequence_length,\n",
        "                vocab_size,\n",
        "                stoi,\n",
        "                START_TOKEN,\n",
        "                END_TOKEN,\n",
        "                PADDING_TOKEN\n",
        "                ):\n",
        "        super().__init__()\n",
        "        self.encoder = Encoder(d_model, ffn_hidden, num_heads, drop_prob, num_layers, max_sequence_length, stoi, START_TOKEN, END_TOKEN, PADDING_TOKEN)\n",
        "        self.decoder = Decoder(d_model, ffn_hidden, num_heads, drop_prob, num_layers, max_sequence_length, stoi, START_TOKEN, END_TOKEN, PADDING_TOKEN)\n",
        "        self.linear = nn.Linear(d_model, vocab_size)\n",
        "        self.device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
        "\n",
        "    def forward(self,\n",
        "                x,\n",
        "                y,\n",
        "                encoder_self_attention_mask=None,\n",
        "                decoder_self_attention_mask=None,\n",
        "                decoder_cross_attention_mask=None,\n",
        "                enc_start_token=False,\n",
        "                enc_end_token=False,\n",
        "                dec_start_token=False, # We should make this true\n",
        "                dec_end_token=False): # x, y are batch of sentences\n",
        "        x = self.encoder(x, encoder_self_attention_mask, start_token=enc_start_token, end_token=enc_end_token)\n",
        "        out = self.decoder(x, y, decoder_self_attention_mask, decoder_cross_attention_mask, start_token=dec_start_token, end_token=dec_end_token)\n",
        "        out = self.linear(out)\n",
        "        return out"
      ],
      "metadata": {
        "id": "-AOiL5SCGmIg"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "NEG_INFTY = -1e9\n",
        "\n",
        "def create_masks(batch_1, batch_2):\n",
        "    num_sentences = len(batch_1)\n",
        "    look_ahead_mask = torch.full([max_sequence_length, max_sequence_length] , True)\n",
        "    look_ahead_mask = torch.triu(look_ahead_mask, diagonal=1)\n",
        "    encoder_padding_mask = torch.full([num_sentences, max_sequence_length, max_sequence_length] , False)\n",
        "    decoder_padding_mask_self_attention = torch.full([num_sentences, max_sequence_length, max_sequence_length] , False)\n",
        "    decoder_padding_mask_cross_attention = torch.full([num_sentences, max_sequence_length, max_sequence_length] , False)\n",
        "\n",
        "    for idx in range(num_sentences):\n",
        "      mol1_length, mol2_length = len(batch_1[idx]), len(batch_2[idx])\n",
        "      encod_to_padding_mask = np.arange(mol1_length + 1, max_sequence_length)\n",
        "      decode_to_padding_mask = np.arange(mol2_length + 1, max_sequence_length)\n",
        "      encoder_padding_mask[idx, :, encod_to_padding_mask] = True\n",
        "      encoder_padding_mask[idx, encod_to_padding_mask, :] = True\n",
        "      decoder_padding_mask_self_attention[idx, :, decode_to_padding_mask] = True\n",
        "      decoder_padding_mask_self_attention[idx, decode_to_padding_mask, :] = True\n",
        "      decoder_padding_mask_cross_attention[idx, :, encod_to_padding_mask] = True\n",
        "      decoder_padding_mask_cross_attention[idx, decode_to_padding_mask, :] = True\n",
        "\n",
        "    encoder_self_attention_mask = torch.where(encoder_padding_mask, NEG_INFTY, 0)\n",
        "    decoder_self_attention_mask =  torch.where(look_ahead_mask + decoder_padding_mask_self_attention, NEG_INFTY, 0)\n",
        "    decoder_cross_attention_mask = torch.where(decoder_padding_mask_cross_attention, NEG_INFTY, 0)\n",
        "    return encoder_self_attention_mask, decoder_self_attention_mask, decoder_cross_attention_mask"
      ],
      "metadata": {
        "id": "oFPAmsplIsiy"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "86BmlqCwmhEB",
        "outputId": "7a85c2a4-987f-4d5d-ef38-f76a92b2c7e3"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import csv\n",
        "def extract_column(csv_file, column_index):\n",
        "    column_values = []\n",
        "    with open(csv_file, 'r', newline='') as file:\n",
        "        reader = csv.reader(file)\n",
        "        for row in reader:\n",
        "            if len(row) > column_index:  # Check if the row has enough columns\n",
        "                column_values.append(row[column_index])\n",
        "    return column_values\n",
        "\n",
        "# Example usage:\n",
        "csv_file_path = '/content/drive/MyDrive/Compounds2.csv'\n",
        "column_index = 0  # Change this to the index of the column you want to extract (0-indexed)\n",
        "column_values = extract_column(csv_file_path, column_index)\n",
        "\n",
        "print(len(column_values))\n"
      ],
      "metadata": {
        "id": "bCXbtOfHKgPf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c2ff82c2-77b1-4855-b9a3-430575c74926"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2316318\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "num=len(column_values)\n",
        "maxlen=102\n",
        "mol_values_all=[]\n",
        "all=''\n",
        "column_values=column_values[:100000]\n",
        "\n",
        "for item in column_values:\n",
        "  if len(item)<=maxlen-2:\n",
        "    mol_values_all.append(item)\n",
        "\n",
        "n=int(0.85*len(mol_values_all))\n",
        "\n",
        "mol_values=mol_values_all[:n]\n",
        "mol_values_test=mol_values_all[n:]\n",
        "\n",
        "print(len(mol_values_all))\n",
        "print(len(mol_values))\n",
        "print(len(mol_values[0]))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "87e90qxQNgnE",
        "outputId": "effcfef3-e65e-4bce-8e4f-90702e611fea"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "97201\n",
            "82620\n",
            "32\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# total_mol=1000\n",
        "# mol_values=mol_values[:total_mol]\n",
        "print(mol_values[0])\n",
        "\n",
        "for i in range(len(mol_values)):\n",
        "    all+=mol_values[i]\n",
        "print(len(all))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZRpBcwP3O31K",
        "outputId": "e13bd3f6-44dc-4f91-a8cd-97f4be12fa5c"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C\n",
            "2827813\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "START_TOKEN = '<START>'\n",
        "PADDING_TOKEN = '<PADDING>'\n",
        "END_TOKEN = '<END>'\n",
        "\n",
        "chars_1=sorted(list(set(all)))\n",
        "chars=[]\n",
        "chars.append(START_TOKEN)\n",
        "for i in range(len(chars_1)):\n",
        "  chars.append(chars_1[i])\n",
        "chars.append(PADDING_TOKEN)\n",
        "chars.append(END_TOKEN)\n",
        "\n",
        "vocab_size=len(chars)\n",
        "print(''.join(chars))\n",
        "print(vocab_size)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pJJKk4U0YsRf",
        "outputId": "3addbc11-a78b-49f3-dff1-2d4bc768078c"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<START>#()+-.123456789=ABCDEFGHIKLMNOPRSTUVWXYZ[]abcdefghiklmnoprstuy<PADDING><END>\n",
            "65\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "stoi={ch:i for i,ch in enumerate(chars)}\n",
        "itos={i:ch for i,ch in enumerate(chars)}\n",
        "encode=lambda s:[stoi[ch] for ch in s] # takes a string, outputs a list of integers\n",
        "decode=lambda l: \"\".join([itos[i] for i in l]) # takes a list of integers, outputs a string\n",
        "\n",
        "data=torch.tensor(encode(all),dtype=torch.long)\n",
        "print(data.shape,data.dtype)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mskxjIHkOPUC",
        "outputId": "5d49a611-b00e-45ed-dff5-eefb5218f685"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([2827813]) torch.int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import Dataset, DataLoader\n",
        "\n",
        "class TextDataset(Dataset):\n",
        "\n",
        "    def __init__(self, mol_values):\n",
        "        self.mol_values = mol_values\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.mol_values)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        return self.mol_values[idx]"
      ],
      "metadata": {
        "id": "lku30jDsHmKQ"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = TextDataset(mol_values)\n",
        "print(len(dataset))\n",
        "dataset[1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "id": "oFtpkzO8H1OA",
        "outputId": "3c5aa5aa-25d4-4e02-fe61-ca10a31d7637"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "82620\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'CC(=O)OC(CC(=O)O)C[N+](C)(C)C'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# from transformer import Transformer\n",
        "d_model = 512\n",
        "batch_size = 30\n",
        "ffn_hidden = 2048\n",
        "num_heads = 8\n",
        "drop_prob = 0.2\n",
        "num_layers = 3\n",
        "max_sequence_length = maxlen\n",
        "vocab_size = len(chars)\n",
        "\n",
        "transformer = Transformer(d_model,\n",
        "                          ffn_hidden,\n",
        "                          num_heads,\n",
        "                          drop_prob,\n",
        "                          num_layers,\n",
        "                          max_sequence_length,\n",
        "                          vocab_size,\n",
        "                          stoi,\n",
        "                          START_TOKEN,\n",
        "                          END_TOKEN,\n",
        "                          PADDING_TOKEN)"
      ],
      "metadata": {
        "id": "VzaP1vVvIOhn"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "transformer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4-J-1FgdJz0g",
        "outputId": "40124e96-0fc2-4879-836a-c857e94ea45e"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Transformer(\n",
              "  (encoder): Encoder(\n",
              "    (sentence_embedding): SentenceEmbedding(\n",
              "      (embedding): Embedding(65, 512)\n",
              "      (position_encoder): PositionalEncoding()\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (layers): SequentialEncoder(\n",
              "      (0): EncoderLayer(\n",
              "        (attention): MultiHeadAttention(\n",
              "          (qkv_layer): Linear(in_features=512, out_features=1536, bias=True)\n",
              "          (linear_layer): Linear(in_features=512, out_features=512, bias=True)\n",
              "        )\n",
              "        (norm1): LayerNormalization()\n",
              "        (dropout1): Dropout(p=0.2, inplace=False)\n",
              "        (ffn): PositionwiseFeedForward(\n",
              "          (linear1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "          (linear2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "          (relu): ReLU()\n",
              "          (dropout): Dropout(p=0.2, inplace=False)\n",
              "        )\n",
              "        (norm2): LayerNormalization()\n",
              "        (dropout2): Dropout(p=0.2, inplace=False)\n",
              "      )\n",
              "      (1): EncoderLayer(\n",
              "        (attention): MultiHeadAttention(\n",
              "          (qkv_layer): Linear(in_features=512, out_features=1536, bias=True)\n",
              "          (linear_layer): Linear(in_features=512, out_features=512, bias=True)\n",
              "        )\n",
              "        (norm1): LayerNormalization()\n",
              "        (dropout1): Dropout(p=0.2, inplace=False)\n",
              "        (ffn): PositionwiseFeedForward(\n",
              "          (linear1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "          (linear2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "          (relu): ReLU()\n",
              "          (dropout): Dropout(p=0.2, inplace=False)\n",
              "        )\n",
              "        (norm2): LayerNormalization()\n",
              "        (dropout2): Dropout(p=0.2, inplace=False)\n",
              "      )\n",
              "      (2): EncoderLayer(\n",
              "        (attention): MultiHeadAttention(\n",
              "          (qkv_layer): Linear(in_features=512, out_features=1536, bias=True)\n",
              "          (linear_layer): Linear(in_features=512, out_features=512, bias=True)\n",
              "        )\n",
              "        (norm1): LayerNormalization()\n",
              "        (dropout1): Dropout(p=0.2, inplace=False)\n",
              "        (ffn): PositionwiseFeedForward(\n",
              "          (linear1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "          (linear2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "          (relu): ReLU()\n",
              "          (dropout): Dropout(p=0.2, inplace=False)\n",
              "        )\n",
              "        (norm2): LayerNormalization()\n",
              "        (dropout2): Dropout(p=0.2, inplace=False)\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (decoder): Decoder(\n",
              "    (sentence_embedding): SentenceEmbedding(\n",
              "      (embedding): Embedding(65, 512)\n",
              "      (position_encoder): PositionalEncoding()\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (layers): SequentialDecoder(\n",
              "      (0): DecoderLayer(\n",
              "        (self_attention): MultiHeadAttention(\n",
              "          (qkv_layer): Linear(in_features=512, out_features=1536, bias=True)\n",
              "          (linear_layer): Linear(in_features=512, out_features=512, bias=True)\n",
              "        )\n",
              "        (layer_norm1): LayerNormalization()\n",
              "        (dropout1): Dropout(p=0.2, inplace=False)\n",
              "        (encoder_decoder_attention): MultiHeadCrossAttention(\n",
              "          (kv_layer): Linear(in_features=512, out_features=1024, bias=True)\n",
              "          (q_layer): Linear(in_features=512, out_features=512, bias=True)\n",
              "          (linear_layer): Linear(in_features=512, out_features=512, bias=True)\n",
              "        )\n",
              "        (layer_norm2): LayerNormalization()\n",
              "        (dropout2): Dropout(p=0.2, inplace=False)\n",
              "        (ffn): PositionwiseFeedForward(\n",
              "          (linear1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "          (linear2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "          (relu): ReLU()\n",
              "          (dropout): Dropout(p=0.2, inplace=False)\n",
              "        )\n",
              "        (layer_norm3): LayerNormalization()\n",
              "        (dropout3): Dropout(p=0.2, inplace=False)\n",
              "      )\n",
              "      (1): DecoderLayer(\n",
              "        (self_attention): MultiHeadAttention(\n",
              "          (qkv_layer): Linear(in_features=512, out_features=1536, bias=True)\n",
              "          (linear_layer): Linear(in_features=512, out_features=512, bias=True)\n",
              "        )\n",
              "        (layer_norm1): LayerNormalization()\n",
              "        (dropout1): Dropout(p=0.2, inplace=False)\n",
              "        (encoder_decoder_attention): MultiHeadCrossAttention(\n",
              "          (kv_layer): Linear(in_features=512, out_features=1024, bias=True)\n",
              "          (q_layer): Linear(in_features=512, out_features=512, bias=True)\n",
              "          (linear_layer): Linear(in_features=512, out_features=512, bias=True)\n",
              "        )\n",
              "        (layer_norm2): LayerNormalization()\n",
              "        (dropout2): Dropout(p=0.2, inplace=False)\n",
              "        (ffn): PositionwiseFeedForward(\n",
              "          (linear1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "          (linear2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "          (relu): ReLU()\n",
              "          (dropout): Dropout(p=0.2, inplace=False)\n",
              "        )\n",
              "        (layer_norm3): LayerNormalization()\n",
              "        (dropout3): Dropout(p=0.2, inplace=False)\n",
              "      )\n",
              "      (2): DecoderLayer(\n",
              "        (self_attention): MultiHeadAttention(\n",
              "          (qkv_layer): Linear(in_features=512, out_features=1536, bias=True)\n",
              "          (linear_layer): Linear(in_features=512, out_features=512, bias=True)\n",
              "        )\n",
              "        (layer_norm1): LayerNormalization()\n",
              "        (dropout1): Dropout(p=0.2, inplace=False)\n",
              "        (encoder_decoder_attention): MultiHeadCrossAttention(\n",
              "          (kv_layer): Linear(in_features=512, out_features=1024, bias=True)\n",
              "          (q_layer): Linear(in_features=512, out_features=512, bias=True)\n",
              "          (linear_layer): Linear(in_features=512, out_features=512, bias=True)\n",
              "        )\n",
              "        (layer_norm2): LayerNormalization()\n",
              "        (dropout2): Dropout(p=0.2, inplace=False)\n",
              "        (ffn): PositionwiseFeedForward(\n",
              "          (linear1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "          (linear2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "          (relu): ReLU()\n",
              "          (dropout): Dropout(p=0.2, inplace=False)\n",
              "        )\n",
              "        (layer_norm3): LayerNormalization()\n",
              "        (dropout3): Dropout(p=0.2, inplace=False)\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (linear): Linear(in_features=512, out_features=65, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_loader = DataLoader(dataset, batch_size)\n",
        "iterator = iter(train_loader)\n"
      ],
      "metadata": {
        "id": "wXiTHGgdHxrJ"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for batch_num, batch in enumerate(iterator):\n",
        "    print(batch)\n",
        "    print(batch_num)\n",
        "    if batch_num > 3:\n",
        "        break"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3s3dY-ZjIZ9T",
        "outputId": "57bcec8f-62c7-4c62-8d6e-6b85d0f71b14"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C', 'CC(=O)OC(CC(=O)O)C[N+](C)(C)C', 'C1=CC(C(C(=C1)C(=O)O)O)O', 'CC(CN)O', 'C(C(=O)COP(=O)(O)O)N', 'C1=CC(=C(C=C1[N+](=O)[O-])[N+](=O)[O-])Cl', 'CCN1C=NC2=C(N=CN=C21)N', 'CCC(C)(C(C(=O)O)O)O', 'C1(C(C(C(C(C1O)O)OP(=O)(O)O)O)O)O', 'C(CCl)Cl', 'C1=C(C=C(C(=C1O)O)O)O', 'C1=CC(=C(C=C1Cl)Cl)Cl', 'CCCCCC(=O)C=CC1C(CC(=O)C1CCCCCCC(=O)O)O', 'CC12CCC(=O)CC1CCC3C2CCC4(C3CCC4O)C', 'C1CCC(=O)NCCCCCC(=O)NCC1', 'C1C=CC(=NC1C(=O)O)C(=O)O', 'C(C(C(C(=O)C(=O)C(=O)O)O)O)O', 'C1=CC(=C(C(=C1)O)O)C(=O)O', 'C1=CC(=C(C(=C1)O)O)CCC(=O)O', 'CCC(C(=O)C)(C(=O)O)O', 'CC(=O)C(C)(C(=O)O)O', 'C(=CC(=C(C(=O)O)N)C(=O)O)C=O', 'C(=O)C(C(=O)O)N', 'C(C=O)Cl', 'C(CCl)O', 'CC(C)C(C(C(=O)O)O)C(=O)O', 'CC(C)(CO)C(=O)C(=O)O', 'CC1(COC(=O)C1=O)C', 'C1C(C(C(OC1O)CO)O)O', 'C(CC(=O)O)C(=O)C=CC=C(C(=O)O)O']\n",
            "0\n",
            "['C(CC(=O)O)C(C(=O)O)O', 'C(C(=O)[O-])(C(=O)[O-])O', 'C(C(=O)O)(C(=O)O)O', 'CCC(C)C(=O)C(=O)O', 'C(CC(=O)N)C(=O)C(=O)O', 'CC(C)C(=O)C(=O)O', 'C(C(C(C(C(=O)C(=O)O)O)O)O)O', 'C(CC(=O)O)C(=O)C(=O)O', 'CC(=CCCC(=CCCC(=CCCC(=CCCC(=CCCC(=CCCC(=CCCC(=CCC1=CC=CC=C1O)C)C)C)C)C)C)C)C', 'CCC(=O)C(=O)O', 'C(C(C(=O)O)OP(=O)(O)O)O', 'C(C(C(=O)O)OP(=O)(O)O)OP(=O)(O)O', 'C1=C(C(C=C(C1O)Cl)Cl)Cl', 'CC(CC(C(=O)O)N)N', 'C1=C(C(C=C(C1O)Cl)O)Cl', 'C1=C(C(=CC(=C1Cl)O)Cl)O', 'C1=CC(=C(C=C1Cl)O)Cl', 'C(C(=O)C(C(C(=O)C(=O)O)O)O)O', 'CC1(C(COP(=O)(OP(=O)(O1)O)O)O)CO', 'CC(=O)C=CC=C(C(=O)O)O', 'CC(C)CC(=O)C(=O)O', 'C(CC(=O)C(=O)O)CC(=O)O', 'C1=CC(=C(C=C1C(=O)O)O)O', 'C1=NC(=C2C(=N1)N(C=N2)C3C(C(C(O3)COP(=O)(O)O)OP(=O)(O)O)O)N', 'C(CN)C=O', 'CC12CCC3C(C1CCC2=O)CC=C4C3(CCC(C4)O)C', 'CC(C)C(CC(=O)O)(C(=O)O)O', 'C(C(C(=O)O)N)Cl', 'C1=CC(=CN=C1)C#N', 'C=C(C(=O)O)OC1CC(=CC(C1O)OP(=O)(O)O)C(=O)O']\n",
            "1\n",
            "['C[N+](C)(C)CC(CC(=O)O)O', 'C1=CC(=C(C(=C1)O)N)C(=O)O', 'CC(CO)C(=O)O', 'C1=CC(=C(C(=C1)O)N)C(=O)CC(C(=O)O)N', 'C1=CC(=CC(=C1)O)CCC(=O)O', 'COC1=CC2=C(C=CN=C2C=C1)C(C3CC4CCN3CC4(C=C)O)O', 'C(CC(=O)O)C(=O)CC(=O)O', 'CC(=O)CC(=O)O', 'C(C(=O)C(=O)O)S', 'C1=CC(=CC(=C1)O)C=O', 'C1=CC(=CC(=C1)O)CO', 'C1C=C(OC1=O)CC(=O)O', 'C(C(=O)C(=O)O)OP(=O)(O)O', 'C(C(C(=O)O)N)OP(=O)(O)O', 'C1=CC=C(C=C1)CCC(=O)O', 'C1=CC(=CN=C1)CC(=O)O', 'C(C(C(=O)O)N)S(=O)O', 'C(C(=O)C(=O)O)S(=O)O', 'C(CNC(=O)N)C(=O)O', 'C1=CC(=CN=C1)CCCC(=O)O', 'C1=NC(=C(N1)C(=O)O)N', 'C(CC=O)CN', 'C(CC(=O)O)CN', 'C1=CC(=O)OC1=CC(=O)O', 'C(C(=O)CC(=O)O)C(=O)C=CC(=O)O', 'C(CC(=O)N)CN=C(N)N', 'CC(CC(=O)C(=O)O)O', 'C1=CC(=CC=C1CO)O', 'C1=CC(=CC=C1C=O)O', 'C1=CC(=CC=C1CC(=O)O)O']\n",
            "2\n",
            "['C1=NC(C(=O)N1)CCC(=O)O', 'CC(C)CCC=O', 'C(C(=O)C(=O)O)C(CC(=O)O)(C(=O)O)O', 'CC(C)(COP(=O)(O)O)C(C(=O)NCCC(=O)O)O', 'C[N+](C)(C)CCCC=O', 'C[N+](C)(C)CCCC(=O)O', 'C1=CC(=CC=C1C(=O)O)O', 'C(CC(=O)O)C(=O)CN', 'C(CCN)CC(=O)O', 'CC(CCCC(C)C1CCC2C1(C(CC3C2C(CC4C3(CCC(C4)O)C)O)O)C)CO', 'C1=CC2=C(C=C1O)C(=CN2)CC(C(=O)O)N', 'CSCC1C(C(C(O1)N2C=NC3=C(N=CN=C32)N)O)O', 'C(C1C(C(C(O1)N)O)O)OP(=O)(O)O', 'C(C1C(C(C(O1)NC(=O)CNC=O)O)O)OP(=O)(O)O', 'C1=NC(=C(N1)C(=O)O)NC(=O)N', 'C1CNC2=NC=NC=C2N1', 'C(C(=O)C(C(C(C(=O)O)O)O)O)O', 'CCCCCC(C=CC1C(CC(=O)C1CC=CCCCC(=O)O)O)O', 'CCCCCC(C=CC1C(CC(C1CC=CCCCC(=O)O)O)O)O', 'CC1(C(N2C(S1)C(C2=O)N)C(=O)O)C', 'C(CC(=O)C=[N+]=[N-])C(C(=O)O)N', 'C1CN=C2C(=N1)C=NC=N2', 'CCCCCC=CCC(C=CC=CC=CC(CCCC(=O)O)O)O', 'CC(C)CCCC(C)C1CCC2C1(CCC3C2=CC=C4C3(CCC(C4)O)C)C', 'CC(C(=O)CCCCCC(=O)O)N', 'C(CO)O', 'CC(=O)[O-]', 'CC(=O)O', 'CC=O', 'CC(=O)N']\n",
            "3\n",
            "['CC(C(=O)C)O', 'CC(=O)C', 'CC(=O)SCCNC(=O)CCNC(=O)C(C(C)(C)COP(=O)(O)OP(=O)(O)OCC1C(C(C(O1)N2C=NC3=C(N=CN=C32)N)O)OP(=O)(O)O)O', 'CC(=O)NC(CCC(=O)O)C(=O)O', 'CC(=O)OP(=O)(O)O', 'CC(=O)OCC[N+](C)(C)C', 'CC(=O)OCC(C(=O)O)N', 'C1=NC2=NC=NC(=C2N1)N', 'C1=NC(=C2C(=N1)N(C=N2)C3C(C(C(O3)CO)O)O)N', 'C1=NC(=C2C(=N1)N(C=N2)C3C(C(C(O3)COP(=O)(O)OP(=O)(O)OCC4C(C(C(O4)O)O)O)O)O)N', 'C1=NC(=C2C(=N1)N(C=N2)C3C(C(C(O3)CSCCC(C(=O)O)N)O)O)N', 'C1=NC(=C2C(=N1)N(C=N2)C3C(C(C(O3)COP(=O)(O)O)O)O)NC(CC(=O)O)C(=O)O', 'C(CCC(=O)O)CC(=O)O', 'C1=NC(=C2C(=N1)N(C=N2)C3C(C(C(O3)COP(=O)(O)OP(=O)(O)O)O)O)N', 'C1=NC(=C2C(=N1)N(C=N2)C3C(C(C(O3)COP(=O)(O)OP(=O)(O)OC4C(C(C(C(O4)CO)O)O)O)O)O)N', 'C(CCN=C(N)N)CN', 'C1=NC(=C(N1C2C(C(C(O2)COP(=O)(O)O)O)O)N)C(=O)N', 'CC(=CCCC(=CCCC(=CCCC(=CCCC(=CCCC(=CCCC(=CCOP(=O)(O)OP(=O)(O)O)C)C)C)C)C)C)C', 'CC(=CCCC(=CCCC(=CCCC(=CCCC(=CCCC(=CCOP(=O)(O)OP(=O)(O)O)C)C)C)C)C)C', 'C(C(=O)O)(NC(=O)N)NC(=O)N', 'C1(C(=O)NC(=O)N1)NC(=O)N', 'CC(C(C(=O)O)N)O', 'C(C1C(C(C(C(O1)O)O)O)O)O', 'C(CC=O)CC(C(=O)O)N', 'C(C1C(C(C(C(O1)O)O)O)O)OP(=O)(O)O', 'CC(=O)NC1C(CC(OC1C(C(CO)O)O)(C(=O)O)OCC2C(C(C(C(O2)OC3C(OC(C(C3O)NC(=O)C)O)CO)O)O)O)O', 'CC1=CC2=C(C=C1C)N(C=N2)C3C(C(C(O3)CO)O)O', 'CCCCCC(C=CC1C(CC(=O)C1CCCCCCC(=O)O)O)O', 'CC(=O)CN', 'CC1=NC=C(C(=N1)N)COP(=O)(O)O']\n",
            "4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# for i in range(2100,2500):\n",
        "#   if len(mol_values[i])==100:\n",
        "#     print(\"Yes\",i)"
      ],
      "metadata": {
        "id": "yIu-O0d_uxjh"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from torch import nn\n",
        "\n",
        "criterion = nn.CrossEntropyLoss(ignore_index=stoi[PADDING_TOKEN],\n",
        "                                reduction='none')\n",
        "\n",
        "# When computing the loss, we are ignoring cases when the label is the padding token\n",
        "for params in transformer.parameters():\n",
        "    if params.dim() > 1:\n",
        "        nn.init.xavier_uniform_(params)\n",
        "\n",
        "optim = torch.optim.Adam(transformer.parameters(), lr=1e-4)\n",
        "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')"
      ],
      "metadata": {
        "id": "ghinXeWyIeCv"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "transformer.train()\n",
        "transformer.to(device)\n",
        "total_loss = 0\n",
        "num_epochs = 5\n",
        "\n",
        "for epoch in range(num_epochs):\n",
        "    print(f\"Epoch {epoch}\")\n",
        "    iterator = iter(train_loader)\n",
        "    for batch_num, batch in enumerate(iterator):\n",
        "        # print(batch_num)\n",
        "        # print(batch)\n",
        "        transformer.train()\n",
        "        mol_batch = batch\n",
        "        encoder_self_attention_mask, decoder_self_attention_mask, decoder_cross_attention_mask = create_masks(mol_batch,mol_batch)\n",
        "        optim.zero_grad()\n",
        "        mol_predictions = transformer(mol_batch,\n",
        "                                     mol_batch,\n",
        "                                     encoder_self_attention_mask.to(device),\n",
        "                                     decoder_self_attention_mask.to(device),\n",
        "                                     decoder_cross_attention_mask.to(device),\n",
        "                                     enc_start_token=False,\n",
        "                                     enc_end_token=False,\n",
        "                                     dec_start_token=True,\n",
        "                                     dec_end_token=True)\n",
        "        labels = transformer.decoder.sentence_embedding.batch_tokenize(mol_batch, start_token=False, end_token=True)\n",
        "        loss = criterion(\n",
        "            mol_predictions.view(-1, vocab_size).to(device),\n",
        "            labels.view(-1).to(device)\n",
        "        ).to(device)\n",
        "        # print(mol_predictions)\n",
        "        # print(labels)\n",
        "        valid_indices = torch.where(labels.view(-1) == stoi[PADDING_TOKEN], False, True)\n",
        "        loss = loss.sum() / valid_indices.sum()\n",
        "        loss.backward()\n",
        "        optim.step()\n",
        "        #train_losses.append(loss.item())\n",
        "        if batch_num % 100 == 0:\n",
        "            print(f\"Iteration {batch_num} : {loss.item()}\")\n",
        "            print(f\"Input molecule: {batch[0]}\")\n",
        "            # print(f\"Output molecule: {batch[0]}\")\n",
        "            mol_predicted = torch.argmax(mol_predictions[0], axis=1)\n",
        "            predicted_molecule = \"\"\n",
        "            # print(mol_predicted)\n",
        "            for idx in mol_predicted:\n",
        "              if idx == stoi[END_TOKEN]:\n",
        "                break\n",
        "              predicted_molecule += itos[idx.item()]\n",
        "            print(f\"Output Prediction: {predicted_molecule}\")\n",
        "\n",
        "\n",
        "            transformer.eval()\n",
        "            mol_gen = (\"\",)\n",
        "            mol2 = (\"CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C\",)\n",
        "            for word_counter in range(max_sequence_length):\n",
        "                encoder_self_attention_mask, decoder_self_attention_mask, decoder_cross_attention_mask= create_masks(mol2, mol_gen)\n",
        "                predictions = transformer(mol2,\n",
        "                                          mol_gen,\n",
        "                                          encoder_self_attention_mask.to(device),\n",
        "                                          decoder_self_attention_mask.to(device),\n",
        "                                          decoder_cross_attention_mask.to(device),\n",
        "                                          enc_start_token=False,\n",
        "                                          enc_end_token=False,\n",
        "                                          dec_start_token=True,\n",
        "                                          dec_end_token=False)\n",
        "                next_token_prob_distribution = predictions[0][word_counter] # not actual probs\n",
        "                next_token_index = torch.argmax(next_token_prob_distribution).item()\n",
        "                next_token = itos[next_token_index]\n",
        "                mol_gen = (mol_gen[0] + next_token, )\n",
        "                if next_token == END_TOKEN:\n",
        "                  break\n",
        "\n",
        "            print(f\"Evaluation output : {mol_gen}\")\n",
        "            print(\"-------------------------------------------\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "WuCR8nBsI2za",
        "outputId": "58242842-2bd3-44ce-c871-13a0143edfde"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 0\n",
            "Iteration 0 : 5.044445991516113\n",
            "Input molecule: CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C\n",
            "Output Prediction: ooo)rMfoffrNff3f)ffoVo31ooo.oooff.ffdM.o)ooo1MdoooooooGG)f1ddr1oo)o)r).odooo..Moor7r)NoT.)dffro)ofofd7\n",
            "Evaluation output : ('CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC',)\n",
            "-------------------------------------------\n",
            "Iteration 100 : 2.031146287918091\n",
            "Input molecule: C1CCC(CC1)COC2=NC(=NC3=C2NC=N3)N\n",
            "Output Prediction: CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC)COCCCC=C)CCCCC)C)CCCC)CCCCCCCCC))C)CCCCCCCCC)C)OCCCCCCCCCCC)CC\n",
            "Evaluation output : ('CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC',)\n",
            "-------------------------------------------\n",
            "Iteration 200 : 1.0544569492340088\n",
            "Input molecule: C=CCCCCCCCCC=O\n",
            "Output Prediction: CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC\n",
            "Evaluation output : ('CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC',)\n",
            "-------------------------------------------\n",
            "Iteration 300 : 1.8500394821166992\n",
            "Input molecule: CC1=CC(=O)OC1=O\n",
            "Output Prediction: CCCCCCCCCCCCC)C)))))CC))C))CC))C)CCCCC)CCC)C)C))C]CCCCC\n",
            "Evaluation output : ('C1CCCCCCC=C1C1C1C=C1C=CCCCCCCCCCCCCCCCCC<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 400 : 1.5875648260116577\n",
            "Input molecule: COC1=CC(=CC(=C1O)C2=C(C(=CC(=C2)C(=O)O)OC)O)C(=O)O\n",
            "Output Prediction: CCCC=C((CC((CCC))C(=C(CCCC(==C=)CC=C)C)C)=C)C\n",
            "Evaluation output : ('CC1=C(=C(=C1)C(C=C=C=C=C=C=C=C=C=C=C=C=Cl<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 500 : 1.6154252290725708\n",
            "Input molecule: CCCCCCCCCCCCCCCCCC(=O)OC(C)C(=O)OC(C)C(=O)[O-].CCCCCCCCCCCCCCCCCC(=O)OC(C)C(=O)OC(C)C(=O)[O-].[Ca+2]\n",
            "Output Prediction: C11CCCCCCCCCCCCCCCCCCCC)CCCCCCC)C)CCCCCCC)CO)C(CCCCCCCCCCCCCCCCC(CCC)C)CCCOCCO)O)C==C)CC)OO)C)CO())))C\n",
            "Evaluation output : ('C1=C(=C(=C1)C1)C(=C=C=C=C=C=C=C=C=C=C=C=C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 600 : 1.4739153385162354\n",
            "Input molecule: CCN(CC)C(=O)NC1CN(C2CC3=CNC4=CC=CC(=C34)C2=C1)C\n",
            "Output Prediction: CC1CC)CCCCC)CC(=(C=C)C=)C====C3CCCC=C=))C=\n",
            "Evaluation output : ('C1=C(=O)(C(=O)(=O)O)O)O)O)O)O-]<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 700 : 1.4307767152786255\n",
            "Input molecule: CC1=C(C(=CC=C1)C)NC(=O)N=C2CCCN2C\n",
            "Output Prediction: CC1=C(=(=C==C1=C=C===C)C2C=)====\n",
            "Evaluation output : ('C1=C(=C(=C1)C(=O)N(=O)[O-](=O)[O-]<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 800 : 1.4384965896606445\n",
            "Input molecule: CC1=NN2C(=N1)CC3=CC=CC=C3N=C2N4CCN(CC4)C\n",
            "Output Prediction: C11=CCCC(=CC)CC2=CCCC(=CCCCC(C(=CCCC)))CCC..C\n",
            "Evaluation output : ('C1=C(=O)C(=O)C(=O)C(=O)[O-]<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 900 : 1.362093448638916\n",
            "Input molecule: Cl[V]Cl\n",
            "Output Prediction: [[[r]\n",
            "Evaluation output : ('C[O-](C)(C)(C)[O-])[O-])[O-].[O-]<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1000 : 0.9261379837989807\n",
            "Input molecule: CC1=C(C=CC(=C1)O)C(=O)C\n",
            "Output Prediction: C1(=CCC(C(==C1)CCCC=O)O\n",
            "Evaluation output : ('C1=CC(=C(C=C1)C(=O)C(=O)Cl)Cl<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1100 : 0.9598956108093262\n",
            "Input molecule: C1=CC(=CC=C1CCO)Cl\n",
            "Output Prediction: CC=CC==CC=C1)lC)O\n",
            "Evaluation output : ('C1=CC=C(C=C1)C(=O)(=O)(=O)(=O)O)F<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1200 : 1.0099583864212036\n",
            "Input molecule: CCCCCCCCCCCCP(=O)(O)O\n",
            "Output Prediction: CCCCCCCCCCCCCC=O)CC)O\n",
            "Evaluation output : ('C1=CC=C(C=C1)C(=O)(=O)[O-])[N+]<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1300 : 1.2837960720062256\n",
            "Input molecule: CCCCCCCCSSC1=NN=C(S1)SSCCCCCCCC\n",
            "Output Prediction: CCCCCCCCCCCC=CCCCCC2)C((2(CC(CCC)\n",
            "Evaluation output : ('CC(C)C(C(C)(C)C(C)(C)(C)(C)(C)C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1400 : 0.8757039308547974\n",
            "Input molecule: CC(CN1CCOCC1)C(=O)OC\n",
            "Output Prediction: CCCC))CCCCCCCC(=O)CC\n",
            "Evaluation output : ('CC(C)(C)C(=O)C(=O)C(=O)C(=O)OC(=O)O<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1500 : 0.7665489315986633\n",
            "Input molecule: CCCCCOC1=CC=C(C=C1)C(=O)Cl\n",
            "Output Prediction: CCCCCCC(=CC=C(==C1)C(=O)O\n",
            "Evaluation output : ('CC(C)(C)C(=O)OC(=O)OC(=O)OC)C(=O)O<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1600 : 1.0174643993377686\n",
            "Input molecule: C1C(O1)C2=C3C=CC4=CC=CC5=C4C3=C(C=C5)C=C2\n",
            "Output Prediction: C1CCCCCC2=CCC3CC=CC=3CC==CCC==CCC=CC)C=CC\n",
            "Evaluation output : ('CC(C)C(C(=O)O)NC(=O)NC(C(C)C(=O)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1700 : 0.9965190291404724\n",
            "Input molecule: CC1=CC2=C(C=C1OC)C(C(=O)C=C2C(C)C)(C)O\n",
            "Output Prediction: C11=CC==C(C=C1)C(C(C(CO)C(C2)(=)C)CC)O\n",
            "Evaluation output : ('CC(=O)OC(=O)C(=O)C(=O)C)C(C)C)Cl<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1800 : 0.6540476083755493\n",
            "Input molecule: COC1=C(C=C(C=C1)C(=O)C2=CC=CC=N2)OC\n",
            "Output Prediction: CCC1=C(C(C(C(C1)C(=O)C2=CC=CC=C2)O\n",
            "Evaluation output : ('CC(=O)OC(=O)C(=O)N)C(=O)[O-])C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1900 : 0.7370736598968506\n",
            "Input molecule: CC(=O)OCOC(=O)CN(CC(=O)OCOC(=O)C)C1=CC(=CC=C1)OC\n",
            "Output Prediction: CC(=O)OCCC(=O)C(CC((=O)O)(C(=O)C(CC=CCCCCC=C1)O\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)N)[O-].[N+])C(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 2000 : 0.6775765419006348\n",
            "Input molecule: C1C2C3=CC=CC=C3C(N2)(C4=CC=CC=C41)C(=O)N\n",
            "Output Prediction: C1C2C2=CC=CC=CCC(=))CC3=CCCCC=C4C)C(=O)N\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)O)[N+]1=[N+](C)C)O<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 2100 : 0.518058717250824\n",
            "Input molecule: CNC(=C)C(=O)O\n",
            "Output Prediction: CN(S=C)C(=O)O\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)O)O.[C1=[N+](C)C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 2200 : 0.199798583984375\n",
            "Input molecule: C1=CC=C(C=C1)C2=CC=C3C4=CC=CC=C4C=C3O2\n",
            "Output Prediction: C1=CC=C(C=C1)C2=CC=CCC3=CC=CC=C3C=C3)2\n",
            "Evaluation output : ('CC(=O)OC(C(=O)[O-])[CC[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 2300 : 0.3933953642845154\n",
            "Input molecule: C1CC(N2C(=O)C(CCC(=O)N2C1)NC(CCC3=CC=CC=C3)C(=O)O)C(=O)O\n",
            "Output Prediction: C1CC(N2C(=O)C(CCC(=O)N2C1)NC(CCC3=CC=C33C3)O3=O)O)O(=O)O)))))))))))))))))))))))]))))))))))))))])))))))\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 2400 : 0.3912963569164276\n",
            "Input molecule: COC1=CC(=CC(=C1OC)OC)C(=O)N2CCC(C2)(CCN3CCC(CC3)(C4=CC=CC=C4)C(=O)N)C5=CC(=C(C=C5)Cl)Cl\n",
            "Output Prediction: COC1=CC(=CC(=C1OC)OC)C(=O)N2CCC(C2)CCNNCCCC(C44)CC4=CC=C==C5)N(=C)CCC(=C(CCCCC)CC)Cl)Cl\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 2500 : 0.16649314761161804\n",
            "Input molecule: C[N+](C)(C)CC(CO)O.[Cl-]\n",
            "Output Prediction: C[N+](C)(C)CC(CO)O.[Cl-]\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 2600 : 0.19843578338623047\n",
            "Input molecule: C=CC=C.C=CC1=CC=CC=C1.C=CC(=O)O.C=C(CC(=O)O)C(=O)O\n",
            "Output Prediction: C=CC=C.C=CC1=CC=CC=C1.C=CC(=O)O.C=C(CC(=O)O)C(=O)O\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C.N.[Ci](C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 2700 : 0.1859479397535324\n",
            "Input molecule: CCC1=CC(=C(N=C1)C2=NC(C(=O)N2)(C)C(C)C)C(=O)[O-].[NH4+]\n",
            "Output Prediction: CCC1=CC(=C(N=CC)C2=NC(C(=O)N2)(C)C(C)C)C(=O)[O-].[NH4+]\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Epoch 1\n",
            "Iteration 0 : 0.010570447891950607\n",
            "Input molecule: CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C\n",
            "Output Prediction: CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 100 : 0.05227645859122276\n",
            "Input molecule: C1CCC(CC1)COC2=NC(=NC3=C2NC=N3)N\n",
            "Output Prediction: C1CCC(CC1)COC2=NN(=NC3=CNNC=N3)N\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 200 : 0.04968583583831787\n",
            "Input molecule: C=CCCCCCCCCC=O\n",
            "Output Prediction: C=CCCCCCCCCC=O\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 300 : 0.012936045415699482\n",
            "Input molecule: CC1=CC(=O)OC1=O\n",
            "Output Prediction: CC1=CC(=O)OC1=O\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 400 : 0.04987793043255806\n",
            "Input molecule: COC1=CC(=CC(=C1O)C2=C(C(=CC(=C2)C(=O)O)OC)O)C(=O)O\n",
            "Output Prediction: COC1=CC(=CC(=C1O)C2=C(C(=CC(=C2)C(=O)O)OC)O)C(=O)O\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 500 : 0.0874113216996193\n",
            "Input molecule: CCCCCCCCCCCCCCCCCC(=O)OC(C)C(=O)OC(C)C(=O)[O-].CCCCCCCCCCCCCCCCCC(=O)OC(C)C(=O)OC(C)C(=O)[O-].[Ca+2]\n",
            "Output Prediction: CCCCCCCCCCCCCCCCCC(=O)OC(C)C(=O)OC(C)C(=O)OO-].CCCCCCCCCCCCCCCCCC(=O)(C(CCCC=O)CCCC)CC=O)CO-]\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 600 : 0.015525823459029198\n",
            "Input molecule: CCN(CC)C(=O)NC1CN(C2CC3=CNC4=CC=CC(=C34)C2=C1)C\n",
            "Output Prediction: CCN(CC)C(=O)NC1CN(C2CC3=CNC4=CC=CC(=C34)C2=C1)C\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 700 : 0.030057772994041443\n",
            "Input molecule: CC1=C(C(=CC=C1)C)NC(=O)N=C2CCCN2C\n",
            "Output Prediction: CC1=C(C(=CC=C1)C)NC(=O)N=C2CCCC2C\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 800 : 0.032059986144304276\n",
            "Input molecule: CC1=NN2C(=N1)CC3=CC=CC=C3N=C2N4CCN(CC4)C\n",
            "Output Prediction: CC1=NN2C(=N1)CC3=CC=CC=C3N=C2N4CCN(CC4)C\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 900 : 0.04103844612836838\n",
            "Input molecule: Cl[V]Cl\n",
            "Output Prediction: Cl[T]Cl\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1000 : 0.011984049342572689\n",
            "Input molecule: CC1=C(C=CC(=C1)O)C(=O)C\n",
            "Output Prediction: CC1=C(C=CC(=C1)O)C(=O)C\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1100 : 0.025363028049468994\n",
            "Input molecule: C1=CC(=CC=C1CCO)Cl\n",
            "Output Prediction: C1=CC(=CC=C1CCO)Cl\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1200 : 0.0444573312997818\n",
            "Input molecule: CCCCCCCCCCCCP(=O)(O)O\n",
            "Output Prediction: CCCCCCCCCCCCP(=O)(O)O\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1300 : 0.16084174811840057\n",
            "Input molecule: CCCCCCCCSSC1=NN=C(S1)SSCCCCCCCC\n",
            "Output Prediction: CCCCCCCCSSC1=NN=C(S1)SSCCCCCCCC\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1400 : 0.02443353459239006\n",
            "Input molecule: CC(CN1CCOCC1)C(=O)OC\n",
            "Output Prediction: CC(CN1CCOCC1)C(=O)OC\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1500 : 0.004172778222709894\n",
            "Input molecule: CCCCCOC1=CC=C(C=C1)C(=O)Cl\n",
            "Output Prediction: CCCCCOC1=CC=C(C=C1)C(=O)Cl\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1600 : 0.017361173406243324\n",
            "Input molecule: C1C(O1)C2=C3C=CC4=CC=CC5=C4C3=C(C=C5)C=C2\n",
            "Output Prediction: C1C(O1)C2=C3C=CC4=CC=CC5=C4C3=C(C=C5)C=C2\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1700 : 0.05002377927303314\n",
            "Input molecule: CC1=CC2=C(C=C1OC)C(C(=O)C=C2C(C)C)(C)O\n",
            "Output Prediction: CC1=CC2=C(C=C1OC)C(C(=O)C=C2C(C)C)(C)O\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1800 : 0.015272462740540504\n",
            "Input molecule: COC1=C(C=C(C=C1)C(=O)C2=CC=CC=N2)OC\n",
            "Output Prediction: COC1=C(C=C(C=C1)C(=O)C2=CC=CC=N2)OC\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 1900 : 0.0024068718776106834\n",
            "Input molecule: CC(=O)OCOC(=O)CN(CC(=O)OCOC(=O)C)C1=CC(=CC=C1)OC\n",
            "Output Prediction: CC(=O)OCOC(=O)CN(CC(=O)OCOC(=O)C)C1=CC(=CC=C1)OC\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 2000 : 0.013665884733200073\n",
            "Input molecule: C1C2C3=CC=CC=C3C(N2)(C4=CC=CC=C41)C(=O)N\n",
            "Output Prediction: C1C2C3=CC=CC=C3C(N2)(C4=CC=CC=C41)C(=O)N\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 2100 : 0.00430601742118597\n",
            "Input molecule: CNC(=C)C(=O)O\n",
            "Output Prediction: CNC(=C)C(=O)O\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 2200 : 0.010044392198324203\n",
            "Input molecule: C1=CC=C(C=C1)C2=CC=C3C4=CC=CC=C4C=C3O2\n",
            "Output Prediction: C1=CC=C(C=C1)C2=CC=C3C4=CC=CC=C4C=C3O2\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 2300 : 0.028318895027041435\n",
            "Input molecule: C1CC(N2C(=O)C(CCC(=O)N2C1)NC(CCC3=CC=CC=C3)C(=O)O)C(=O)O\n",
            "Output Prediction: C1CC(N2C(=O)C(CCC(=O)N2C1)NC(CCC3=CC=CC=C3)C(=O)O)C(=O)O\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 2400 : 0.02489069662988186\n",
            "Input molecule: COC1=CC(=CC(=C1OC)OC)C(=O)N2CCC(C2)(CCN3CCC(CC3)(C4=CC=CC=C4)C(=O)N)C5=CC(=C(C=C5)Cl)Cl\n",
            "Output Prediction: COC1=CC(=CC(=C1OC)OC)C(=O)N2CCC(C2)(CCN3CCC(CC3)(C4=CC=CC=C4)C(=O)N)C5=CC(=C(C=CC)Cl)Cl\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 2500 : 0.009579474106431007\n",
            "Input molecule: C[N+](C)(C)CC(CO)O.[Cl-]\n",
            "Output Prediction: C[N+](C)(C)CC(CO)O.[Cl-]\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 2600 : 0.05042141675949097\n",
            "Input molecule: C=CC=C.C=CC1=CC=CC=C1.C=CC(=O)O.C=C(CC(=O)O)C(=O)O\n",
            "Output Prediction: C=CC=C.C=CC1=CC=CC=C1.C=CC(=O)O.C=C(CCC=O)O)C(=O)O\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 2700 : 0.023565344512462616\n",
            "Input molecule: CCC1=CC(=C(N=C1)C2=NC(C(=O)N2)(C)C(C)C)C(=O)[O-].[NH4+]\n",
            "Output Prediction: CCC1=CC(=C(N=C1)C2=NC(C(=O)N2)(C)C(C)C)C(=O)[O-].[NH4+]\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Epoch 2\n",
            "Iteration 0 : 0.0032109953463077545\n",
            "Input molecule: CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C\n",
            "Output Prediction: CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n",
            "Iteration 100 : 0.003191170748323202\n",
            "Input molecule: C1CCC(CC1)COC2=NC(=NC3=C2NC=N3)N\n",
            "Output Prediction: C1CCC(CC1)COC2=NC(=NC3=C2NC=N3)N\n",
            "Evaluation output : ('CC(=O)OC(CC(=O)[O-])C[N+](C)(C)C<END>',)\n",
            "-------------------------------------------\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-32-1d5ab2b43fdd>\u001b[0m in \u001b[0;36m<cell line: 6>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m         mol_predictions = transformer(mol_batch,\n\u001b[1;32m     17\u001b[0m                                      \u001b[0mmol_batch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m                                      \u001b[0mencoder_self_attention_mask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m                                      \u001b[0mdecoder_self_attention_mask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m                                      \u001b[0mdecoder_cross_attention_mask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "transformer.eval()\n",
        "def produce_new(new_mol):\n",
        "  new_mol = (new_mol,)\n",
        "  gen_mol = (\"\",)\n",
        "  for word_counter in range(max_sequence_length):\n",
        "    encoder_self_attention_mask, decoder_self_attention_mask, decoder_cross_attention_mask= create_masks(new_mol, gen_mol)\n",
        "    predictions = transformer(new_mol,\n",
        "                              gen_mol,\n",
        "                              encoder_self_attention_mask.to(device),\n",
        "                              decoder_self_attention_mask.to(device),\n",
        "                              decoder_cross_attention_mask.to(device),\n",
        "                              enc_start_token=False,\n",
        "                              enc_end_token=False,\n",
        "                              dec_start_token=True,\n",
        "                              dec_end_token=False)\n",
        "    next_token_prob_distribution = predictions[0][word_counter]\n",
        "    next_token_index = torch.argmax(next_token_prob_distribution).item()\n",
        "    next_token = itos[next_token_index]\n",
        "    gen_mol = (gen_mol[0] + next_token, )\n",
        "    if next_token == END_TOKEN:\n",
        "      break\n",
        "  return gen_mol[0]\n"
      ],
      "metadata": {
        "id": "f03dZpHyJcvQ"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "!git clone https://github.com/rajkumar1501/drug_analog_data.git\n",
        "!pip install rdkit-pypi"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qsogLM0VyVsc",
        "outputId": "fae25da8-b2dc-486b-e3a2-77c28d686679"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'drug_analog_data'...\n",
            "remote: Enumerating objects: 4, done.\u001b[K\n",
            "remote: Counting objects: 100% (4/4), done.\u001b[K\n",
            "remote: Compressing objects: 100% (4/4), done.\u001b[K\n",
            "remote: Total 4 (delta 0), reused 0 (delta 0), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (4/4), 4.48 MiB | 33.03 MiB/s, done.\n",
            "Collecting rdkit-pypi\n",
            "  Downloading rdkit_pypi-2022.9.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (29.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m29.4/29.4 MB\u001b[0m \u001b[31m28.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from rdkit-pypi) (1.25.2)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from rdkit-pypi) (9.4.0)\n",
            "Installing collected packages: rdkit-pypi\n",
            "Successfully installed rdkit-pypi-2022.9.5\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title  Enter SMILES\n",
        "d = input('Enter a chemical SMILES(length < 100): ').strip()\n",
        "print(d)\n",
        "from rdkit.Chem import Draw\n",
        "from rdkit import Chem\n",
        "mol = Chem.MolFromSmiles(d)\n",
        "\n",
        "print(\"Length of the SMILES is \" +str(len(d))+\". The model may not perfom well on SMILES length more than 80.\")\n",
        "Draw.MolToImage(mol)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        },
        "id": "GTB0nMX5y4ng",
        "outputId": "7b8b7a26-c02c-4d71-9aa6-da6b3836ce0c"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Enter a chemical SMILES(length < 100): COC(CC=CC=CCCC=CCC(C)C=CC(CC(O)=O)=CC(O)=O)C(C)=CC=C(C)C(O)=O\n",
            "COC(CC=CC=CCCC=CCC(C)C=CC(CC(O)=O)=CC(O)=O)C(C)=CC=C(C)C(O)=O\n",
            "Length of the SMILES is 61. The model may not perfom well on SMILES length more than 80.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<PIL.PngImagePlugin.PngImageFile image mode=RGB size=300x300>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAASwAAAEsCAIAAAD2HxkiAAAcGUlEQVR4nO3deVSUddsH8GsYdkQjjEXZRAHBR3AH3AnMXJ6lg+KSkrjAMQjLSq0sKXvLUlIzeyJFclf0tPmmaWooihtHyIAYEBBRRBBQkEFmYH7vH3fx8uDysAxzsXw/p9PhwH3/7guZ78y9/BaZEIIAgI8edwEAXR1CCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAGUIIwAwhBGCGEAIwQwgBmCGEAMwQQgBmCCEAM4QQgBlCCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAGUIIwAwhBGCGEAIwQwgBmCGEAMwQQgBmCCEAM4QQgBlCCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAGUIIwAwhBGCGEAIwQwgBmCGEAMwQQgBmCCEAM4QQgBlCCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAGUIIwAwhBGCGEAIwQwgBmCGEAMwQQgBmCCEAM4QQgBlCCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAmT53AW2vro7kcu4ioP25f58+/phMTcncnCIjGQvp1CH85Rc6epRMTMjOjsLCuKuBdmbLFpo1i/72N1q3jjIyyMODq5BOF8LSUsrKIoWC7OwoPp62bCEiCg0lIUgm4y4O2pMbN6hPHyKiPn3oxg2EsCWUSmV2dnZWVlZ2dvbMe/ecT5+mrCwqK/vzx0FBZGHx59cGBlRXR/od+JcF7RszhuLjac4cOn6cVq+m2lq6e5d69tR9ITIhhO6P2kqLFy8+fPhwQUFBffG/jh8/PiGBiKh7d3J1JRcX8vMjpZKefpqsrOjYMYqOZiwY2qn//V9KT6cpU8jKioKCqLqaTp8mIyMdV9EhQzhz5sz9+/cbGhr27dvXzc3N1dX1X56evvb25OZG1tb/sWlqKlVU0OjRFBdHvXrRpElMJUM7dvEirV9PFy9Sbi4FB9P27To+focMYU5Ojkwmc3R0lDfxtueRIzR5MllY0MWL1K9fG1cHj1JTQxUV9MwzVFpKZmZkbEzZ2URELi7MhalU5OpK+fkUEkL795NSSV99pePbeB0yhM0mBM2YQQcOUP/+dOECde/OXVDXk51NR45QZCR9/TX5+NDPP9Mzz5BGQ6WltGwZc21JSeTnR2o1RUTQpk1kYEAnTtCYMTo7ftd4WC+T0bZtNGAAZWbSvHnUFd532qHz5yk2ls6eJSLKzaWQEFqwgHJyuMsiGjmS1q0jISgujl58kdTqsvfeKyoq0tnxu0YIiahbN/r2W+rRg7777taXX3JX0yUNGEBTptDAgUREGs2f32wnb4ivvELBwXT/Pl24cGXOHMekpKCgILVa3dTdCwrop5+ouJiIKCODiKiykm7ebOLeXSaEROTqSjt2bBo/3vmNN06cOMFdTddjbk42Nn9eC/j60urVtHo1jRrFXdZfYmI0vr47bW1fvXmzZ69eiYmJr7/+epN2/OMPio6m3r3prbfo9m2S3uKvXqWjR5t45K5xTdhAVFTU+++///TTT1+6dMnZ2fm/bq9Wq3Nzcw8dOjRq1ChfX18dVNg5aTRUW0uGhqRWk1xOenqkVBIRyWS0eDFNnkxBQdwlUkF+/pBhw+7cuRMaGrpz587q6urY2Nj58+f/x0Y1NZSTQ1lZlJVF2dlUUkJjxtA//0n9+tGpU1RYSDExNGEC3bpFQ4ZQo30fo8uFUAgxbdq0b7/91svLKykpydTUtOFPy8vL09PTMzIycv+Snp7+4MED6afvvvvuBx98wFF157VjB730EnXrRufO0d/+xl0NnTx5cuLEiXV1dREREZs2bTIyMlq3bp1MJlMoFFlZWaOI3j1+nOrq/mOfzz8nHx8aPpy++46MjOjwYfriC0pJoZQUhPCxKisrvb29//jjjylTpsyYMUOhUEg9b7KyspTS23MDcrnc0dGxrq4uPz/f2Ng4MTFx2LBhLGV3WgsW0LZt5OJCFy/SU09xV0Pr1q178803pXfnRq+HYba2l0pKyMmJXF3Jze3PbiFeXhQVRb16UXExrVtHy5bRZ5/Rb7/Rb79RcHBTjtgVQ0hE6enpw4YNMzc3Lykpafh9CwsLZ2dnZ2dnDw+PAQMGODs7u7u7S3+PiIiIzZs329vbJycnW1lZMRXeGT14QGPH0qVL9NxzdPhwexjyMmvWrH379hGRiYmJj4+Pi4uLq6urq6urm5uba58+ZGCg3cN10RDW1NTY2Njcu3dv0qRJQ4cO7d+/v/QP3aNHj8ftolarAwICTp8+PXr06JMnTxpo+y/RpeXn07BhVFaWum7doNde466GiouL7ezs1Gp1WlragAED2vx4okvau3cvEXl5eTVrr6Kiot69exPRa6+91jZ1dV2a48f/x8dHJpP98MMP3LWIr7/+mojGjRunm8N10RA+++yzRPTvf/+7uTueO3fOyMiIiL755pu2KKwrW7t2LRGZm5unp6f/143Ly8svXLiwa9euv//971r/WwwfPpyIdu3apd1mH6crhjAnJ0dPT8/ExKS8vLwFu3/11VdEZGJikpyc3Nx9Hzx40IIjdh0zZ84kIjc3t7t379Z/s6amJicn55dfftmwYUNoaGhAQICzs7OswehQuVz+/vvva6uG3377jYieeuqpqqoqbbX5ZBwhvHpVvPuuWLlSpKUxHF2IFStWEFFISEiLW1i0aBEROTo6lpSUPG4blUolvXRiYmIiIyOll46jo2OLD9oVKJVKe3t76d82IiLiueeec3Jy0tN7RJcSMzOzwYMHBwUFjRw5UiaTGRgYnDp1Sis1hIeHE1FkZKRWWmsKjhszixbR5s2kp0ehobRtm44PXltb6+joWFhYmJSU1OKH72q12s/P7+zZswEBAT///LNcLr9161Z6err0qEN6ppSfn19bW9toR1NT07KyMiOdj1jrKFQqVa9evUpLS/X09DR/dW3T19d3cHBodNdaCqcQYt26dX/88UdcXJy1tXVycrKdnV1rCqiuru7du3d5eXlqaqqXl5c2fqcm0Fnc/19YWOMvdOjbb78lInd391a2U1hYaGtrS0QrVqwQQsybN+/hf1tbW1s/P7/AwMDZs2fPmTPnhRdesLe3Hzp0aEJCgjZ+lU4oPj6eiJydnY2NjYlo7dq1WVlZKpXqcdvHxMQQkZOTU0BAABF5e3u38oR/+/btROTj49OaRpqLI4SLF4uyMlFZKRYu1P3BJ02aREQbNmxofVNnz541NDSUyWT79+/fvHnz2LFjZ86cuWjRopdffnnhwoWTJ092dnZ++FRKT0+vf//+Orve6FgmTJhARKGhoUTUv39/jUbz5O1VKtXo0aOJaNy4cVInxODg4NYUILUWGxvbmkaaS7chLCoSa9aIjAyxZo346CNRWKjTowtRUFAgl8uNjIyecC3XLBs3bqS/etU88smhkZGRh4fHCy+8sGzZsq1bt8bHx7u4uBDRrFmztFJAZ5KbmyvdMJOiuH79+qbsdevWrV69ehFRSEiImZkZEcXExLSsgMzMTJlM1q1bt4qKipa10DK6DeEnnwgiMXWqCAoSR4/q9NBCCCFWrVpFRLNnz9Zimw17sVlYWIwaNSo0NHTNmjXx8fFpaWm1tbWNts/MzJS6BERHR2uxjE7gnXfeIaLAwEC5XG5oaFhcXNzEHZOSkqRTkldeeYWIDAwMTp8+3YICli5dSkRhOr9K0mEINRrh4iKIxMKFf0ZRt+rq6hwdHYno119/1WKzGo3mp59+On36dNOvRn744QeZTCaXy49yvBO1U2r17YCAtT4+80NCWnCmsGnTJiIyMzObO3cuEdnY2Ny8ebNZLdTU1DzzzDNE1IInT62kwxCePCmIhJ2d8PAQRELnHSN++uknIurbt+9/vdLQgZUrVxKRpaVlbm4udy3tww8/CCLh5qYaNOj8uHEXmn/vShpz5OjoKF3X+fr61tTUNH33uLg4IvL09GzucVtPhyGcPVsQiZAQQSRsbMTjb3m1heLiYqmXzCeffKLL4z5OXV3dlClTiGjQoEG4SSOEEFOnCiKxYIEgEs7OovlvlNXV1VJPFz8/vz59+hBReHj44zYuKytLTk7evn378uXLp0+fPnToUD09PZlMFhUV1bpfoyV0FcLSUmFsLPT0RGCgIBLvvNN2h3rw4EFaWlp8fPyaNWukDhY2NjZE1L17dxMTk5SUlLY7dLOUlZX169ePiObMmdPEXYqLixMTE2NjYz/44IM2rU3XbtwQcrkwNBSTJgkisWZNy5rJz8+XTikXLVpkYmIi3ee8d+/epUuX9uzZs2rVqlmzZg0dOrT7o2b6kslkgwYNKioq0u5v1hQ6elj/fUxMn7173S0sDI8do+pqys6mvn1b36wQ4ty5c6WlpdJoQOlZ+c1Hze1hYWGhr69fUlLi4+OTkJDQTh6XZ2Zment7V1RUfPHFF1JHjXo1NTVXr15tOLw4LS2tfvYhmUx27949c3NzjqrbwOrV9N579M9/0uHDJARdv062ti1rqX5U7pIlSzZs2GBmZlZVVfXwZpaWltLQJGn0jPR/Kbe6p6MQDhw4MC0tLSIszCMtLbBfP6tvvtFKs+PHjz9z5kzdf450NjQ0tLOzs7e3t7KyMjU1ValUd+/eJaJdu3aNGDEiJycnODh4u84neH2c7777LjAwUF9f//jx42PHjt2xY8c333zzhLcSFxcXaSRHUFDQjBkzZJ1ggQ2Nhvr2pWvXaMECio2lwEA6eLA17UVHR//444/x8fFeXl4qler+/fv29vb1XW2cnZ0HDBhg29KQtwVdhDApKWnUqFHW1ta2trapqanx8fHTp09vfbOpqamDBw/W09Pz9/fv2bOnqalpXV1dZWXlzZs3s7KyyuoXpSAiIrlcXlVVpVAofH19lUplTEyM9ES4PVi2bNnatWulXle7d++WurbWv5XY2tp2795do9GUlpZev35doVDcv39f2nHq1KmHDh1irV0bamvpwAE6fJjOn6erV+nIEXr++VY2WVdXd/78+dGjR1tbW1+/ft3Q0FArlbYVHZzySl26XnrpJSLq2bOntkYSREREENErr7wi9XVqpHv37sOGDZs9e3ZUVNSePXuSk5PVarUQYvfu3URkYGCQmJiolTJar66u7vnnnyciDw+Pjz/+eN68eYGBgT4+PpaWlo/8k9nY2Li7u0s3Evbv389dfqtduSI+/FBER4vffxerV4u6Oq20Kr3e3n77ba201qbaPIR37941MzOTyWRBQUFE9MYbb2ilWaVSaWFhQUQpKSm///67u7v7v/71rzfffHPLli2nTp168uX1q6++Si16lNR2SkpKrK2tHz63lDrcTJ8+ffny5TExMYmJiffu3ZN2+eKLL4jIzMzsypUrvMW3ilotXnpJ1NaKrCyxcqW2Wr17966pqalMJsvOztZWm22nDUNYVFR06tSpOXPmEJGVlZXUpSgjI0Mrje/YsYOIvL29W7CvWq0eP348Nf9RUps6dOjQ8OHDfXx8pLeShISEW7duPXmXBQsWEJGTk9OdO3d0U6T23bgh6u/0aq+rivQOFRAQoK0G21STQ5iUJAoKhBBCOv+5eFGsXi3WrxfSuWVFhUhOFnv3ivffF7NnfzptWqPJWqSVW3x9fbVV95gxY4ho69atLdv9zp070qOkiIgIbZWke/VPxiZMmPBw/7iOoabmz378t26JZcu01ergwYOJqKOcqzc5hLGx4vJlIYQIDxdVVWLRIqHRiN9/Fx9+KIYMEUQN/9s/diwRPf30097e3sHBwe+99570ih8wYIBWPnm00tH28uXL9Y+SWl8Sl+vXr0tPxlZq71xO1xISxKpV4oMPRFmZVtq7ePEiEVlaWnaUeQyaE8LwcPHhh2LcOJGZKep7uIeFCT8/YWIiPD3FtGnirbdEXFzJhQuNhikUFRVJoy1fffXV1hctzU8eGhraynakc1pjY+OLFy+2viouJ0+e1NfXl8lkBw4c4K6lXZDmPXj99de5C2mqFn0SVlaKl18WQojcXLFqlSgvb0ono/opkuLi4lparRANOtpeunSpNe1IFi9eTEQODg5N77PfDkVHRxORubl5GtOMIe1HZWWl1CGmKbNFtRNNDuHJkyIvTwghpJO3Y8dEVJT48EPRnBNCaRy0sbFxazqqS7OyaqujbU1NzciRI4lo0qRJWmmQi3RH3tXVteEUSV3Qli1biGjs2LHchTSDrkfWN2WKpCfz9/cnos2bN2urpKKiohEjRpw7d05bDbJQKpVDhgwhon/84x91j3/UplKpFArFoUOHoqOjw8LCPD093dzcWnlu0q6MGDGCiHbu3MldSDPoOoT18xH4+/s3/YaeWq3Oz88XDQZfl2npIr4zuXbtWs+ePYlo9erV0nceHisgzd3SkEwms7CwKJBufXdwV65cId3OVqgVDHPMFBYWSvMRLF++/JEbPPzSMTU1NTIyqq2tfeutt6TON7otucOQpn4jIjs7O+nBbCN6enp9+vR57rnnpFWHfvzxR+lsfPDgwUqlkrv81qrvRMVdSPPwrEWRlJTk5+enVqv37dsXFBSUl5e3ffv2+pEQFRUVjbaXyWQODg5Hjx4dO3ZscXHxmTNnRrWfxSXbmRdffHHPnj3S1w+vb9O/f/9G4SwvLx8+fHhOTs7cuXOl28UdRXV1df1yWtnZ2QqF4sKFCxqN5vLly9Jzwo6CbUGYTZs2RUZGduvW7dy5c/fv3284Baj00rG1tTUzMzM1NdVoNBUVFQqFIjMzUwhhaWnZaCklaOTs2bN37twZN27cU01baezKlSsjR46sqqr66quvwsLC2ri6FiosLKwf2CWtIXnt2rX6uUklenp6EydOPHz4MFeRLcO5KtP8+fPj4uJcXFz27du3YcMGAwMDtVpdVlZ27dq1q1ev1tTUNNrewMDAzMxs3759EydOZCm4E9u7d+/s2bMNDAxOnDgh9UZqJ2JiYlasWKFUKlUqVaMfGRoaSp/zlpaWhoaGarX6zp07W7dubVfDlJqCM4TV1dW+vr5S/+OHfyp9HjYaBvbwfQXQlqVLl65fv97GxiY5OVkassju5s2bTk5O0s1eCwsLd3d3BweHbt261dbWVlVVlZWV5eXl5eXlNXz9HD9+XLp/3oHoMx7bxMRk2rRpV65cMTY29vLycv2LNMz5kfcVoO2sXbs2PT392LFj06dPT0hIaMoYvMrKyvqrsiVLljxhdceW2blzZ21t7ZgxY5YuXbp06dLz588nJSU12sbMzKz+NePq6qqL5QS1jXmR0CFDhqSkpOzfv18a6AS8SktLhw8fnpeXFx4eLg1EqKdWqwsKChpekuXm5jb8FDp//ry3t7cWixFCuLm5ZWdnHzlyxMTEZPz48U9YlEKLx2XAck9W0uE62nYFKSkp0vLg0giVhISEyZMn9+vXT1//ESdNxsbGnp6eU6dOnT59+qeffioNm9aWEydOEJG9vX1tba1SqXzyohQdGufpqNTDaN68ee1k2iUgokGDBsXExMydOzciIsLT07Oqqqr+ZqOtra27u7uVlZX096qpqSkpKcnIyJAekR84cGDr1q0ZGRlyLS06L708Fi5cKJfLTUxMpOUDOiW209H79+/37t27oqIiPT3dw8ODpQZ4nPDw8C+//NLW1nbJkiUKhUKpVN6+fTsnJ6egoODhjXv06OHg4JCZmalWq1etWhUVFdX6AkpLS+3s7FQqVV5enoODQ+sbbNe4PoKl97kxY8ZwFQBPoFKphgwZ8vC1loGBgbOzc0BAQGRkZExMzC+//JKTkyPNaH7ixAlpRNXBgwdbX8Bnn31GRFOmTGl9U+0fWwiljrY7duzgKgCeLDU11d/f383NLTw8/PPPP//5559zcnKe3N33008/pSYvOv9k0k3O77//vpXtdAg8IZSuInr06NGxOtrCk2k0mhkzZhCRm5tb/YRULXDmzBkisrGx6ax3Yhrhubf79ddfE9HcuXOlG3HQOchkstjY2IEDByoUCmmxzpa1I12qzJ8//5FLPnZCus99w9kKdX90aGt5eXnSjKkfffRRC3bvWLMVagXDJ+HBgwfLy8tHjBgxaNAg3R8d2pqTk9PevXvlcvnKlSuPHDnS3N13796tVCr9/f2l1XK6AoYQSicb0hB76JQmTJgQFRWl0WhefPHFnJycZu27detW6mIvD10/J1QoFO7u7mZmZoWFhZ1nUSF4iBAiKCjo4MGDnp6eSUlJj+sJLIQoKCio74CakpKSmJhobm5++/btrtOFQ9c9ZtLS0rp37x4UFIQEdm4ymSwuLk7qT7No0SJpnHF5eXn9Sm9SB9SG69tILC0tKyoqUlNTtdsTtT1j6DFTVVVVVVVlZWWl4+OC7tUvwGhlZVVXV1daWvrwNtbW1v37969fJ/D777/fvn27ra1tcnKyNA1Kp8c8igI6vY0bN7722mvSy8zIyKhv3771A0Q9PDwGDhzYaABUbW3thAkTEhISRo4c+euvv7b3Vc20ASGENpeVlXXq1KmpU6c2ccx7cXHx0KFDb9y4ERkZuXHjxrYujx1CCO1RSkrKqFGjqqurt23bFhISwl1O20IIoZ3auXNncHCwsbFxYmLisGHDuMtpQx18SDJ0XnPnzg0LC3vw4EFgYGDnnl8Pn4TQfqnVan9//8TExGefffbo0aOPHN3fCeCTENovAwOD+Pj43r17nzx58u233+Yup63gkxDau3Pnzo0fP16lUu3bt08aKtXJ4JMQ2jtfX19poH1ISMjly5e5y9E+hBA6gPDw8Hnz5gkhcnNzuWvRPpyOQsfw4MEDhULh5eXFXYj2IYQAzHA6CsAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAGUIIwAwhBGCGEAIwQwgBmCGEAMwQQgBmCCEAM4QQgBlCCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAGUIIwAwhBGCGEAIwQwgBmCGEAMwQQgBmCCEAM4QQgBlCCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAGUIIwAwhBGCGEAIwQwgBmCGEAMwQQgBmCCEAM4QQgBlCCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAGUIIwAwhBGCGEAIwQwgBmCGEAMwQQgBmCCEAM4QQgBlCCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCA2f8BTLeQ6I2w/TkAAAAASUVORK5CYII=\n"
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "out=produce_new(d)\n",
        "out=out[:-5]\n",
        "print(\"The output SMILES is:\", out)\n",
        "\n",
        "mol = Chem.MolFromSmiles(out)\n",
        "Draw.MolToImage(mol)"
      ],
      "metadata": {
        "id": "Hn7krJJ2Jd49",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 335
        },
        "outputId": "0339ea1e-d169-4cef-e3ca-4daf6551f764"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The output SMILES is: COC(CC=CC=CCCC=CCC(C)C=CC(CC(O)=O)=CC(O)=O)C(C)=CC=C(C)C(O)=O\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<PIL.PngImagePlugin.PngImageFile image mode=RGB size=300x300>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAASwAAAEsCAIAAAD2HxkiAAAcGUlEQVR4nO3deVSUddsH8GsYdkQjjEXZRAHBR3AH3AnMXJ6lg+KSkrjAMQjLSq0sKXvLUlIzeyJFclf0tPmmaWooihtHyIAYEBBRRBBQkEFmYH7vH3fx8uDysAxzsXw/p9PhwH3/7guZ78y9/BaZEIIAgI8edwEAXR1CCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAGUIIwAwhBGCGEAIwQwgBmCGEAMwQQgBmCCEAM4QQgBlCCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAGUIIwAwhBGCGEAIwQwgBmCGEAMwQQgBmCCEAM4QQgBlCCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAGUIIwAwhBGCGEAIwQwgBmCGEAMwQQgBmCCEAM4QQgBlCCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAGUIIwAwhBGCGEAIwQwgBmCGEAMwQQgBmCCEAM4QQgBlCCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAmT53AW2vro7kcu4ioP25f58+/phMTcncnCIjGQvp1CH85Rc6epRMTMjOjsLCuKuBdmbLFpo1i/72N1q3jjIyyMODq5BOF8LSUsrKIoWC7OwoPp62bCEiCg0lIUgm4y4O2pMbN6hPHyKiPn3oxg2EsCWUSmV2dnZWVlZ2dvbMe/ecT5+mrCwqK/vzx0FBZGHx59cGBlRXR/od+JcF7RszhuLjac4cOn6cVq+m2lq6e5d69tR9ITIhhO6P2kqLFy8+fPhwQUFBffG/jh8/PiGBiKh7d3J1JRcX8vMjpZKefpqsrOjYMYqOZiwY2qn//V9KT6cpU8jKioKCqLqaTp8mIyMdV9EhQzhz5sz9+/cbGhr27dvXzc3N1dX1X56evvb25OZG1tb/sWlqKlVU0OjRFBdHvXrRpElMJUM7dvEirV9PFy9Sbi4FB9P27To+focMYU5Ojkwmc3R0lDfxtueRIzR5MllY0MWL1K9fG1cHj1JTQxUV9MwzVFpKZmZkbEzZ2URELi7MhalU5OpK+fkUEkL795NSSV99pePbeB0yhM0mBM2YQQcOUP/+dOECde/OXVDXk51NR45QZCR9/TX5+NDPP9Mzz5BGQ6WltGwZc21JSeTnR2o1RUTQpk1kYEAnTtCYMTo7ftd4WC+T0bZtNGAAZWbSvHnUFd532qHz5yk2ls6eJSLKzaWQEFqwgHJyuMsiGjmS1q0jISgujl58kdTqsvfeKyoq0tnxu0YIiahbN/r2W+rRg7777taXX3JX0yUNGEBTptDAgUREGs2f32wnb4ivvELBwXT/Pl24cGXOHMekpKCgILVa3dTdCwrop5+ouJiIKCODiKiykm7ebOLeXSaEROTqSjt2bBo/3vmNN06cOMFdTddjbk42Nn9eC/j60urVtHo1jRrFXdZfYmI0vr47bW1fvXmzZ69eiYmJr7/+epN2/OMPio6m3r3prbfo9m2S3uKvXqWjR5t45K5xTdhAVFTU+++///TTT1+6dMnZ2fm/bq9Wq3Nzcw8dOjRq1ChfX18dVNg5aTRUW0uGhqRWk1xOenqkVBIRyWS0eDFNnkxBQdwlUkF+/pBhw+7cuRMaGrpz587q6urY2Nj58+f/x0Y1NZSTQ1lZlJVF2dlUUkJjxtA//0n9+tGpU1RYSDExNGEC3bpFQ4ZQo30fo8uFUAgxbdq0b7/91svLKykpydTUtOFPy8vL09PTMzIycv+Snp7+4MED6afvvvvuBx98wFF157VjB730EnXrRufO0d/+xl0NnTx5cuLEiXV1dREREZs2bTIyMlq3bp1MJlMoFFlZWaOI3j1+nOrq/mOfzz8nHx8aPpy++46MjOjwYfriC0pJoZQUhPCxKisrvb29//jjjylTpsyYMUOhUEg9b7KyspTS23MDcrnc0dGxrq4uPz/f2Ng4MTFx2LBhLGV3WgsW0LZt5OJCFy/SU09xV0Pr1q178803pXfnRq+HYba2l0pKyMmJXF3Jze3PbiFeXhQVRb16UXExrVtHy5bRZ5/Rb7/Rb79RcHBTjtgVQ0hE6enpw4YNMzc3Lykpafh9CwsLZ2dnZ2dnDw+PAQMGODs7u7u7S3+PiIiIzZs329vbJycnW1lZMRXeGT14QGPH0qVL9NxzdPhwexjyMmvWrH379hGRiYmJj4+Pi4uLq6urq6urm5uba58+ZGCg3cN10RDW1NTY2Njcu3dv0qRJQ4cO7d+/v/QP3aNHj8ftolarAwICTp8+PXr06JMnTxpo+y/RpeXn07BhVFaWum7doNde466GiouL7ezs1Gp1WlragAED2vx4okvau3cvEXl5eTVrr6Kiot69exPRa6+91jZ1dV2a48f/x8dHJpP98MMP3LWIr7/+mojGjRunm8N10RA+++yzRPTvf/+7uTueO3fOyMiIiL755pu2KKwrW7t2LRGZm5unp6f/143Ly8svXLiwa9euv//971r/WwwfPpyIdu3apd1mH6crhjAnJ0dPT8/ExKS8vLwFu3/11VdEZGJikpyc3Nx9Hzx40IIjdh0zZ84kIjc3t7t379Z/s6amJicn55dfftmwYUNoaGhAQICzs7OswehQuVz+/vvva6uG3377jYieeuqpqqoqbbX5ZBwhvHpVvPuuWLlSpKUxHF2IFStWEFFISEiLW1i0aBEROTo6lpSUPG4blUolvXRiYmIiIyOll46jo2OLD9oVKJVKe3t76d82IiLiueeec3Jy0tN7RJcSMzOzwYMHBwUFjRw5UiaTGRgYnDp1Sis1hIeHE1FkZKRWWmsKjhszixbR5s2kp0ehobRtm44PXltb6+joWFhYmJSU1OKH72q12s/P7+zZswEBAT///LNcLr9161Z6err0qEN6ppSfn19bW9toR1NT07KyMiOdj1jrKFQqVa9evUpLS/X09DR/dW3T19d3cHBodNdaCqcQYt26dX/88UdcXJy1tXVycrKdnV1rCqiuru7du3d5eXlqaqqXl5c2fqcm0Fnc/19YWOMvdOjbb78lInd391a2U1hYaGtrS0QrVqwQQsybN+/hf1tbW1s/P7/AwMDZs2fPmTPnhRdesLe3Hzp0aEJCgjZ+lU4oPj6eiJydnY2NjYlo7dq1WVlZKpXqcdvHxMQQkZOTU0BAABF5e3u38oR/+/btROTj49OaRpqLI4SLF4uyMlFZKRYu1P3BJ02aREQbNmxofVNnz541NDSUyWT79+/fvHnz2LFjZ86cuWjRopdffnnhwoWTJ092dnZ++FRKT0+vf//+Orve6FgmTJhARKGhoUTUv39/jUbz5O1VKtXo0aOJaNy4cVInxODg4NYUILUWGxvbmkaaS7chLCoSa9aIjAyxZo346CNRWKjTowtRUFAgl8uNjIyecC3XLBs3bqS/etU88smhkZGRh4fHCy+8sGzZsq1bt8bHx7u4uBDRrFmztFJAZ5KbmyvdMJOiuH79+qbsdevWrV69ehFRSEiImZkZEcXExLSsgMzMTJlM1q1bt4qKipa10DK6DeEnnwgiMXWqCAoSR4/q9NBCCCFWrVpFRLNnz9Zimw17sVlYWIwaNSo0NHTNmjXx8fFpaWm1tbWNts/MzJS6BERHR2uxjE7gnXfeIaLAwEC5XG5oaFhcXNzEHZOSkqRTkldeeYWIDAwMTp8+3YICli5dSkRhOr9K0mEINRrh4iKIxMKFf0ZRt+rq6hwdHYno119/1WKzGo3mp59+On36dNOvRn744QeZTCaXy49yvBO1U2r17YCAtT4+80NCWnCmsGnTJiIyMzObO3cuEdnY2Ny8ebNZLdTU1DzzzDNE1IInT62kwxCePCmIhJ2d8PAQRELnHSN++uknIurbt+9/vdLQgZUrVxKRpaVlbm4udy3tww8/CCLh5qYaNOj8uHEXmn/vShpz5OjoKF3X+fr61tTUNH33uLg4IvL09GzucVtPhyGcPVsQiZAQQSRsbMTjb3m1heLiYqmXzCeffKLL4z5OXV3dlClTiGjQoEG4SSOEEFOnCiKxYIEgEs7OovlvlNXV1VJPFz8/vz59+hBReHj44zYuKytLTk7evn378uXLp0+fPnToUD09PZlMFhUV1bpfoyV0FcLSUmFsLPT0RGCgIBLvvNN2h3rw4EFaWlp8fPyaNWukDhY2NjZE1L17dxMTk5SUlLY7dLOUlZX169ePiObMmdPEXYqLixMTE2NjYz/44IM2rU3XbtwQcrkwNBSTJgkisWZNy5rJz8+XTikXLVpkYmIi3ee8d+/epUuX9uzZs2rVqlmzZg0dOrT7o2b6kslkgwYNKioq0u5v1hQ6elj/fUxMn7173S0sDI8do+pqys6mvn1b36wQ4ty5c6WlpdJoQOlZ+c1Hze1hYWGhr69fUlLi4+OTkJDQTh6XZ2Zment7V1RUfPHFF1JHjXo1NTVXr15tOLw4LS2tfvYhmUx27949c3NzjqrbwOrV9N579M9/0uHDJARdv062ti1rqX5U7pIlSzZs2GBmZlZVVfXwZpaWltLQJGn0jPR/Kbe6p6MQDhw4MC0tLSIszCMtLbBfP6tvvtFKs+PHjz9z5kzdf450NjQ0tLOzs7e3t7KyMjU1ValUd+/eJaJdu3aNGDEiJycnODh4u84neH2c7777LjAwUF9f//jx42PHjt2xY8c333zzhLcSFxcXaSRHUFDQjBkzZJ1ggQ2Nhvr2pWvXaMECio2lwEA6eLA17UVHR//444/x8fFeXl4qler+/fv29vb1XW2cnZ0HDBhg29KQtwVdhDApKWnUqFHW1ta2trapqanx8fHTp09vfbOpqamDBw/W09Pz9/fv2bOnqalpXV1dZWXlzZs3s7KyyuoXpSAiIrlcXlVVpVAofH19lUplTEyM9ES4PVi2bNnatWulXle7d++WurbWv5XY2tp2795do9GUlpZev35doVDcv39f2nHq1KmHDh1irV0bamvpwAE6fJjOn6erV+nIEXr++VY2WVdXd/78+dGjR1tbW1+/ft3Q0FArlbYVHZzySl26XnrpJSLq2bOntkYSREREENErr7wi9XVqpHv37sOGDZs9e3ZUVNSePXuSk5PVarUQYvfu3URkYGCQmJiolTJar66u7vnnnyciDw+Pjz/+eN68eYGBgT4+PpaWlo/8k9nY2Li7u0s3Evbv389dfqtduSI+/FBER4vffxerV4u6Oq20Kr3e3n77ba201qbaPIR37941MzOTyWRBQUFE9MYbb2ilWaVSaWFhQUQpKSm///67u7v7v/71rzfffHPLli2nTp168uX1q6++Si16lNR2SkpKrK2tHz63lDrcTJ8+ffny5TExMYmJiffu3ZN2+eKLL4jIzMzsypUrvMW3ilotXnpJ1NaKrCyxcqW2Wr17966pqalMJsvOztZWm22nDUNYVFR06tSpOXPmEJGVlZXUpSgjI0Mrje/YsYOIvL29W7CvWq0eP348Nf9RUps6dOjQ8OHDfXx8pLeShISEW7duPXmXBQsWEJGTk9OdO3d0U6T23bgh6u/0aq+rivQOFRAQoK0G21STQ5iUJAoKhBBCOv+5eFGsXi3WrxfSuWVFhUhOFnv3ivffF7NnfzptWqPJWqSVW3x9fbVV95gxY4ho69atLdv9zp070qOkiIgIbZWke/VPxiZMmPBw/7iOoabmz378t26JZcu01ergwYOJqKOcqzc5hLGx4vJlIYQIDxdVVWLRIqHRiN9/Fx9+KIYMEUQN/9s/diwRPf30097e3sHBwe+99570ih8wYIBWPnm00tH28uXL9Y+SWl8Sl+vXr0tPxlZq71xO1xISxKpV4oMPRFmZVtq7ePEiEVlaWnaUeQyaE8LwcPHhh2LcOJGZKep7uIeFCT8/YWIiPD3FtGnirbdEXFzJhQuNhikUFRVJoy1fffXV1hctzU8eGhraynakc1pjY+OLFy+2viouJ0+e1NfXl8lkBw4c4K6lXZDmPXj99de5C2mqFn0SVlaKl18WQojcXLFqlSgvb0ono/opkuLi4lparRANOtpeunSpNe1IFi9eTEQODg5N77PfDkVHRxORubl5GtOMIe1HZWWl1CGmKbNFtRNNDuHJkyIvTwghpJO3Y8dEVJT48EPRnBNCaRy0sbFxazqqS7OyaqujbU1NzciRI4lo0qRJWmmQi3RH3tXVteEUSV3Qli1biGjs2LHchTSDrkfWN2WKpCfz9/cnos2bN2urpKKiohEjRpw7d05bDbJQKpVDhgwhon/84x91j3/UplKpFArFoUOHoqOjw8LCPD093dzcWnlu0q6MGDGCiHbu3MldSDPoOoT18xH4+/s3/YaeWq3Oz88XDQZfl2npIr4zuXbtWs+ePYlo9erV0nceHisgzd3SkEwms7CwKJBufXdwV65cId3OVqgVDHPMFBYWSvMRLF++/JEbPPzSMTU1NTIyqq2tfeutt6TON7otucOQpn4jIjs7O+nBbCN6enp9+vR57rnnpFWHfvzxR+lsfPDgwUqlkrv81qrvRMVdSPPwrEWRlJTk5+enVqv37dsXFBSUl5e3ffv2+pEQFRUVjbaXyWQODg5Hjx4dO3ZscXHxmTNnRrWfxSXbmRdffHHPnj3S1w+vb9O/f/9G4SwvLx8+fHhOTs7cuXOl28UdRXV1df1yWtnZ2QqF4sKFCxqN5vLly9Jzwo6CbUGYTZs2RUZGduvW7dy5c/fv3284Baj00rG1tTUzMzM1NdVoNBUVFQqFIjMzUwhhaWnZaCklaOTs2bN37twZN27cU01baezKlSsjR46sqqr66quvwsLC2ri6FiosLKwf2CWtIXnt2rX6uUklenp6EydOPHz4MFeRLcO5KtP8+fPj4uJcXFz27du3YcMGAwMDtVpdVlZ27dq1q1ev1tTUNNrewMDAzMxs3759EydOZCm4E9u7d+/s2bMNDAxOnDgh9UZqJ2JiYlasWKFUKlUqVaMfGRoaSp/zlpaWhoaGarX6zp07W7dubVfDlJqCM4TV1dW+vr5S/+OHfyp9HjYaBvbwfQXQlqVLl65fv97GxiY5OVkassju5s2bTk5O0s1eCwsLd3d3BweHbt261dbWVlVVlZWV5eXl5eXlNXz9HD9+XLp/3oHoMx7bxMRk2rRpV65cMTY29vLycv2LNMz5kfcVoO2sXbs2PT392LFj06dPT0hIaMoYvMrKyvqrsiVLljxhdceW2blzZ21t7ZgxY5YuXbp06dLz588nJSU12sbMzKz+NePq6qqL5QS1jXmR0CFDhqSkpOzfv18a6AS8SktLhw8fnpeXFx4eLg1EqKdWqwsKChpekuXm5jb8FDp//ry3t7cWixFCuLm5ZWdnHzlyxMTEZPz48U9YlEKLx2XAck9W0uE62nYFKSkp0vLg0giVhISEyZMn9+vXT1//ESdNxsbGnp6eU6dOnT59+qeffioNm9aWEydOEJG9vX1tba1SqXzyohQdGufpqNTDaN68ee1k2iUgokGDBsXExMydOzciIsLT07Oqqqr+ZqOtra27u7uVlZX096qpqSkpKcnIyJAekR84cGDr1q0ZGRlyLS06L708Fi5cKJfLTUxMpOUDOiW209H79+/37t27oqIiPT3dw8ODpQZ4nPDw8C+//NLW1nbJkiUKhUKpVN6+fTsnJ6egoODhjXv06OHg4JCZmalWq1etWhUVFdX6AkpLS+3s7FQqVV5enoODQ+sbbNe4PoKl97kxY8ZwFQBPoFKphgwZ8vC1loGBgbOzc0BAQGRkZExMzC+//JKTkyPNaH7ixAlpRNXBgwdbX8Bnn31GRFOmTGl9U+0fWwiljrY7duzgKgCeLDU11d/f383NLTw8/PPPP//5559zcnKe3N33008/pSYvOv9k0k3O77//vpXtdAg8IZSuInr06NGxOtrCk2k0mhkzZhCRm5tb/YRULXDmzBkisrGx6ax3Yhrhubf79ddfE9HcuXOlG3HQOchkstjY2IEDByoUCmmxzpa1I12qzJ8//5FLPnZCus99w9kKdX90aGt5eXnSjKkfffRRC3bvWLMVagXDJ+HBgwfLy8tHjBgxaNAg3R8d2pqTk9PevXvlcvnKlSuPHDnS3N13796tVCr9/f2l1XK6AoYQSicb0hB76JQmTJgQFRWl0WhefPHFnJycZu27detW6mIvD10/J1QoFO7u7mZmZoWFhZ1nUSF4iBAiKCjo4MGDnp6eSUlJj+sJLIQoKCio74CakpKSmJhobm5++/btrtOFQ9c9ZtLS0rp37x4UFIQEdm4ymSwuLk7qT7No0SJpnHF5eXn9Sm9SB9SG69tILC0tKyoqUlNTtdsTtT1j6DFTVVVVVVVlZWWl4+OC7tUvwGhlZVVXV1daWvrwNtbW1v37969fJ/D777/fvn27ra1tcnKyNA1Kp8c8igI6vY0bN7722mvSy8zIyKhv3771A0Q9PDwGDhzYaABUbW3thAkTEhISRo4c+euvv7b3Vc20ASGENpeVlXXq1KmpU6c2ccx7cXHx0KFDb9y4ERkZuXHjxrYujx1CCO1RSkrKqFGjqqurt23bFhISwl1O20IIoZ3auXNncHCwsbFxYmLisGHDuMtpQx18SDJ0XnPnzg0LC3vw4EFgYGDnnl8Pn4TQfqnVan9//8TExGefffbo0aOPHN3fCeCTENovAwOD+Pj43r17nzx58u233+Yup63gkxDau3Pnzo0fP16lUu3bt08aKtXJ4JMQ2jtfX19poH1ISMjly5e5y9E+hBA6gPDw8Hnz5gkhcnNzuWvRPpyOQsfw4MEDhULh5eXFXYj2IYQAzHA6CsAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAGUIIwAwhBGCGEAIwQwgBmCGEAMwQQgBmCCEAM4QQgBlCCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAGUIIwAwhBGCGEAIwQwgBmCGEAMwQQgBmCCEAM4QQgBlCCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAGUIIwAwhBGCGEAIwQwgBmCGEAMwQQgBmCCEAM4QQgBlCCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCAGUIIwAwhBGCGEAIwQwgBmCGEAMwQQgBmCCEAM4QQgBlCCMAMIQRghhACMEMIAZghhADMEEIAZgghADOEEIAZQgjADCEEYIYQAjBDCAGYIYQAzBBCAGYIIQAzhBCA2f8BTLeQ6I2w/TkAAAAASUVORK5CYII=\n"
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "xun9yVHFzcrw"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}