Keras_Final_Project.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Keras-Final-Project.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mkh2097/CI_Text_Recognition/blob/main/Keras_Final_Project.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CI-OGo3BWCpr"
      },
      "source": [
        "import sys\n",
        "import os\n",
        "import re\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "from keras.preprocessing import sequence\n",
        "from keras.models import Sequential\n",
        "from keras.layers.core import Dense, Dropout, Activation\n",
        "from keras.layers.embeddings import Embedding\n",
        "from keras.layers.recurrent import GRU\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.layers.convolutional import Convolution1D\n",
        "from keras import backend as K"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yXKpdtgLXQ-i",
        "outputId": "85f1fa2a-21f6-4c15-984f-239e7269fcfe"
      },
      "source": [
        "!pip install hazm\n",
        "lineList = list()\n",
        "with open('stop_words.txt') as f:\n",
        "  for line in f:\n",
        "    lineList.append(line)\n",
        "\n",
        "lineList = [x.strip() for x in lineList] "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: hazm in /usr/local/lib/python3.6/dist-packages (0.7.0)\n",
            "Requirement already satisfied: nltk==3.3 in /usr/local/lib/python3.6/dist-packages (from hazm) (3.3)\n",
            "Requirement already satisfied: libwapiti>=0.2.1; platform_system != \"Windows\" in /usr/local/lib/python3.6/dist-packages (from hazm) (0.2.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk==3.3->hazm) (1.15.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tEENw_wKWPOk",
        "outputId": "8330b152-cefb-4ce4-9dac-3614e6d1b943"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aKY8TUo2WRQy"
      },
      "source": [
        "df = pd.read_csv('/content/drive/MyDrive/train.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sO_JSoWXYEvd"
      },
      "source": [
        "from __future__ import unicode_literals\n",
        "from hazm import *\n",
        "import re, string\n",
        "\n",
        "def remove_stopwords(text):\n",
        "    text = [word for word in text.split() if word not in lineList]\n",
        "    return \" \".join(text)\n",
        "\n",
        "normalizer = Normalizer()\n",
        "\n",
        "\n",
        "special = re.compile(r'\\W')\n",
        "single = re.compile(r'\\s+', flags=re.I)\n",
        "\n",
        "number = re.compile(r\"[-+]?[0-9]+\")\n",
        "pnumber = re.compile(r\"[-+][\\u06F0-\\u06F90-9]+\")\n",
        "url = re.compile(r\"https?://\\S+|www\\.\\S+\")\n",
        "html = re.compile(r\"<.*?>\")\n",
        "emoji_pattern = re.compile(\n",
        "    \"[\"\n",
        "    u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
        "    u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
        "    u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
        "    u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
        "    u\"\\U00002702-\\U000027B0\"\n",
        "    u\"\\U000024C2-\\U0001F251\"\n",
        "    \"]+\",\n",
        "    flags=re.UNICODE,\n",
        ")\n",
        "\n",
        "\n",
        "df[\"Text\"] = df.Text.map(remove_stopwords)\n",
        "df[\"Text\"] = df.Text.map(lambda x: url.sub(r\" \",x))\n",
        "df[\"Text\"] = df.Text.map(lambda x: html.sub(r\" \",x))\n",
        "df[\"Text\"] = df.Text.map(lambda x: emoji_pattern.sub(r\" \",x))\n",
        "df[\"Text\"] = df.Text.map(lambda x: number.sub(r\" \",x))\n",
        "df[\"Text\"] = df.Text.map(lambda x: pnumber.sub(r\" \",x))\n",
        "df[\"Text\"] = df.Text.map(lambda x: x.translate(str.maketrans(\" \", \" \", string.punctuation)))\n",
        "df[\"Text\"] = df.Text.map(lambda x: special.sub(r\" \",x))\n",
        "df[\"Text\"] = df.Text.map(lambda x: single.sub(r\" \", x))\n",
        "df[\"Text\"] = df.Text.map(lambda x: normalizer.normalize(x))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "38dRn5pZWRwr"
      },
      "source": [
        "df.Category = df.Category.astype('category')\n",
        "\n",
        "nb_classes = df.Category.cat.codes.max() + 1\n",
        "\n",
        "df[\"CategoryCodes\"] = df.Category.cat.codes\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OEN0SDSTWVxj"
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(df.Text, df.CategoryCodes, test_size=0.1, random_state=42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XrlvX9i3WWaU"
      },
      "source": [
        "#best 20000 1000\n",
        "max_features = 20000\n",
        "maxlen = 1000"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DRQo0at0WbAb"
      },
      "source": [
        "tokenizer = Tokenizer(num_words=max_features)\n",
        "tokenizer.fit_on_texts(X_train)\n",
        "sequences_train = tokenizer.texts_to_sequences(X_train)\n",
        "sequences_test = tokenizer.texts_to_sequences(X_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HOdXrPBht5xp"
      },
      "source": [
        "# max_features =  len(tokenizer.word_index) + 1\n",
        "# maxlen =  max([len(s.split()) for s in df.Text])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o5NIDqpsWc4G",
        "outputId": "0bc4d904-0431-49da-c789-10d72894d2d1"
      },
      "source": [
        "X_train = sequence.pad_sequences(sequences_train, maxlen=maxlen, padding='post')\n",
        "X_test = sequence.pad_sequences(sequences_test, maxlen=maxlen, padding='post')\n",
        "\n",
        "\n",
        "# Y_train = np_utils.to_categorical(y_train, nb_classes)\n",
        "# Y_test = np_utils.to_categorical(y_test, nb_classes)\n",
        "\n",
        "\n",
        "print('X_train shape:', X_train.shape)\n",
        "print('X_test shape:', X_test.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "X_train shape: (135086, 1000)\n",
            "X_test shape: (15010, 1000)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vpglaYQuWdXk"
      },
      "source": [
        "#best = 32\n",
        "batch_size = 32"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 666
        },
        "id": "peK9HngIWiZD",
        "outputId": "c83378e6-a524-4397-8ced-af61a41a681c"
      },
      "source": [
        "#best 256 512 nodropout\n",
        "from keras.callbacks import ModelCheckpoint\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Embedding(max_features, 300, input_length=maxlen))\n",
        "model.add(GRU(512, return_sequences=False))\n",
        "# model.add(Dropout(0.5))\n",
        "model.add(Dense(nb_classes, activation='sigmoid'))\n",
        "model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "\n",
        "filepath=\"weights-improvement-{epoch:02d}-{val_accuracy:.2f}.hdf5\"\n",
        "checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')\n",
        "callbacks_list = [checkpoint]\n",
        "\n",
        "\n",
        "model.fit(X_train, y_train, batch_size=batch_size, epochs=5,\n",
        "          validation_data=(X_test, y_test), callbacks=callbacks_list)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "4222/4222 [==============================] - 2050s 485ms/step - loss: 1.4604 - accuracy: 0.5558 - val_loss: 0.5536 - val_accuracy: 0.8249\n",
            "\n",
            "Epoch 00001: val_accuracy improved from -inf to 0.82485, saving model to weights-improvement-01-0.82.hdf5\n",
            "Epoch 2/5\n",
            "4222/4222 [==============================] - 2049s 485ms/step - loss: 0.4524 - accuracy: 0.8599 - val_loss: 0.5242 - val_accuracy: 0.8353\n",
            "\n",
            "Epoch 00002: val_accuracy improved from 0.82485 to 0.83531, saving model to weights-improvement-02-0.84.hdf5\n",
            "Epoch 3/5\n",
            "4222/4222 [==============================] - 2045s 484ms/step - loss: 0.2855 - accuracy: 0.9118 - val_loss: 0.5573 - val_accuracy: 0.8331\n",
            "\n",
            "Epoch 00003: val_accuracy did not improve from 0.83531\n",
            "Epoch 4/5\n",
            "4222/4222 [==============================] - 2042s 484ms/step - loss: 0.1649 - accuracy: 0.9496 - val_loss: 0.6462 - val_accuracy: 0.8282\n",
            "\n",
            "Epoch 00004: val_accuracy did not improve from 0.83531\n",
            "Epoch 5/5\n",
            "1474/4222 [=========>....................] - ETA: 21:29 - loss: 0.0910 - accuracy: 0.9733"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-76-bc3a20759ee3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m model.fit(X_train, y_train, batch_size=batch_size, epochs=5,\n\u001b[0;32m---> 16\u001b[0;31m           validation_data=(X_test, y_test), callbacks=callbacks_list)\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1098\u001b[0m                 _r=1):\n\u001b[1;32m   1099\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1100\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1101\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1102\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    826\u001b[0m     \u001b[0mtracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    827\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_name\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtm\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 828\u001b[0;31m       \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    829\u001b[0m       \u001b[0mcompiler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"xla\"\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_experimental_compile\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m\"nonXla\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    830\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    853\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    854\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 855\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    856\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    857\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2941\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   2942\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 2943\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   2944\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2945\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1917\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1918\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1919\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1920\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1921\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    558\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    559\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 560\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    561\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    562\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HpiUmx_74Vgh"
      },
      "source": [
        "df_test = pd.read_csv('/content/drive/MyDrive/test.csv.zip')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1AR7MPxc5HV2"
      },
      "source": [
        "df_test[\"Text\"] = df_test.Text.map(remove_stopwords)\n",
        "df_test[\"Text\"] = df_test.Text.map(lambda x: url.sub(r\" \",x))\n",
        "df_test[\"Text\"] = df_test.Text.map(lambda x: html.sub(r\" \",x))\n",
        "df_test[\"Text\"] = df_test.Text.map(lambda x: emoji_pattern.sub(r\" \",x))\n",
        "df_test[\"Text\"] = df_test.Text.map(lambda x: number.sub(r\" \",x))\n",
        "df_test[\"Text\"] = df_test.Text.map(lambda x: pnumber.sub(r\" \",x))\n",
        "df_test[\"Text\"] = df_test.Text.map(lambda x: x.translate(str.maketrans(\" \", \" \", string.punctuation)))\n",
        "df_test[\"Text\"] = df_test.Text.map(lambda x: special.sub(r\" \",x))\n",
        "df_test[\"Text\"] = df_test.Text.map(lambda x: single.sub(r\" \", x))\n",
        "df_test[\"Text\"] = df_test.Text.map(lambda x: normalizer.normalize(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PbLb7VfE5ciQ"
      },
      "source": [
        "sequences_final = tokenizer.texts_to_sequences(df_test.Text)\n",
        "X_final = sequence.pad_sequences(sequences_final, maxlen=maxlen, padding='post')\n",
        "model.load_weights(\"/content/weights-improvement-02-0.84.hdf5\")\n",
        "Y_final = model.predict(X_final)\n",
        "Y_classes = Y_final.argmax(axis=-1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Uf3Q6JbBZeK"
      },
      "source": [
        "# df = pd.read_csv('/content/drive/MyDrive/train.csv')\n",
        "# df.Category = df.Category.astype('category')\n",
        "\n",
        "Y_classes_Name = df.Category.cat.categories[Y_classes]\n",
        "\n",
        "df_test = df_test.rename(columns = {\"Text\":\"Category\"}) \n",
        "df_test.Category = Y_classes_Name\n",
        "\n",
        "df_test.to_csv(\"Final.csv\", index=False)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}