Kaggle_CommonLit_Challenge_Generate_Trng_Set_K_Dim.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import spacy\n",
    "import nltk\n",
    "import string\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch import Tensor\n",
    "import math"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read the GloVe embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run Kaggle_CommonLit_Challenge_TE_Func.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read the Glove embeddings for each word token\n",
    "embeddings_dict = {}\n",
    "# embeddings_dim = 300\n",
    "# embeddings_dim = 100\n",
    "embeddings_dim = 200\n",
    "\n",
    "# with open('C:/Work/Vivek_Work/NLP_Vocab/glove_6B/glove.6B.300d.txt', 'r', encoding=\"utf-8\") as f:\n",
    "# with open('C:/Work/Vivek_Work/NLP_Vocab/glove_6B/glove.6B.100d.txt', 'r', encoding=\"utf-8\") as f:\n",
    "with open('C:/Work/Vivek_Work/NLP_Vocab/glove_6B/glove.6B.200d.txt', 'r', encoding=\"utf-8\") as f:\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        vector = np.asarray(values[1:], \"float32\")\n",
    "        embeddings_dict[word] = vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of word tokens\n",
    "len(embeddings_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keyset = list(embeddings_dict)\n",
    "# keyset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Identify stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "punctuation_labels = list(string.punctuation)\n",
    "# punctuation_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "spacy_stopwords = spacy.lang.en.stop_words.STOP_WORDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "spacy_stopwords = spacy_stopwords | set(punctuation_labels)\n",
    "len(spacy_stopwords)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate the word embedding matrix per excerpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trng_set = pd.read_csv('C:/Work/Vivek_Work/Challenge_Data_Sets/Kaggle/CommonLit_Readability/train.csv')\n",
    "trng_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trng_set.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trng_set['excerpt'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 1: Identify the common number of words across all excerpts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "non_missing_words_per_excerpt = []\n",
    "num_excerpts = trng_set.shape[0]\n",
    "\n",
    "for idx in range(num_excerpts):\n",
    "    if idx%100 == 0:\n",
    "        print(idx)\n",
    "    non_missing_words_per_excerpt.append(excerpt_embedding(trng_set['excerpt'][idx], 0, embeddings_dim))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pd.Series(non_missing_words_per_excerpt).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.unique(np.array(non_missing_words_per_excerpt) >= 77, return_counts=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 2: Consider only those excerpts for which the word length exceeds 77"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx_valid_excerpt = []\n",
    "num_excerpts = trng_set.shape[0]\n",
    "# trng_embeddings = np.zeros((1,50))\n",
    "trng_embeddings = np.zeros((1,embeddings_dim))\n",
    "\n",
    "for idx in range(num_excerpts):\n",
    "    if idx%100 == 0:\n",
    "        print(idx)    \n",
    "    \n",
    "    temp = excerpt_embedding(trng_set['excerpt'][idx], 1, embeddings_dim)\n",
    "    \n",
    "    if temp.shape[0] >= 77:\n",
    "        idx_valid_excerpt.append(idx)\n",
    "        trng_embeddings = np.concatenate((trng_embeddings, temp[0:77, :]), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trng_embeddings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of input samples for which embeddings could be generated\n",
    "# trng_embeddings.shape[0] - 1\n",
    "165242/77"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the excerpt embeddings along with the output values\n",
    "output_matrix = trng_set.iloc[idx_valid_excerpt, [4, 5]]\n",
    "output_matrix.to_csv('Scores_Valid_Excerpt.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.save('Embeddings_Valid_Excerpt_Dim_300.npy', trng_embeddings[1:, :])\n",
    "# np.save('Embeddings_Valid_Excerpt_Dim_100.npy', trng_embeddings[1:, :])\n",
    "np.save('Embeddings_Valid_Excerpt_Dim_200.npy', trng_embeddings[1:, :])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python_3_7",
   "language": "python",
   "name": "python_3_7"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}