data-cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Cleaning Pipeline for Religious Texts\n",
    "### Deep Learning Final Project"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "file_path = os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "bible = pd.read_csv('bible1.csv')\n",
    "# bible = bible.replace(r'¶','', regex=True)\n",
    "# bible = bible.drop('Book Number', axis=1)\n",
    "# bible.to_csv(f'{file_path}/bible1.csv', index=False)\n",
    "\n",
    "buddha = pd.read_csv('buddha1.csv')\n",
    "# buddha = buddha.drop('root-pli-ms', axis=1)\n",
    "# buddha['section_num_long'] = buddha[\"segment_id\"].str.slice(2)\n",
    "# buddha['level_1'] = buddha['section_num_long'].str.split(':').str[0]\n",
    "# buddha['remainder'] = buddha['section_num_long'].str.split(':').str[-1]\n",
    "# levels = buddha['remainder'].str.split('.', expand=True)\n",
    "# buddha['level_2'] = levels[0]\n",
    "# buddha['level_3'] = levels[1]\n",
    "# buddha['level_4'] = levels[2]\n",
    "# buddha['level_5'] = levels[3]\n",
    "# buddha[['level_1', 'level_2', 'level_3', 'level_4', 'level_5']] = buddha[['level_1', 'level_2', 'level_3', 'level_4', 'level_5']].fillna(0)\n",
    "# buddha = buddha.drop('remainder', axis=1)\n",
    "# buddha.to_csv(f'{file_path}/buddha1.csv', index=False)\n",
    "\n",
    "\n",
    "mormon = pd.read_csv('mormon1.csv', engine='python', error_bad_lines=False)\n",
    "# mormon = mormon.drop(['book_id', \n",
    "#              'chapter_id', \n",
    "#              'volume_subtitle', \n",
    "#              'book_subtitle', \n",
    "#              'volume_lds_url', \n",
    "#              'book_lds_url', \n",
    "#              'verse_title', \n",
    "#              'volume_long_title', \n",
    "#              'book_long_title'], axis=1)\n",
    "# mormon.to_csv(f'{file_path}/mormon1.csv', index=False)\n",
    "\n",
    "quran = pd.read_csv('quran.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explore: Bible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "bible_OHCO = ['Book Name', 'Chapter', 'Verse']\n",
    "\n",
    "bible = bible.set_index(bible_OHCO[:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Verse ID</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Book Name</th>\n",
       "      <th>Chapter</th>\n",
       "      <th>Verse</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Genesis</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>In the beginning God created the heaven and t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>And the earth was without form, and void; and ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>And God said, Let there be light: and there w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>And God saw the light, that [it was] good: and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>And God called the light Day, and the darkness...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Verse ID  \\\n",
       "Book Name Chapter Verse             \n",
       "Genesis   1       1             1   \n",
       "                  2             2   \n",
       "                  3             3   \n",
       "                  4             4   \n",
       "                  5             5   \n",
       "\n",
       "                                                                      Text  \n",
       "Book Name Chapter Verse                                                     \n",
       "Genesis   1       1       In the beginning God created the heaven and t...  \n",
       "                  2      And the earth was without form, and void; and ...  \n",
       "                  3       And God said, Let there be light: and there w...  \n",
       "                  4      And God saw the light, that [it was] good: and...  \n",
       "                  5      And God called the light Day, and the darkness...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bible.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explore: Gospel of Buddha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "buddha_OHCO = ['level_1', 'level_2', 'level_3', 'level_4', 'level_5']\n",
    "\n",
    "buddha = buddha.set_index(buddha_OHCO[:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>segment_id</th>\n",
       "      <th>translation-en-sujato</th>\n",
       "      <th>comment-en-sujato</th>\n",
       "      <th>section_num_long</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>level_1</th>\n",
       "      <th>level_2</th>\n",
       "      <th>level_3</th>\n",
       "      <th>level_4</th>\n",
       "      <th>level_5</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">0</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>dn1:0.1</td>\n",
       "      <td>Long Discourses 1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1:0.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>dn1:0.2</td>\n",
       "      <td>The Prime Net</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1:0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>dn1:0.3</td>\n",
       "      <td>1. Talk on Wanderers</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1:0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <td>dn1:1.1.1</td>\n",
       "      <td>So I have heard.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1:1.1.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <th>0</th>\n",
       "      <td>dn1:1.1.2</td>\n",
       "      <td>At one time the Buddha was traveling along the...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1:1.1.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        segment_id  \\\n",
       "level_1 level_2 level_3 level_4 level_5              \n",
       "1       0       1       0       0          dn1:0.1   \n",
       "                2       0       0          dn1:0.2   \n",
       "                3       0       0          dn1:0.3   \n",
       "        1       1       1       0        dn1:1.1.1   \n",
       "                        2       0        dn1:1.1.2   \n",
       "\n",
       "                                                                     translation-en-sujato  \\\n",
       "level_1 level_2 level_3 level_4 level_5                                                      \n",
       "1       0       1       0       0                                        Long Discourses 1   \n",
       "                2       0       0                                           The Prime Net    \n",
       "                3       0       0                                     1. Talk on Wanderers   \n",
       "        1       1       1       0                                        So I have heard.    \n",
       "                        2       0        At one time the Buddha was traveling along the...   \n",
       "\n",
       "                                        comment-en-sujato section_num_long  \n",
       "level_1 level_2 level_3 level_4 level_5                                     \n",
       "1       0       1       0       0                     NaN            1:0.1  \n",
       "                2       0       0                     NaN            1:0.2  \n",
       "                3       0       0                     NaN            1:0.3  \n",
       "        1       1       1       0                     NaN          1:1.1.1  \n",
       "                        2       0                     NaN          1:1.1.2  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "buddha.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explore: Book of Mormon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "mormon_OHCO = ['volume_title', 'book_title', 'chapter_number' ,'verse_number']\n",
    "\n",
    "mormon = mormon.set_index(mormon_OHCO[:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>volume_id</th>\n",
       "      <th>verse_id</th>\n",
       "      <th>volume_short_title</th>\n",
       "      <th>book_short_title</th>\n",
       "      <th>scripture_text</th>\n",
       "      <th>verse_short_title</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>volume_title</th>\n",
       "      <th>book_title</th>\n",
       "      <th>chapter_number</th>\n",
       "      <th>verse_number</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Old Testament</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">Genesis</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>OT</td>\n",
       "      <td>Gen.</td>\n",
       "      <td>In the beginning God created the heaven and th...</td>\n",
       "      <td>Gen. 1:1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>OT</td>\n",
       "      <td>Gen.</td>\n",
       "      <td>And the earth was without form, and void; and ...</td>\n",
       "      <td>Gen. 1:2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>OT</td>\n",
       "      <td>Gen.</td>\n",
       "      <td>And God said, Let there be light: and there wa...</td>\n",
       "      <td>Gen. 1:3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>OT</td>\n",
       "      <td>Gen.</td>\n",
       "      <td>And God saw the light, that it was good: and G...</td>\n",
       "      <td>Gen. 1:4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>OT</td>\n",
       "      <td>Gen.</td>\n",
       "      <td>And God called the light Day, and the darkness...</td>\n",
       "      <td>Gen. 1:5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      volume_id  verse_id  \\\n",
       "volume_title  book_title chapter_number verse_number                        \n",
       "Old Testament Genesis    1              1                     1         1   \n",
       "                                        2                     1         2   \n",
       "                                        3                     1         3   \n",
       "                                        4                     1         4   \n",
       "                                        5                     1         5   \n",
       "\n",
       "                                                     volume_short_title  \\\n",
       "volume_title  book_title chapter_number verse_number                      \n",
       "Old Testament Genesis    1              1                            OT   \n",
       "                                        2                            OT   \n",
       "                                        3                            OT   \n",
       "                                        4                            OT   \n",
       "                                        5                            OT   \n",
       "\n",
       "                                                     book_short_title  \\\n",
       "volume_title  book_title chapter_number verse_number                    \n",
       "Old Testament Genesis    1              1                        Gen.   \n",
       "                                        2                        Gen.   \n",
       "                                        3                        Gen.   \n",
       "                                        4                        Gen.   \n",
       "                                        5                        Gen.   \n",
       "\n",
       "                                                                                         scripture_text  \\\n",
       "volume_title  book_title chapter_number verse_number                                                      \n",
       "Old Testament Genesis    1              1             In the beginning God created the heaven and th...   \n",
       "                                        2             And the earth was without form, and void; and ...   \n",
       "                                        3             And God said, Let there be light: and there wa...   \n",
       "                                        4             And God saw the light, that it was good: and G...   \n",
       "                                        5             And God called the light Day, and the darkness...   \n",
       "\n",
       "                                                     verse_short_title  \n",
       "volume_title  book_title chapter_number verse_number                    \n",
       "Old Testament Genesis    1              1                     Gen. 1:1  \n",
       "                                        2                     Gen. 1:2  \n",
       "                                        3                     Gen. 1:3  \n",
       "                                        4                     Gen. 1:4  \n",
       "                                        5                     Gen. 1:5  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mormon.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explore: Qur'an"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "quran_OHCO = ['JuzNo', 'SurahNo', 'AyahNo']\n",
    "\n",
    "quran = quran.set_index(quran_OHCO[:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>SrNo</th>\n",
       "      <th>EnglishTranslation</th>\n",
       "      <th>ArabicWordCount</th>\n",
       "      <th>ArabicLetterCount</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JuzNo</th>\n",
       "      <th>SurahNo</th>\n",
       "      <th>AyahNo</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>In the name of Allah, Most Gracious, Most Merc...</td>\n",
       "      <td>4</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Praise be to Allah, the Cherisher and Sustaine...</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Most Gracious, Most Merciful;</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Master of the Day of Judgment.</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>Thee do we worship, and Thine aid we seek.</td>\n",
       "      <td>4</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      SrNo                                 EnglishTranslation  \\\n",
       "JuzNo SurahNo AyahNo                                                            \n",
       "1     1       1          1  In the name of Allah, Most Gracious, Most Merc...   \n",
       "              2          2  Praise be to Allah, the Cherisher and Sustaine...   \n",
       "              3          3                      Most Gracious, Most Merciful;   \n",
       "              4          4                     Master of the Day of Judgment.   \n",
       "              5          5         Thee do we worship, and Thine aid we seek.   \n",
       "\n",
       "                      ArabicWordCount  ArabicLetterCount  \n",
       "JuzNo SurahNo AyahNo                                      \n",
       "1     1       1                     4                 19  \n",
       "              2                     4                 18  \n",
       "              3                     2                 12  \n",
       "              4                     3                 12  \n",
       "              5                     4                 19  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quran.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate Tokens and Vocab tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/ufu2rg/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /home/ufu2rg/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package stopwords to /home/ufu2rg/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package tagsets to /home/ufu2rg/nltk_data...\n",
      "[nltk_data]   Package tagsets is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /home/ufu2rg/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "\n",
    "nltk_resources = [\n",
    "    'punkt', \n",
    "    'averaged_perceptron_tagger', \n",
    "    'stopwords', \n",
    "    'tagsets',\n",
    "    'wordnet'\n",
    "]\n",
    "\n",
    "for rsc in nltk_resources:\n",
    "    nltk.download(rsc)\n",
    "    \n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Verse ID</th>\n",
       "      <th>Text</th>\n",
       "      <th>Clean Text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Book Name</th>\n",
       "      <th>Chapter</th>\n",
       "      <th>Verse</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Genesis</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>In the beginning God created the heaven and t...</td>\n",
       "      <td>In the beginning God created the heaven and t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>And the earth was without form, and void; and ...</td>\n",
       "      <td>And the earth was without form and void and da...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>And God said, Let there be light: and there w...</td>\n",
       "      <td>And God said Let there be light and there was...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>And God saw the light, that [it was] good: and...</td>\n",
       "      <td>And God saw the light that it was good and God...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>And God called the light Day, and the darkness...</td>\n",
       "      <td>And God called the light Day and the darkness ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Verse ID  \\\n",
       "Book Name Chapter Verse             \n",
       "Genesis   1       1             1   \n",
       "                  2             2   \n",
       "                  3             3   \n",
       "                  4             4   \n",
       "                  5             5   \n",
       "\n",
       "                                                                      Text  \\\n",
       "Book Name Chapter Verse                                                      \n",
       "Genesis   1       1       In the beginning God created the heaven and t...   \n",
       "                  2      And the earth was without form, and void; and ...   \n",
       "                  3       And God said, Let there be light: and there w...   \n",
       "                  4      And God saw the light, that [it was] good: and...   \n",
       "                  5      And God called the light Day, and the darkness...   \n",
       "\n",
       "                                                                Clean Text  \n",
       "Book Name Chapter Verse                                                     \n",
       "Genesis   1       1       In the beginning God created the heaven and t...  \n",
       "                  2      And the earth was without form and void and da...  \n",
       "                  3       And God said Let there be light and there was...  \n",
       "                  4      And God saw the light that it was good and God...  \n",
       "                  5      And God called the light Day and the darkness ...  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bible.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove punctuation from text - \"Clean Text\" column\n",
    "bible[\"Clean Text\"] = bible['Text'].str.replace('[^\\w\\s]','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Verse ID</th>\n",
       "      <th>Text</th>\n",
       "      <th>Clean Text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Book Name</th>\n",
       "      <th>Chapter</th>\n",
       "      <th>Verse</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Genesis</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>In the beginning God created the heaven and t...</td>\n",
       "      <td>In the beginning God created the heaven and t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>And the earth was without form, and void; and ...</td>\n",
       "      <td>And the earth was without form and void and da...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>And God said, Let there be light: and there w...</td>\n",
       "      <td>And God said Let there be light and there was...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>And God saw the light, that [it was] good: and...</td>\n",
       "      <td>And God saw the light that it was good and God...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>And God called the light Day, and the darkness...</td>\n",
       "      <td>And God called the light Day and the darkness ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Verse ID  \\\n",
       "Book Name Chapter Verse             \n",
       "Genesis   1       1             1   \n",
       "                  2             2   \n",
       "                  3             3   \n",
       "                  4             4   \n",
       "                  5             5   \n",
       "\n",
       "                                                                      Text  \\\n",
       "Book Name Chapter Verse                                                      \n",
       "Genesis   1       1       In the beginning God created the heaven and t...   \n",
       "                  2      And the earth was without form, and void; and ...   \n",
       "                  3       And God said, Let there be light: and there w...   \n",
       "                  4      And God saw the light, that [it was] good: and...   \n",
       "                  5      And God called the light Day, and the darkness...   \n",
       "\n",
       "                                                                Clean Text  \n",
       "Book Name Chapter Verse                                                     \n",
       "Genesis   1       1       In the beginning God created the heaven and t...  \n",
       "                  2      And the earth was without form and void and da...  \n",
       "                  3       And God said Let there be light and there was...  \n",
       "                  4      And God saw the light that it was good and God...  \n",
       "                  5      And God called the light Day and the darkness ...  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bible.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate pos, tokens, terms\n",
    "bible_tokens = bible['Clean Text'].apply(lambda x: pd.Series(nltk.pos_tag(nltk.word_tokenize(x)))).stack().to_frame('pos_tuple')\n",
    "\n",
    "bible_tokens['pos'] = bible_tokens.pos_tuple.apply(lambda x: x[1])\n",
    "bible_tokens['token'] = bible_tokens.pos_tuple.apply(lambda x: x[0])\n",
    "bible_tokens['term'] = bible_tokens.token.str.lower()\n",
    "\n",
    "# lemmatize\n",
    "lemmatizer = nltk.stem.WordNetLemmatizer()\n",
    "\n",
    "lemmatized = []\n",
    "for word in bible_tokens['term']:\n",
    "    lemmatized.append(lemmatizer.lemmatize(word))\n",
    "    \n",
    "bible_tokens['lemma'] = lemmatized   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>pos_tuple</th>\n",
       "      <th>pos</th>\n",
       "      <th>token</th>\n",
       "      <th>term</th>\n",
       "      <th>lemma</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Book Name</th>\n",
       "      <th>Chapter</th>\n",
       "      <th>Verse</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"10\" valign=\"top\">Genesis</th>\n",
       "      <th rowspan=\"10\" valign=\"top\">1</th>\n",
       "      <th rowspan=\"10\" valign=\"top\">1</th>\n",
       "      <th>0</th>\n",
       "      <td>(In, IN)</td>\n",
       "      <td>IN</td>\n",
       "      <td>In</td>\n",
       "      <td>in</td>\n",
       "      <td>in</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(the, DT)</td>\n",
       "      <td>DT</td>\n",
       "      <td>the</td>\n",
       "      <td>the</td>\n",
       "      <td>the</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(beginning, NN)</td>\n",
       "      <td>NN</td>\n",
       "      <td>beginning</td>\n",
       "      <td>beginning</td>\n",
       "      <td>beginning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(God, NNP)</td>\n",
       "      <td>NNP</td>\n",
       "      <td>God</td>\n",
       "      <td>god</td>\n",
       "      <td>god</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(created, VBD)</td>\n",
       "      <td>VBD</td>\n",
       "      <td>created</td>\n",
       "      <td>created</td>\n",
       "      <td>created</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>(the, DT)</td>\n",
       "      <td>DT</td>\n",
       "      <td>the</td>\n",
       "      <td>the</td>\n",
       "      <td>the</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>(heaven, NN)</td>\n",
       "      <td>NN</td>\n",
       "      <td>heaven</td>\n",
       "      <td>heaven</td>\n",
       "      <td>heaven</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>(and, CC)</td>\n",
       "      <td>CC</td>\n",
       "      <td>and</td>\n",
       "      <td>and</td>\n",
       "      <td>and</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>(the, DT)</td>\n",
       "      <td>DT</td>\n",
       "      <td>the</td>\n",
       "      <td>the</td>\n",
       "      <td>the</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>(earth, NN)</td>\n",
       "      <td>NN</td>\n",
       "      <td>earth</td>\n",
       "      <td>earth</td>\n",
       "      <td>earth</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 pos_tuple  pos      token       term  \\\n",
       "Book Name Chapter Verse                                                 \n",
       "Genesis   1       1     0         (In, IN)   IN         In         in   \n",
       "                        1        (the, DT)   DT        the        the   \n",
       "                        2  (beginning, NN)   NN  beginning  beginning   \n",
       "                        3       (God, NNP)  NNP        God        god   \n",
       "                        4   (created, VBD)  VBD    created    created   \n",
       "                        5        (the, DT)   DT        the        the   \n",
       "                        6     (heaven, NN)   NN     heaven     heaven   \n",
       "                        7        (and, CC)   CC        and        and   \n",
       "                        8        (the, DT)   DT        the        the   \n",
       "                        9      (earth, NN)   NN      earth      earth   \n",
       "\n",
       "                               lemma  \n",
       "Book Name Chapter Verse               \n",
       "Genesis   1       1     0         in  \n",
       "                        1        the  \n",
       "                        2  beginning  \n",
       "                        3        god  \n",
       "                        4    created  \n",
       "                        5        the  \n",
       "                        6     heaven  \n",
       "                        7        and  \n",
       "                        8        the  \n",
       "                        9      earth  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bible_tokens.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#bible_tokens.reset_index().to_csv(f'{file_path}/bible_tokens.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "bible_vocab = bible_tokens.groupby(level=0)['lemma'].value_counts().to_frame('n')\n",
    "bible_vocab = bible_vocab.reset_index()\n",
    "bible_vocab = bible_vocab.set_index('lemma')\n",
    "\n",
    "# add stopwords\n",
    "new_stopwords = [\"unto\", \"said\", \"thy\", \"thou\", \"thee\", \"shall\", \"upon\", \"ye\", \"saying\", \"saith\", \"hath\", \"forth\", \"also\", \"began\", \"put\", \"wa\"]\n",
    "stpwrd = nltk.corpus.stopwords.words('english')\n",
    "stpwrd.extend(new_stopwords)\n",
    "\n",
    "#sw = pd.DataFrame(nltk.corpus.stopwords.words('english'), columns=['term'])\n",
    "sw = pd.DataFrame(stpwrd, columns=['lemma'])\n",
    "sw = sw. reset_index().set_index('lemma')\n",
    "sw.columns = ['dummy']\n",
    "sw.dummy = 1\n",
    "\n",
    "bible_vocab['stop'] = bible_vocab.index.map(sw.dummy)\n",
    "bible_vocab['stop'] = bible_vocab['stop'].fillna(0).astype(int)\n",
    "\n",
    "# drop stopwords\n",
    "bible_vocab = bible_vocab[bible_vocab['stop'] != 1].drop('stop', axis=1).reset_index().set_index(['Book Name', 'lemma'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>n</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Book Name</th>\n",
       "      <th>lemma</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"25\" valign=\"top\">1 Chronicles</th>\n",
       "      <th>son</th>\n",
       "      <td>632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>david</th>\n",
       "      <td>190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lord</th>\n",
       "      <td>179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>god</th>\n",
       "      <td>119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>israel</th>\n",
       "      <td>114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>house</th>\n",
       "      <td>109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>father</th>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>brother</th>\n",
       "      <td>96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>begat</th>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>thousand</th>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>king</th>\n",
       "      <td>76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>child</th>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>men</th>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>chief</th>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>came</th>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>people</th>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>suburb</th>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hundred</th>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city</th>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>day</th>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>even</th>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hand</th>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>captain</th>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tribe</th>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         n\n",
       "Book Name    lemma        \n",
       "1 Chronicles son       632\n",
       "             david     190\n",
       "             lord      179\n",
       "             god       119\n",
       "             israel    114\n",
       "             house     109\n",
       "             father    105\n",
       "             brother    96\n",
       "             begat      86\n",
       "             thousand   82\n",
       "             king       76\n",
       "             child      66\n",
       "             men        62\n",
       "             chief      54\n",
       "             name       51\n",
       "             came       50\n",
       "             people     44\n",
       "             suburb     44\n",
       "             hundred    43\n",
       "             city       39\n",
       "             day        38\n",
       "             even       37\n",
       "             hand       37\n",
       "             captain    36\n",
       "             tribe      36"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bible_vocab.head(25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bible_vocab.reset_index().to_csv(f'{file_path}/bible_lemma.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lemmatize\n",
    "#bible_vocab.reset_index()\n",
    "#w_tokenizer = nltk.tokenize.WhitespaceTokenizer()\n",
    "lemmatizer = nltk.stem.WordNetLemmatizer()\n",
    "# def lemmatize(s):\n",
    "#     '''For lemmatizing the word\n",
    "#     '''\n",
    "#     s = [lemmatizer.lemmatize(word) for word in s]\n",
    "#     return s\n",
    "\n",
    "# bible_vocab = bible_vocab.assign(col_lemma = bible_vocab.term.apply(lambda x: lemmatize(x)))\n",
    "\n",
    "lemmatized = []\n",
    "for word in bible_vocab['term']:\n",
    "    lemmatized.append(lemmatizer.lemmatize(word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "bible_vocab['lemma'] = lemmatized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Book Name</th>\n",
       "      <th>term</th>\n",
       "      <th>n</th>\n",
       "      <th>col_lemma</th>\n",
       "      <th>lemma</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>son</td>\n",
       "      <td>324</td>\n",
       "      <td>[s, o, n]</td>\n",
       "      <td>son</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>sons</td>\n",
       "      <td>308</td>\n",
       "      <td>[s, o, n, s]</td>\n",
       "      <td>son</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>david</td>\n",
       "      <td>181</td>\n",
       "      <td>[d, a, v, i, d]</td>\n",
       "      <td>david</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>lord</td>\n",
       "      <td>177</td>\n",
       "      <td>[l, o, r, d]</td>\n",
       "      <td>lord</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>god</td>\n",
       "      <td>114</td>\n",
       "      <td>[g, o, d]</td>\n",
       "      <td>god</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>israel</td>\n",
       "      <td>114</td>\n",
       "      <td>[i, s, r, a, e, l]</td>\n",
       "      <td>israel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>house</td>\n",
       "      <td>106</td>\n",
       "      <td>[h, o, u, s, e]</td>\n",
       "      <td>house</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>begat</td>\n",
       "      <td>86</td>\n",
       "      <td>[b, e, g, a, t]</td>\n",
       "      <td>begat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>brethren</td>\n",
       "      <td>77</td>\n",
       "      <td>[b, r, e, t, h, r, e, n]</td>\n",
       "      <td>brother</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>thousand</td>\n",
       "      <td>75</td>\n",
       "      <td>[t, h, o, u, s, a, n, d]</td>\n",
       "      <td>thousand</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>children</td>\n",
       "      <td>66</td>\n",
       "      <td>[c, h, i, l, d, r, e, n]</td>\n",
       "      <td>child</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>men</td>\n",
       "      <td>62</td>\n",
       "      <td>[m, e, n]</td>\n",
       "      <td>men</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>king</td>\n",
       "      <td>60</td>\n",
       "      <td>[k, i, n, g]</td>\n",
       "      <td>king</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>father</td>\n",
       "      <td>55</td>\n",
       "      <td>[f, a, t, h, e, r]</td>\n",
       "      <td>father</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>chief</td>\n",
       "      <td>54</td>\n",
       "      <td>[c, h, i, e, f]</td>\n",
       "      <td>chief</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>came</td>\n",
       "      <td>50</td>\n",
       "      <td>[c, a, m, e]</td>\n",
       "      <td>came</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>fathers</td>\n",
       "      <td>50</td>\n",
       "      <td>[f, a, t, h, e, r, s]</td>\n",
       "      <td>father</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>name</td>\n",
       "      <td>44</td>\n",
       "      <td>[n, a, m, e]</td>\n",
       "      <td>name</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>people</td>\n",
       "      <td>44</td>\n",
       "      <td>[p, e, o, p, l, e]</td>\n",
       "      <td>people</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>suburbs</td>\n",
       "      <td>44</td>\n",
       "      <td>[s, u, b, u, r, b, s]</td>\n",
       "      <td>suburb</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>hundred</td>\n",
       "      <td>38</td>\n",
       "      <td>[h, u, n, d, r, e, d]</td>\n",
       "      <td>hundred</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>even</td>\n",
       "      <td>37</td>\n",
       "      <td>[e, v, e, n]</td>\n",
       "      <td>even</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>went</td>\n",
       "      <td>36</td>\n",
       "      <td>[w, e, n, t]</td>\n",
       "      <td>went</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>levites</td>\n",
       "      <td>35</td>\n",
       "      <td>[l, e, v, i, t, e, s]</td>\n",
       "      <td>levite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1 Chronicles</td>\n",
       "      <td>three</td>\n",
       "      <td>35</td>\n",
       "      <td>[t, h, r, e, e]</td>\n",
       "      <td>three</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Book Name      term    n                 col_lemma     lemma\n",
       "0   1 Chronicles       son  324                 [s, o, n]       son\n",
       "1   1 Chronicles      sons  308              [s, o, n, s]       son\n",
       "2   1 Chronicles     david  181           [d, a, v, i, d]     david\n",
       "3   1 Chronicles      lord  177              [l, o, r, d]      lord\n",
       "4   1 Chronicles       god  114                 [g, o, d]       god\n",
       "5   1 Chronicles    israel  114        [i, s, r, a, e, l]    israel\n",
       "6   1 Chronicles     house  106           [h, o, u, s, e]     house\n",
       "7   1 Chronicles     begat   86           [b, e, g, a, t]     begat\n",
       "8   1 Chronicles  brethren   77  [b, r, e, t, h, r, e, n]   brother\n",
       "9   1 Chronicles  thousand   75  [t, h, o, u, s, a, n, d]  thousand\n",
       "10  1 Chronicles  children   66  [c, h, i, l, d, r, e, n]     child\n",
       "11  1 Chronicles       men   62                 [m, e, n]       men\n",
       "12  1 Chronicles      king   60              [k, i, n, g]      king\n",
       "13  1 Chronicles    father   55        [f, a, t, h, e, r]    father\n",
       "14  1 Chronicles     chief   54           [c, h, i, e, f]     chief\n",
       "15  1 Chronicles      came   50              [c, a, m, e]      came\n",
       "16  1 Chronicles   fathers   50     [f, a, t, h, e, r, s]    father\n",
       "17  1 Chronicles      name   44              [n, a, m, e]      name\n",
       "18  1 Chronicles    people   44        [p, e, o, p, l, e]    people\n",
       "19  1 Chronicles   suburbs   44     [s, u, b, u, r, b, s]    suburb\n",
       "20  1 Chronicles   hundred   38     [h, u, n, d, r, e, d]   hundred\n",
       "21  1 Chronicles      even   37              [e, v, e, n]      even\n",
       "22  1 Chronicles      went   36              [w, e, n, t]      went\n",
       "23  1 Chronicles   levites   35     [l, e, v, i, t, e, s]    levite\n",
       "24  1 Chronicles     three   35           [t, h, r, e, e]     three"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bible_vocab.head(25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Buddha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove punctuation from text - \"Clean Text\" column\n",
    "buddha[\"Clean Text\"] = buddha['translation-en-sujato'].str.replace('[^\\w\\s]','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "buddha.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-101-72827ef40999>:2: DeprecationWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning.\n",
      "  buddha_tokens = buddha['Clean Text'].apply(lambda x: pd.Series(nltk.pos_tag(nltk.word_tokenize(str(x))))).stack().to_frame('pos_tuple')\n"
     ]
    }
   ],
   "source": [
    "# generate pos, tokens, terms\n",
    "buddha_tokens = buddha['Clean Text'].apply(lambda x: pd.Series(nltk.pos_tag(nltk.word_tokenize(str(x))))).stack().to_frame('pos_tuple')\n",
    "\n",
    "buddha_tokens['pos'] = buddha_tokens.pos_tuple.apply(lambda x: x[1])\n",
    "buddha_tokens['token'] = buddha_tokens.pos_tuple.apply(lambda x: x[0])\n",
    "buddha_tokens['term'] = buddha_tokens.token.str.lower()\n",
    "\n",
    "# lemmatize\n",
    "lemmatizer = nltk.stem.WordNetLemmatizer()\n",
    "\n",
    "lemmatized = []\n",
    "for word in buddha_tokens['term']:\n",
    "    lemmatized.append(lemmatizer.lemmatize(word))\n",
    "    \n",
    "buddha_tokens['lemma'] = lemmatized   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>pos_tuple</th>\n",
       "      <th>pos</th>\n",
       "      <th>token</th>\n",
       "      <th>term</th>\n",
       "      <th>lemma</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>level_1</th>\n",
       "      <th>level_2</th>\n",
       "      <th>level_3</th>\n",
       "      <th>level_4</th>\n",
       "      <th>level_5</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">0</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">1</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">0</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">0</th>\n",
       "      <th>0</th>\n",
       "      <td>(Long, JJ)</td>\n",
       "      <td>JJ</td>\n",
       "      <td>Long</td>\n",
       "      <td>long</td>\n",
       "      <td>long</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(Discourses, VBZ)</td>\n",
       "      <td>VBZ</td>\n",
       "      <td>Discourses</td>\n",
       "      <td>discourses</td>\n",
       "      <td>discourse</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(1, CD)</td>\n",
       "      <td>CD</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
       "      <th>0</th>\n",
       "      <td>(The, DT)</td>\n",
       "      <td>DT</td>\n",
       "      <td>The</td>\n",
       "      <td>the</td>\n",
       "      <td>the</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(Prime, NNP)</td>\n",
       "      <td>NNP</td>\n",
       "      <td>Prime</td>\n",
       "      <td>prime</td>\n",
       "      <td>prime</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   pos_tuple  pos       token  \\\n",
       "level_1 level_2 level_3 level_4 level_5                                         \n",
       "1       0       1       0       0       0         (Long, JJ)   JJ        Long   \n",
       "                                        1  (Discourses, VBZ)  VBZ  Discourses   \n",
       "                                        2            (1, CD)   CD           1   \n",
       "                2       0       0       0          (The, DT)   DT         The   \n",
       "                                        1       (Prime, NNP)  NNP       Prime   \n",
       "\n",
       "                                                 term      lemma  \n",
       "level_1 level_2 level_3 level_4 level_5                           \n",
       "1       0       1       0       0       0        long       long  \n",
       "                                        1  discourses  discourse  \n",
       "                                        2           1          1  \n",
       "                2       0       0       0         the        the  \n",
       "                                        1       prime      prime  "
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "buddha_tokens.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>pos_tuple</th>\n",
       "      <th>pos</th>\n",
       "      <th>token</th>\n",
       "      <th>term</th>\n",
       "      <th>lemma</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>level_1</th>\n",
       "      <th>level_2</th>\n",
       "      <th>level_3</th>\n",
       "      <th>level_4</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">0</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">1</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">0</th>\n",
       "      <th>0</th>\n",
       "      <td>(Long, JJ)</td>\n",
       "      <td>JJ</td>\n",
       "      <td>Long</td>\n",
       "      <td>long</td>\n",
       "      <td>long</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(Discourses, VBZ)</td>\n",
       "      <td>VBZ</td>\n",
       "      <td>Discourses</td>\n",
       "      <td>discourses</td>\n",
       "      <td>discourse</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(1, CD)</td>\n",
       "      <td>CD</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">2</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
       "      <th>0</th>\n",
       "      <td>(The, DT)</td>\n",
       "      <td>DT</td>\n",
       "      <td>The</td>\n",
       "      <td>the</td>\n",
       "      <td>the</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(Prime, NNP)</td>\n",
       "      <td>NNP</td>\n",
       "      <td>Prime</td>\n",
       "      <td>prime</td>\n",
       "      <td>prime</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">22</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">22</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">27</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">0</th>\n",
       "      <th>6</th>\n",
       "      <td>(what, WP)</td>\n",
       "      <td>WP</td>\n",
       "      <td>what</td>\n",
       "      <td>what</td>\n",
       "      <td>what</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>(the, DT)</td>\n",
       "      <td>DT</td>\n",
       "      <td>the</td>\n",
       "      <td>the</td>\n",
       "      <td>the</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>(Buddha, NNP)</td>\n",
       "      <td>NNP</td>\n",
       "      <td>Buddha</td>\n",
       "      <td>buddha</td>\n",
       "      <td>buddha</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>(said, VBD)</td>\n",
       "      <td>VBD</td>\n",
       "      <td>said</td>\n",
       "      <td>said</td>\n",
       "      <td>said</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>(nan, NN)</td>\n",
       "      <td>NN</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>185544 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           pos_tuple  pos       token  \\\n",
       "level_1 level_2 level_3 level_4                                         \n",
       "1       0       1       0       0         (Long, JJ)   JJ        Long   \n",
       "                                1  (Discourses, VBZ)  VBZ  Discourses   \n",
       "                                2            (1, CD)   CD           1   \n",
       "                2       0       0          (The, DT)   DT         The   \n",
       "                                1       (Prime, NNP)  NNP       Prime   \n",
       "...                                              ...  ...         ...   \n",
       "22      22      27      0       6         (what, WP)   WP        what   \n",
       "                                7          (the, DT)   DT         the   \n",
       "                                8      (Buddha, NNP)  NNP      Buddha   \n",
       "                                9        (said, VBD)  VBD        said   \n",
       "                28      0       0          (nan, NN)   NN         nan   \n",
       "\n",
       "                                         term      lemma  \n",
       "level_1 level_2 level_3 level_4                           \n",
       "1       0       1       0       0        long       long  \n",
       "                                1  discourses  discourse  \n",
       "                                2           1          1  \n",
       "                2       0       0         the        the  \n",
       "                                1       prime      prime  \n",
       "...                                       ...        ...  \n",
       "22      22      27      0       6        what       what  \n",
       "                                7         the        the  \n",
       "                                8      buddha     buddha  \n",
       "                                9        said       said  \n",
       "                28      0       0         nan        nan  \n",
       "\n",
       "[185544 rows x 5 columns]"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#buddha_tokens.reset_index(level=4, drop=True).reset_index().to_csv(f'{file_path}/buddha_tokens.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "buddha_vocab = buddha_tokens.groupby(level=0)['lemma'].value_counts().to_frame('n')\n",
    "buddha_vocab = buddha_vocab.reset_index()\n",
    "buddha_vocab = buddha_vocab[buddha_vocab['lemma'] != 'nan']\n",
    "buddha_vocab = buddha_vocab.set_index('lemma')\n",
    "\n",
    "# add stopwords\n",
    "new_stopwords = [\"unto\", \"said\", \"thy\", \"thou\", \"thee\", \"shall\", \"upon\", \"ye\", \"saying\", \"saith\", \"hath\", \"forth\", \"also\", \"began\", \"put\", \"wa\"]\n",
    "stpwrd = nltk.corpus.stopwords.words('english')\n",
    "stpwrd.extend(new_stopwords)\n",
    "\n",
    "#sw = pd.DataFrame(nltk.corpus.stopwords.words('english'), columns=['term'])\n",
    "sw = pd.DataFrame(stpwrd, columns=['lemma'])\n",
    "sw = sw.reset_index().set_index('lemma')\n",
    "sw.columns = ['dummy']\n",
    "sw.dummy = 1\n",
    "\n",
    "buddha_vocab['stop'] = buddha_vocab.index.map(sw.dummy)\n",
    "buddha_vocab['stop'] = buddha_vocab['stop'].fillna(0).astype(int)\n",
    "\n",
    "# drop stopwords\n",
    "buddha_vocab = buddha_vocab[buddha_vocab['stop'] != 1].drop('stop', axis=1).reset_index().set_index(['level_1', 'lemma'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>n</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>level_1</th>\n",
       "      <th>lemma</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>ascetic</th>\n",
       "      <td>126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ground</th>\n",
       "      <td>117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>self</th>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>one</th>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>brahmin</th>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   n\n",
       "level_1 lemma       \n",
       "1       ascetic  126\n",
       "        ground   117\n",
       "        self     102\n",
       "        one       99\n",
       "        brahmin   97"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "buddha_vocab.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "#buddha_vocab.reset_index().to_csv(f'{file_path}/buddha_lemma.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mormon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove punctuation from text - \"Clean Text\" column\n",
    "mormon[\"Clean Text\"] = mormon['scripture_text'].str.replace('[^\\w\\s]','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>volume_id</th>\n",
       "      <th>verse_id</th>\n",
       "      <th>volume_short_title</th>\n",
       "      <th>book_short_title</th>\n",
       "      <th>scripture_text</th>\n",
       "      <th>verse_short_title</th>\n",
       "      <th>Clean Text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>volume_title</th>\n",
       "      <th>book_title</th>\n",
       "      <th>chapter_number</th>\n",
       "      <th>verse_number</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Old Testament</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">Genesis</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>OT</td>\n",
       "      <td>Gen.</td>\n",
       "      <td>In the beginning God created the heaven and th...</td>\n",
       "      <td>Gen. 1:1</td>\n",
       "      <td>In the beginning God created the heaven and th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>OT</td>\n",
       "      <td>Gen.</td>\n",
       "      <td>And the earth was without form, and void; and ...</td>\n",
       "      <td>Gen. 1:2</td>\n",
       "      <td>And the earth was without form and void and da...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>OT</td>\n",
       "      <td>Gen.</td>\n",
       "      <td>And God said, Let there be light: and there wa...</td>\n",
       "      <td>Gen. 1:3</td>\n",
       "      <td>And God said Let there be light and there was ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>OT</td>\n",
       "      <td>Gen.</td>\n",
       "      <td>And God saw the light, that it was good: and G...</td>\n",
       "      <td>Gen. 1:4</td>\n",
       "      <td>And God saw the light that it was good and God...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>OT</td>\n",
       "      <td>Gen.</td>\n",
       "      <td>And God called the light Day, and the darkness...</td>\n",
       "      <td>Gen. 1:5</td>\n",
       "      <td>And God called the light Day and the darkness ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      volume_id  verse_id  \\\n",
       "volume_title  book_title chapter_number verse_number                        \n",
       "Old Testament Genesis    1              1                     1         1   \n",
       "                                        2                     1         2   \n",
       "                                        3                     1         3   \n",
       "                                        4                     1         4   \n",
       "                                        5                     1         5   \n",
       "\n",
       "                                                     volume_short_title  \\\n",
       "volume_title  book_title chapter_number verse_number                      \n",
       "Old Testament Genesis    1              1                            OT   \n",
       "                                        2                            OT   \n",
       "                                        3                            OT   \n",
       "                                        4                            OT   \n",
       "                                        5                            OT   \n",
       "\n",
       "                                                     book_short_title  \\\n",
       "volume_title  book_title chapter_number verse_number                    \n",
       "Old Testament Genesis    1              1                        Gen.   \n",
       "                                        2                        Gen.   \n",
       "                                        3                        Gen.   \n",
       "                                        4                        Gen.   \n",
       "                                        5                        Gen.   \n",
       "\n",
       "                                                                                         scripture_text  \\\n",
       "volume_title  book_title chapter_number verse_number                                                      \n",
       "Old Testament Genesis    1              1             In the beginning God created the heaven and th...   \n",
       "                                        2             And the earth was without form, and void; and ...   \n",
       "                                        3             And God said, Let there be light: and there wa...   \n",
       "                                        4             And God saw the light, that it was good: and G...   \n",
       "                                        5             And God called the light Day, and the darkness...   \n",
       "\n",
       "                                                     verse_short_title  \\\n",
       "volume_title  book_title chapter_number verse_number                     \n",
       "Old Testament Genesis    1              1                     Gen. 1:1   \n",
       "                                        2                     Gen. 1:2   \n",
       "                                        3                     Gen. 1:3   \n",
       "                                        4                     Gen. 1:4   \n",
       "                                        5                     Gen. 1:5   \n",
       "\n",
       "                                                                                             Clean Text  \n",
       "volume_title  book_title chapter_number verse_number                                                     \n",
       "Old Testament Genesis    1              1             In the beginning God created the heaven and th...  \n",
       "                                        2             And the earth was without form and void and da...  \n",
       "                                        3             And God said Let there be light and there was ...  \n",
       "                                        4             And God saw the light that it was good and God...  \n",
       "                                        5             And God called the light Day and the darkness ...  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mormon.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate pos, tokens, terms\n",
    "mormon_tokens = mormon['Clean Text'].apply(lambda x: pd.Series(nltk.pos_tag(nltk.word_tokenize(str(x))))).stack().to_frame('pos_tuple')\n",
    "\n",
    "mormon_tokens['pos'] = mormon_tokens.pos_tuple.apply(lambda x: x[1])\n",
    "mormon_tokens['token'] = mormon_tokens.pos_tuple.apply(lambda x: x[0])\n",
    "mormon_tokens['term'] = mormon_tokens.token.str.lower()\n",
    "\n",
    "# lemmatize\n",
    "lemmatizer = nltk.stem.WordNetLemmatizer()\n",
    "\n",
    "lemmatized = []\n",
    "for word in mormon_tokens['term']:\n",
    "    lemmatized.append(lemmatizer.lemmatize(word))\n",
    "    \n",
    "mormon_tokens['lemma'] = lemmatized   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "mormon_vocab = mormon_tokens.groupby(level=1)['lemma'].value_counts().to_frame('n') # group by book title (level 1)\n",
    "mormon_vocab = mormon_vocab.reset_index()\n",
    "#buddha_vocab = buddha_vocab[buddha_vocab['lemma'] != 'nan']\n",
    "mormon_vocab = mormon_vocab.set_index('lemma')\n",
    "\n",
    "# add stopwords\n",
    "new_stopwords = [\"unto\", \"said\", \"thy\", \"thou\", \"thee\", \"shall\", \"upon\", \"ye\", \"saying\", \"saith\", \"hath\", \"forth\", \"also\", \"began\", \"put\", \"wa\"]\n",
    "stpwrd = nltk.corpus.stopwords.words('english')\n",
    "stpwrd.extend(new_stopwords)\n",
    "\n",
    "#sw = pd.DataFrame(nltk.corpus.stopwords.words('english'), columns=['term'])\n",
    "sw = pd.DataFrame(stpwrd, columns=['lemma'])\n",
    "sw = sw.reset_index().set_index('lemma')\n",
    "sw.columns = ['dummy']\n",
    "sw.dummy = 1\n",
    "\n",
    "mormon_vocab['stop'] = mormon_vocab.index.map(sw.dummy)\n",
    "mormon_vocab['stop'] = mormon_vocab['stop'].fillna(0).astype(int)\n",
    "\n",
    "# drop stopwords\n",
    "mormon_vocab = mormon_vocab[mormon_vocab['stop'] != 1].drop('stop', axis=1).reset_index().set_index(['book_title', 'lemma'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>n</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>book_title</th>\n",
       "      <th>lemma</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1 Chronicles</th>\n",
       "      <th>son</th>\n",
       "      <td>632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>david</th>\n",
       "      <td>190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lord</th>\n",
       "      <td>179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>god</th>\n",
       "      <td>119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>israel</th>\n",
       "      <td>114</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       n\n",
       "book_title   lemma      \n",
       "1 Chronicles son     632\n",
       "             david   190\n",
       "             lord    179\n",
       "             god     119\n",
       "             israel  114"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mormon_vocab.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#mormon_vocab.reset_index().to_csv(f'{file_path}/mormon_lemma.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Qur'an"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove punctuation from text - \"Clean Text\" column\n",
    "quran[\"Clean Text\"] = quran['EnglishTranslation'].str.replace('[^\\w\\s]','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>SrNo</th>\n",
       "      <th>EnglishTranslation</th>\n",
       "      <th>ArabicWordCount</th>\n",
       "      <th>ArabicLetterCount</th>\n",
       "      <th>Clean Text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JuzNo</th>\n",
       "      <th>SurahNo</th>\n",
       "      <th>AyahNo</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>In the name of Allah, Most Gracious, Most Merc...</td>\n",
       "      <td>4</td>\n",
       "      <td>19</td>\n",
       "      <td>In the name of Allah Most Gracious Most Merciful</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Praise be to Allah, the Cherisher and Sustaine...</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>Praise be to Allah the Cherisher and Sustainer...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Most Gracious, Most Merciful;</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>Most Gracious Most Merciful</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Master of the Day of Judgment.</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "      <td>Master of the Day of Judgment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>Thee do we worship, and Thine aid we seek.</td>\n",
       "      <td>4</td>\n",
       "      <td>19</td>\n",
       "      <td>Thee do we worship and Thine aid we seek</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      SrNo                                 EnglishTranslation  \\\n",
       "JuzNo SurahNo AyahNo                                                            \n",
       "1     1       1          1  In the name of Allah, Most Gracious, Most Merc...   \n",
       "              2          2  Praise be to Allah, the Cherisher and Sustaine...   \n",
       "              3          3                      Most Gracious, Most Merciful;   \n",
       "              4          4                     Master of the Day of Judgment.   \n",
       "              5          5         Thee do we worship, and Thine aid we seek.   \n",
       "\n",
       "                      ArabicWordCount  ArabicLetterCount  \\\n",
       "JuzNo SurahNo AyahNo                                       \n",
       "1     1       1                     4                 19   \n",
       "              2                     4                 18   \n",
       "              3                     2                 12   \n",
       "              4                     3                 12   \n",
       "              5                     4                 19   \n",
       "\n",
       "                                                             Clean Text  \n",
       "JuzNo SurahNo AyahNo                                                     \n",
       "1     1       1        In the name of Allah Most Gracious Most Merciful  \n",
       "              2       Praise be to Allah the Cherisher and Sustainer...  \n",
       "              3                             Most Gracious Most Merciful  \n",
       "              4                           Master of the Day of Judgment  \n",
       "              5                Thee do we worship and Thine aid we seek  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quran.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate pos, tokens, terms\n",
    "quran_tokens = quran['Clean Text'].apply(lambda x: pd.Series(nltk.pos_tag(nltk.word_tokenize(str(x))))).stack().to_frame('pos_tuple')\n",
    "\n",
    "quran_tokens['pos'] = quran_tokens.pos_tuple.apply(lambda x: x[1])\n",
    "quran_tokens['token'] = quran_tokens.pos_tuple.apply(lambda x: x[0])\n",
    "quran_tokens['term'] = quran_tokens.token.str.lower()\n",
    "\n",
    "# lemmatize\n",
    "lemmatizer = nltk.stem.WordNetLemmatizer()\n",
    "\n",
    "lemmatized = []\n",
    "for word in quran_tokens['term']:\n",
    "    lemmatized.append(lemmatizer.lemmatize(word))\n",
    "    \n",
    "quran_tokens['lemma'] = lemmatized   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "quran_vocab = quran_tokens.groupby(level=0)['lemma'].value_counts().to_frame('n') # group by juz\n",
    "quran_vocab = quran_vocab.reset_index()\n",
    "#buddha_vocab = buddha_vocab[buddha_vocab['lemma'] != 'nan']\n",
    "quran_vocab = quran_vocab.set_index('lemma')\n",
    "\n",
    "# add stopwords\n",
    "new_stopwords = [\"unto\", \"said\", \"thy\", \"thou\", \"thee\", \"shall\", \"upon\", \"ye\", \"saying\", \"saith\", \"hath\", \"forth\", \"also\", \"began\", \"put\", \"wa\"]\n",
    "stpwrd = nltk.corpus.stopwords.words('english')\n",
    "stpwrd.extend(new_stopwords)\n",
    "\n",
    "#sw = pd.DataFrame(nltk.corpus.stopwords.words('english'), columns=['term'])\n",
    "sw = pd.DataFrame(stpwrd, columns=['lemma'])\n",
    "sw = sw.reset_index().set_index('lemma')\n",
    "sw.columns = ['dummy']\n",
    "sw.dummy = 1\n",
    "\n",
    "quran_vocab['stop'] = quran_vocab.index.map(sw.dummy)\n",
    "quran_vocab['stop'] = quran_vocab['stop'].fillna(0).astype(int)\n",
    "\n",
    "# drop stopwords\n",
    "quran_vocab = quran_vocab[quran_vocab['stop'] != 1].drop('stop', axis=1).reset_index().set_index(['JuzNo', 'lemma'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>n</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JuzNo</th>\n",
       "      <th>lemma</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>allah</th>\n",
       "      <td>92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>say</th>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>believe</th>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>faith</th>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lord</th>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                n\n",
       "JuzNo lemma      \n",
       "1     allah    92\n",
       "      say      36\n",
       "      believe  32\n",
       "      faith    27\n",
       "      lord     27"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quran_vocab.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "#quran_vocab.reset_index().to_csv(f'{file_path}/quran_lemma.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EDA: Religious Texts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Verse ID</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Book Name</th>\n",
       "      <th>Chapter</th>\n",
       "      <th>Verse</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Genesis</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>In the beginning God created the heaven and t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>And the earth was without form, and void; and ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>And God said, Let there be light: and there w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>And God saw the light, that [it was] good: and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>And God called the light Day, and the darkness...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Verse ID  \\\n",
       "Book Name Chapter Verse             \n",
       "Genesis   1       1             1   \n",
       "                  2             2   \n",
       "                  3             3   \n",
       "                  4             4   \n",
       "                  5             5   \n",
       "\n",
       "                                                                      Text  \n",
       "Book Name Chapter Verse                                                     \n",
       "Genesis   1       1       In the beginning God created the heaven and t...  \n",
       "                  2      And the earth was without form, and void; and ...  \n",
       "                  3       And God said, Let there be light: and there w...  \n",
       "                  4      And God saw the light, that [it was] good: and...  \n",
       "                  5      And God called the light Day, and the darkness...  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bible.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "66"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(bible.groupby(level=0)) # number of chapters in Bible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "150"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(bible.groupby(level=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "176"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(bible.groupby(level=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bible['Text'].isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gospel of Buddha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>segment_id</th>\n",
       "      <th>translation-en-sujato</th>\n",
       "      <th>comment-en-sujato</th>\n",
       "      <th>section_num_long</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>level_1</th>\n",
       "      <th>level_2</th>\n",
       "      <th>level_3</th>\n",
       "      <th>level_4</th>\n",
       "      <th>level_5</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">0</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>dn1:0.1</td>\n",
       "      <td>Long Discourses 1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1:0.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>dn1:0.2</td>\n",
       "      <td>The Prime Net</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1:0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>dn1:0.3</td>\n",
       "      <td>1. Talk on Wanderers</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1:0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <td>dn1:1.1.1</td>\n",
       "      <td>So I have heard.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1:1.1.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <th>0</th>\n",
       "      <td>dn1:1.1.2</td>\n",
       "      <td>At one time the Buddha was traveling along the...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1:1.1.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        segment_id  \\\n",
       "level_1 level_2 level_3 level_4 level_5              \n",
       "1       0       1       0       0          dn1:0.1   \n",
       "                2       0       0          dn1:0.2   \n",
       "                3       0       0          dn1:0.3   \n",
       "        1       1       1       0        dn1:1.1.1   \n",
       "                        2       0        dn1:1.1.2   \n",
       "\n",
       "                                                                     translation-en-sujato  \\\n",
       "level_1 level_2 level_3 level_4 level_5                                                      \n",
       "1       0       1       0       0                                        Long Discourses 1   \n",
       "                2       0       0                                           The Prime Net    \n",
       "                3       0       0                                     1. Talk on Wanderers   \n",
       "        1       1       1       0                                        So I have heard.    \n",
       "                        2       0        At one time the Buddha was traveling along the...   \n",
       "\n",
       "                                        comment-en-sujato section_num_long  \n",
       "level_1 level_2 level_3 level_4 level_5                                     \n",
       "1       0       1       0       0                     NaN            1:0.1  \n",
       "                2       0       0                     NaN            1:0.2  \n",
       "                3       0       0                     NaN            1:0.3  \n",
       "        1       1       1       0                     NaN          1:1.1.1  \n",
       "                        2       0                     NaN          1:1.1.2  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "buddha.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16357"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(buddha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "35"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(buddha.groupby(level=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "109"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(buddha.groupby(level=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "280"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(buddha.groupby(level=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "214"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(buddha.groupby(level=3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(buddha.groupby(level=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "69"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "buddha['translation-en-sujato'].isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Book of Mormon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>volume_id</th>\n",
       "      <th>verse_id</th>\n",
       "      <th>volume_short_title</th>\n",
       "      <th>book_short_title</th>\n",
       "      <th>scripture_text</th>\n",
       "      <th>verse_short_title</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>volume_title</th>\n",
       "      <th>book_title</th>\n",
       "      <th>chapter_number</th>\n",
       "      <th>verse_number</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Old Testament</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">Genesis</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>OT</td>\n",
       "      <td>Gen.</td>\n",
       "      <td>In the beginning God created the heaven and th...</td>\n",
       "      <td>Gen. 1:1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>OT</td>\n",
       "      <td>Gen.</td>\n",
       "      <td>And the earth was without form, and void; and ...</td>\n",
       "      <td>Gen. 1:2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>OT</td>\n",
       "      <td>Gen.</td>\n",
       "      <td>And God said, Let there be light: and there wa...</td>\n",
       "      <td>Gen. 1:3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>OT</td>\n",
       "      <td>Gen.</td>\n",
       "      <td>And God saw the light, that it was good: and G...</td>\n",
       "      <td>Gen. 1:4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>OT</td>\n",
       "      <td>Gen.</td>\n",
       "      <td>And God called the light Day, and the darkness...</td>\n",
       "      <td>Gen. 1:5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      volume_id  verse_id  \\\n",
       "volume_title  book_title chapter_number verse_number                        \n",
       "Old Testament Genesis    1              1                     1         1   \n",
       "                                        2                     1         2   \n",
       "                                        3                     1         3   \n",
       "                                        4                     1         4   \n",
       "                                        5                     1         5   \n",
       "\n",
       "                                                     volume_short_title  \\\n",
       "volume_title  book_title chapter_number verse_number                      \n",
       "Old Testament Genesis    1              1                            OT   \n",
       "                                        2                            OT   \n",
       "                                        3                            OT   \n",
       "                                        4                            OT   \n",
       "                                        5                            OT   \n",
       "\n",
       "                                                     book_short_title  \\\n",
       "volume_title  book_title chapter_number verse_number                    \n",
       "Old Testament Genesis    1              1                        Gen.   \n",
       "                                        2                        Gen.   \n",
       "                                        3                        Gen.   \n",
       "                                        4                        Gen.   \n",
       "                                        5                        Gen.   \n",
       "\n",
       "                                                                                         scripture_text  \\\n",
       "volume_title  book_title chapter_number verse_number                                                      \n",
       "Old Testament Genesis    1              1             In the beginning God created the heaven and th...   \n",
       "                                        2             And the earth was without form, and void; and ...   \n",
       "                                        3             And God said, Let there be light: and there wa...   \n",
       "                                        4             And God saw the light, that it was good: and G...   \n",
       "                                        5             And God called the light Day, and the darkness...   \n",
       "\n",
       "                                                     verse_short_title  \n",
       "volume_title  book_title chapter_number verse_number                    \n",
       "Old Testament Genesis    1              1                     Gen. 1:1  \n",
       "                                        2                     Gen. 1:2  \n",
       "                                        3                     Gen. 1:3  \n",
       "                                        4                     Gen. 1:4  \n",
       "                                        5                     Gen. 1:5  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mormon.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "41995"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(mormon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(mormon.groupby(level=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "87"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(mormon.groupby(level=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "150"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(mormon.groupby(level=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "176"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(mormon.groupby(level=3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mormon['scripture_text'].isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Qur'an"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>SrNo</th>\n",
       "      <th>EnglishTranslation</th>\n",
       "      <th>ArabicWordCount</th>\n",
       "      <th>ArabicLetterCount</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JuzNo</th>\n",
       "      <th>SurahNo</th>\n",
       "      <th>AyahNo</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>In the name of Allah, Most Gracious, Most Merc...</td>\n",
       "      <td>4</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Praise be to Allah, the Cherisher and Sustaine...</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Most Gracious, Most Merciful;</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Master of the Day of Judgment.</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>Thee do we worship, and Thine aid we seek.</td>\n",
       "      <td>4</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      SrNo                                 EnglishTranslation  \\\n",
       "JuzNo SurahNo AyahNo                                                            \n",
       "1     1       1          1  In the name of Allah, Most Gracious, Most Merc...   \n",
       "              2          2  Praise be to Allah, the Cherisher and Sustaine...   \n",
       "              3          3                      Most Gracious, Most Merciful;   \n",
       "              4          4                     Master of the Day of Judgment.   \n",
       "              5          5         Thee do we worship, and Thine aid we seek.   \n",
       "\n",
       "                      ArabicWordCount  ArabicLetterCount  \n",
       "JuzNo SurahNo AyahNo                                      \n",
       "1     1       1                     4                 19  \n",
       "              2                     4                 18  \n",
       "              3                     2                 12  \n",
       "              4                     3                 12  \n",
       "              5                     4                 19  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quran.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6236"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(quran)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(quran.groupby(level=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "114"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(quran.groupby(level=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "286"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(quran.groupby(level=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quran['EnglishTranslation'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}