topic_modelling_IMDB.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "topic modelling IMDB.ipynb",
      "provenance": [],
      "mount_file_id": "1G1_iNXBnna0fEGCJC1wC6R0AI-PImuZR",
      "authorship_tag": "ABX9TyN0dslKpKPYGZHDzMM4+dIC",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/michaelajao/topic-modelling-with-LDA/blob/master/topic_modelling_IMDB.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IlhtooAU89Uf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        },
        "outputId": "ce168142-f36f-4f1e-ea90-15031f7fe7eb"
      },
      "source": [
        "import re\n",
        "# for numerical analysis\n",
        "import numpy as np \n",
        "# to store and process in a dataframe\n",
        "import pandas as pd \n",
        "# to create word clouds\n",
        "from wordcloud import WordCloud, STOPWORDS \n",
        "\n",
        "import gensim\n",
        "from gensim.utils import simple_preprocess\n",
        "from gensim.parsing.preprocessing import STOPWORDS\n",
        "from nltk.stem import WordNetLemmatizer, SnowballStemmer\n",
        "from nltk.stem.porter import *\n",
        "import numpy as np\n",
        "np.random.seed(2018)\n",
        "import nltk\n",
        "nltk.download('wordnet')\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "\n",
        "from nltk.stem import PorterStemmer\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.corpus import wordnet\n",
        "from nltk import pos_tag"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sjGAqvPS9Cvn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "outputId": "571c3604-f6cb-4123-daba-5ac2a89d5779"
      },
      "source": [
        "df = pd.read_csv(\"/content/drive/My Drive/crawler/IMDB Dataset.csv\")\n",
        "df['ID'] = [x for x in range(1, len(df.values)+1)]\n",
        "df.head()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>review</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>ID</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>One of the other reviewers has mentioned that ...</td>\n",
              "      <td>positive</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>A wonderful little production. &lt;br /&gt;&lt;br /&gt;The...</td>\n",
              "      <td>positive</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I thought this was a wonderful way to spend ti...</td>\n",
              "      <td>positive</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Basically there's a family where a little boy ...</td>\n",
              "      <td>negative</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Petter Mattei's \"Love in the Time of Money\" is...</td>\n",
              "      <td>positive</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                              review sentiment  ID\n",
              "0  One of the other reviewers has mentioned that ...  positive   1\n",
              "1  A wonderful little production. <br /><br />The...  positive   2\n",
              "2  I thought this was a wonderful way to spend ti...  positive   3\n",
              "3  Basically there's a family where a little boy ...  negative   4\n",
              "4  Petter Mattei's \"Love in the Time of Money\" is...  positive   5"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SB_mEDAkJXL4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Lemmatization is the process of grouping together the different inflected forms of a word so they can be analysed as a single item\n",
        "lem = WordNetLemmatizer()\n",
        "\n",
        "# this function loops through the words by properly with the use of a vocabulary and morphological analysis of words, normally aiming to remove inflectional endings only and to return the base or dictionary form of a word\n",
        "def stop_lemmatize(doc):\n",
        "    tokens = nltk.word_tokenize(doc)\n",
        "    tmp = \"\"\n",
        "    for w in tokens:\n",
        "        if w not in stop:\n",
        "            tmp += lem.lemmatize(w, get_wordnet_pos(w)) + \" \"\n",
        "    return tmp"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nyWhwIbVQUMv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# function to tag first charater lemmatize accepts\n",
        "def get_wordnet_pos(word):\n",
        "    \"\"\"Map POS tag to first character lemmatize() accepts\"\"\"\n",
        "    tag = pos_tag([word])[0][1][0].upper()\n",
        "    tag_dict = {\"J\": wordnet.ADJ,\n",
        "                \"N\": wordnet.NOUN,\n",
        "                \"V\": wordnet.VERB,\n",
        "                \"R\": wordnet.ADV}\n",
        "\n",
        "    return tag_dict.get(tag, wordnet.NOUN)\n",
        "    "
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3pfNFoLVHmaB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "stop = stopwords.words('english')\n",
        "\n",
        "def preprocess(text):\n",
        "    result = []\n",
        "    for token in gensim.utils.simple_preprocess(text):\n",
        "        if token not in gensim.parsing.preprocessing.STOPWORDS and len(token) > 3:\n",
        "            result.append(stop_lemmatize(token))\n",
        "    return result"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VyRdl-AXNoJZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 202
        },
        "outputId": "f74dd9bb-e3c6-471e-e2fe-e7109c1866e3"
      },
      "source": [
        "processed_docs = df['review'].map(preprocess)\n",
        "processed_docs[:10]"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    [reviewer , mention , watch , episode , hooked...\n",
              "1    [wonderful , little , production , film , tech...\n",
              "2    [thought , wonderful , spend , time , summer ,...\n",
              "3    [basically , family , little , jake , think , ...\n",
              "4    [petter , mattei , love , time , money , visua...\n",
              "5    [probably , time , favorite , movie , story , ...\n",
              "6    [sure , like , resurrection , date , seahunt ,...\n",
              "7    [amaze , fresh , innovative , idea , air , yea...\n",
              "8    [encourage , positive , comment , film , look ...\n",
              "9    [like , original , wrench , laughter , like , ...\n",
              "Name: review, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5o4YuSDbRqVE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 202
        },
        "outputId": "8bba9c64-de8f-4db8-a45d-acc96fa31035"
      },
      "source": [
        "dictionary = gensim.corpora.Dictionary(processed_docs)\n",
        "count = 0\n",
        "for k, v in dictionary.iteritems():\n",
        "    print(k, v, dictionary[k])\n",
        "    count += 1\n",
        "    if count > 10:\n",
        "        break"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0  \n",
            "1 accustom  accustom \n",
            "2 agenda  agenda \n",
            "3 agreement  agreement \n",
            "4 appeal  appeal \n",
            "5 aryan  aryan \n",
            "6 audience  audience \n",
            "7 away  away \n",
            "8 bitch  bitch \n",
            "9 brutality  brutality \n",
            "10 call  call \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "giCfstRCRx-r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dictionary.filter_extremes(no_below=15, no_above=0.5, keep_n=100000)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MVOl-K1v0dsU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 386
        },
        "outputId": "8d7cb680-cef1-4e36-b136-81709617339e"
      },
      "source": [
        "bow_corpus = [dictionary.doc2bow(doc) for doc in processed_docs]\n",
        "bow_corpus[4310]"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(39, 1),\n",
              " (88, 1),\n",
              " (113, 1),\n",
              " (150, 1),\n",
              " (184, 1),\n",
              " (227, 1),\n",
              " (441, 1),\n",
              " (634, 2),\n",
              " (642, 1),\n",
              " (777, 1),\n",
              " (1091, 1),\n",
              " (1355, 1),\n",
              " (1792, 4),\n",
              " (2616, 1),\n",
              " (3207, 1),\n",
              " (3436, 1),\n",
              " (4094, 1),\n",
              " (4754, 1),\n",
              " (5139, 1),\n",
              " (5726, 1),\n",
              " (5906, 4),\n",
              " (6266, 1)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZUHoR6h00fRe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 386
        },
        "outputId": "6302036b-0a62-4022-b048-3562bf04fa08"
      },
      "source": [
        "bow_doc_4310 = bow_corpus[4310]\n",
        "for i in range(len(bow_doc_4310)):\n",
        "    print(\"Word {} (\\\"{}\\\") appears {} time.\".format(bow_doc_4310[i][0], \n",
        "                                               dictionary[bow_doc_4310[i][0]], \n",
        "bow_doc_4310[i][1]))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Word 39 (\"go \") appears 1 time.\n",
            "Word 88 (\"show \") appears 1 time.\n",
            "Word 113 (\"concern \") appears 1 time.\n",
            "Word 150 (\"time \") appears 1 time.\n",
            "Word 184 (\"love \") appears 1 time.\n",
            "Word 227 (\"instead \") appears 1 time.\n",
            "Word 441 (\"positive \") appears 1 time.\n",
            "Word 634 (\"relationship \") appears 2 time.\n",
            "Word 642 (\"stereotype \") appears 1 time.\n",
            "Word 777 (\"especially \") appears 1 time.\n",
            "Word 1091 (\"culture \") appears 1 time.\n",
            "Word 1355 (\"express \") appears 1 time.\n",
            "Word 1792 (\"american \") appears 4 time.\n",
            "Word 2616 (\"manner \") appears 1 time.\n",
            "Word 3207 (\"allows \") appears 1 time.\n",
            "Word 3436 (\"fictional \") appears 1 time.\n",
            "Word 4094 (\"community \") appears 1 time.\n",
            "Word 4754 (\"plague \") appears 1 time.\n",
            "Word 5139 (\"humorous \") appears 1 time.\n",
            "Word 5726 (\"outcome \") appears 1 time.\n",
            "Word 5906 (\"african \") appears 4 time.\n",
            "Word 6266 (\"undesirable \") appears 1 time.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MNDzW0px1k_Y",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "c906d05c-2180-4f66-ed4a-4a9c91e412c8"
      },
      "source": [
        "from gensim import corpora, models\n",
        "tfidf = models.TfidfModel(bow_corpus)\n",
        "corpus_tfidf = tfidf[bow_corpus]\n",
        "from pprint import pprint\n",
        "for doc in corpus_tfidf:\n",
        "    pprint(doc)\n",
        "    break"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[(0, 0.04387515478473986),\n",
            " (1, 0.12152012766136024),\n",
            " (2, 0.10218726083118981),\n",
            " (3, 0.11813930243915195),\n",
            " (4, 0.06609045079801779),\n",
            " (5, 0.13461248540772194),\n",
            " (6, 0.04468545483405076),\n",
            " (7, 0.08423763300596263),\n",
            " (8, 0.10593650300462389),\n",
            " (9, 0.10606725353057242),\n",
            " (10, 0.04816121778078706),\n",
            " (11, 0.08889905318025004),\n",
            " (12, 0.06226718332872643),\n",
            " (13, 0.08154234559512293),\n",
            " (14, 0.11809759539084436),\n",
            " (15, 0.06174669799607301),\n",
            " (16, 0.04792185290435016),\n",
            " (17, 0.09730847898279013),\n",
            " (18, 0.11382201689103133),\n",
            " (19, 0.07948455629483547),\n",
            " (20, 0.09899780709388842),\n",
            " (21, 0.1282001411512331),\n",
            " (22, 0.04888268357002287),\n",
            " (23, 0.07576842559866823),\n",
            " (24, 0.11342248671392115),\n",
            " (25, 0.06603373744475663),\n",
            " (26, 0.10661056741866674),\n",
            " (27, 0.05964405706530004),\n",
            " (28, 0.05474953197144464),\n",
            " (29, 0.10327717744045185),\n",
            " (30, 0.046076324401779015),\n",
            " (31, 0.0378890969244496),\n",
            " (32, 0.11001346051640104),\n",
            " (33, 0.06096112126337572),\n",
            " (34, 0.1912882022424186),\n",
            " (35, 0.12348332542600231),\n",
            " (36, 0.13060078429454258),\n",
            " (37, 0.03156067057213172),\n",
            " (38, 0.08405835673695122),\n",
            " (39, 0.024367829634250115),\n",
            " (40, 0.07715885331036995),\n",
            " (41, 0.08480673256840789),\n",
            " (42, 0.04478503286484057),\n",
            " (43, 0.0983883209949407),\n",
            " (44, 0.08614214851627547),\n",
            " (45, 0.08824878002703551),\n",
            " (46, 0.04971409303015353),\n",
            " (47, 0.09380369790997153),\n",
            " (48, 0.11067939287231482),\n",
            " (49, 0.22274188842187834),\n",
            " (50, 0.0926475634797165),\n",
            " (51, 0.07381110097964011),\n",
            " (52, 0.041693432276045814),\n",
            " (53, 0.04958321835321888),\n",
            " (54, 0.1186604437997598),\n",
            " (55, 0.05695796285317246),\n",
            " (56, 0.04598791106365063),\n",
            " (57, 0.07576842559866823),\n",
            " (58, 0.08894962457245659),\n",
            " (59, 0.11507681334023609),\n",
            " (60, 0.11209016484025014),\n",
            " (61, 0.05245404925266962),\n",
            " (62, 0.06410278568144749),\n",
            " (63, 0.060383334759740795),\n",
            " (64, 0.10844915970702187),\n",
            " (65, 0.07866589891825222),\n",
            " (66, 0.14358340751054108),\n",
            " (67, 0.11666081619879247),\n",
            " (68, 0.057311755331592565),\n",
            " (69, 0.13594480728139294),\n",
            " (70, 0.07966494026371802),\n",
            " (71, 0.04997873570681441),\n",
            " (72, 0.03886161688369738),\n",
            " (73, 0.23414352073308797),\n",
            " (74, 0.1366499218934444),\n",
            " (75, 0.060633179448817735),\n",
            " (76, 0.08194941471563828),\n",
            " (77, 0.07785555907377659),\n",
            " (78, 0.08081917936762321),\n",
            " (79, 0.07277259452286859),\n",
            " (80, 0.07767954074004985),\n",
            " (81, 0.06614734362421491),\n",
            " (82, 0.02391343009383148),\n",
            " (83, 0.1425558067975525),\n",
            " (84, 0.08609887985983039),\n",
            " (85, 0.09075960156740148),\n",
            " (86, 0.06952227502556121),\n",
            " (87, 0.11788428939641722),\n",
            " (88, 0.03496583328785836),\n",
            " (89, 0.0750296580935386),\n",
            " (90, 0.10047376316492773),\n",
            " (91, 0.05942709414816648),\n",
            " (92, 0.06345350530013859),\n",
            " (93, 0.18760739581994307),\n",
            " (94, 0.08671432786169293),\n",
            " (95, 0.0703668037437115),\n",
            " (96, 0.07987770154064824),\n",
            " (97, 0.02549727193452876),\n",
            " (98, 0.12492234267559184),\n",
            " (99, 0.05583189316219877),\n",
            " (100, 0.07684618250636767),\n",
            " (101, 0.03713024282307561),\n",
            " (102, 0.09454866060204169),\n",
            " (103, 0.13278262411143638),\n",
            " (104, 0.04727209299523357),\n",
            " (105, 0.24291109472824707),\n",
            " (106, 0.05414580288072697),\n",
            " (107, 0.09839243074877614)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wiKc72c-1ugu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lda_model = gensim.models.LdaMulticore(bow_corpus, num_topics=10, id2word=dictionary, passes=2, workers=2)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iGkmnv_6122T",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 373
        },
        "outputId": "fcf08158-4921-457a-c7db-79d42f4cbb30"
      },
      "source": [
        "for idx, topic in lda_model.print_topics(-1):\n",
        "    print('Topic: {} \\nWords: {}'.format(idx, topic))"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Topic: 0 \n",
            "Words: 0.008*\"time \" + 0.008*\"story \" + 0.007*\"character \" + 0.006*\"like \" + 0.006*\"people \" + 0.005*\"world \" + 0.005*\"watch \" + 0.005*\"great \" + 0.005*\"life \" + 0.004*\"work \"\n",
            "Topic: 1 \n",
            "Words: 0.006*\"great \" + 0.006*\"horror \" + 0.006*\"time \" + 0.006*\"star \" + 0.005*\"scene \" + 0.005*\"best \" + 0.005*\"like \" + 0.005*\"musical \" + 0.005*\"music \" + 0.004*\"good \"\n",
            "Topic: 2 \n",
            "Words: 0.012*\"story \" + 0.009*\"life \" + 0.009*\"character \" + 0.007*\"love \" + 0.006*\"performance \" + 0.005*\"time \" + 0.004*\"young \" + 0.004*\"like \" + 0.004*\"woman \" + 0.004*\"best \"\n",
            "Topic: 3 \n",
            "Words: 0.006*\"like \" + 0.005*\"scene \" + 0.005*\"western \" + 0.004*\"great \" + 0.004*\"director \" + 0.004*\"work \" + 0.003*\"time \" + 0.003*\"character \" + 0.003*\"shot \" + 0.003*\"john \"\n",
            "Topic: 4 \n",
            "Words: 0.013*\"character \" + 0.008*\"like \" + 0.006*\"go \" + 0.006*\"time \" + 0.005*\"kill \" + 0.005*\"\" + 0.005*\"story \" + 0.005*\"get \" + 0.004*\"scene \" + 0.004*\"plot \"\n",
            "Topic: 5 \n",
            "Words: 0.011*\"like \" + 0.010*\"character \" + 0.010*\"scene \" + 0.007*\"time \" + 0.007*\"game \" + 0.007*\"story \" + 0.006*\"action \" + 0.006*\"watch \" + 0.006*\"good \" + 0.005*\"fight \"\n",
            "Topic: 6 \n",
            "Words: 0.010*\"like \" + 0.009*\"bad \" + 0.009*\"act \" + 0.008*\"watch \" + 0.008*\"look \" + 0.008*\"time \" + 0.008*\"scene \" + 0.007*\"see \" + 0.007*\"\" + 0.006*\"horror \"\n",
            "Topic: 7 \n",
            "Words: 0.010*\"great \" + 0.010*\"love \" + 0.010*\"like \" + 0.009*\"series \" + 0.009*\"time \" + 0.009*\"comedy \" + 0.008*\"episode \" + 0.008*\"funny \" + 0.007*\"good \" + 0.007*\"watch \"\n",
            "Topic: 8 \n",
            "Words: 0.028*\"like \" + 0.022*\"good \" + 0.019*\"watch \" + 0.012*\"think \" + 0.012*\"time \" + 0.011*\"\" + 0.010*\"people \" + 0.009*\"thing \" + 0.009*\"know \" + 0.009*\"story \"\n",
            "Topic: 9 \n",
            "Words: 0.009*\"like \" + 0.008*\"life \" + 0.006*\"come \" + 0.006*\"time \" + 0.005*\"good \" + 0.005*\"love \" + 0.005*\"watch \" + 0.005*\"thing \" + 0.005*\"great \" + 0.005*\"\"\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LHVUUT7N1_cd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 205
        },
        "outputId": "e5330f40-6e6b-4e7b-bdfa-b43dd1f5733c"
      },
      "source": [
        "lda_model_tfidf = gensim.models.LdaMulticore(corpus_tfidf, num_topics=10, id2word=dictionary, passes=2, workers=4)\n",
        "for idx, topic in lda_model_tfidf.print_topics(-1):\n",
        "    print('Topic: {} Word: {}'.format(idx, topic))"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Topic: 0 Word: 0.004*\"columbo \" + 0.002*\"scooby \" + 0.002*\"muppets \" + 0.002*\"great \" + 0.002*\"sinatra \" + 0.002*\"freddy \" + 0.002*\"preminger \" + 0.002*\"gandhi \" + 0.002*\"muppet \" + 0.002*\"like \"\n",
            "Topic: 1 Word: 0.002*\"character \" + 0.002*\"good \" + 0.002*\"like \" + 0.002*\"watch \" + 0.002*\"story \" + 0.002*\"time \" + 0.002*\"\" + 0.002*\"think \" + 0.002*\"people \" + 0.002*\"great \"\n",
            "Topic: 2 Word: 0.002*\"good \" + 0.002*\"like \" + 0.002*\"story \" + 0.002*\"great \" + 0.002*\"scene \" + 0.002*\"character \" + 0.002*\"plot \" + 0.002*\"look \" + 0.002*\"time \" + 0.002*\"watch \"\n",
            "Topic: 3 Word: 0.002*\"character \" + 0.002*\"story \" + 0.002*\"great \" + 0.002*\"love \" + 0.002*\"good \" + 0.002*\"like \" + 0.002*\"time \" + 0.002*\"watch \" + 0.002*\"scene \" + 0.002*\"life \"\n",
            "Topic: 4 Word: 0.002*\"like \" + 0.002*\"watch \" + 0.002*\"good \" + 0.002*\"great \" + 0.002*\"time \" + 0.002*\"story \" + 0.002*\"\" + 0.002*\"character \" + 0.002*\"people \" + 0.002*\"see \"\n",
            "Topic: 5 Word: 0.002*\"love \" + 0.002*\"great \" + 0.002*\"story \" + 0.002*\"life \" + 0.002*\"like \" + 0.002*\"watch \" + 0.002*\"good \" + 0.002*\"character \" + 0.002*\"performance \" + 0.001*\"time \"\n",
            "Topic: 6 Word: 0.002*\"watch \" + 0.002*\"like \" + 0.002*\"good \" + 0.002*\"story \" + 0.002*\"time \" + 0.002*\"character \" + 0.002*\"people \" + 0.002*\"scene \" + 0.002*\"think \" + 0.002*\"great \"\n",
            "Topic: 7 Word: 0.002*\"story \" + 0.002*\"character \" + 0.002*\"great \" + 0.002*\"love \" + 0.002*\"like \" + 0.002*\"watch \" + 0.002*\"life \" + 0.002*\"time \" + 0.002*\"good \" + 0.002*\"scene \"\n",
            "Topic: 8 Word: 0.002*\"paulie \" + 0.002*\"watch \" + 0.002*\"great \" + 0.002*\"good \" + 0.002*\"series \" + 0.002*\"episode \" + 0.002*\"see \" + 0.002*\"best \" + 0.002*\"time \" + 0.002*\"story \"\n",
            "Topic: 9 Word: 0.004*\"watch \" + 0.004*\"bad \" + 0.004*\"funny \" + 0.003*\"laugh \" + 0.003*\"act \" + 0.003*\"like \" + 0.003*\"good \" + 0.003*\"see \" + 0.003*\"waste \" + 0.003*\"think \"\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JLQdAYIG2GAO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 538
        },
        "outputId": "1eac4f18-4f36-4339-db49-4737a1cd8124"
      },
      "source": [
        "processed_docs[4310]"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['especially ',\n",
              " 'african ',\n",
              " 'american ',\n",
              " 'time ',\n",
              " 'movie ',\n",
              " 'express ',\n",
              " 'show ',\n",
              " 'concern ',\n",
              " 'go ',\n",
              " 'african ',\n",
              " 'american ',\n",
              " 'relationship ',\n",
              " 'allows ',\n",
              " 'culture ',\n",
              " 'fictional ',\n",
              " 'humorous ',\n",
              " 'manner ',\n",
              " 'positive ',\n",
              " 'african ',\n",
              " 'american ',\n",
              " 'relationship ',\n",
              " 'outcome ',\n",
              " 'instead ',\n",
              " 'undesirable ',\n",
              " 'stereotype ',\n",
              " 'plague ',\n",
              " 'african ',\n",
              " 'american ',\n",
              " 'community ',\n",
              " 'love ',\n",
              " 'film ']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2aB8hdtN2J8l",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 138
        },
        "outputId": "0ef463dc-5f2e-4c45-9a6d-2e2a6fe14f24"
      },
      "source": [
        "for index, score in sorted(lda_model[bow_corpus[4310]], key=lambda tup: -1*tup[1]):\n",
        "    print(\"\\nScore: {}\\t \\nTopic: {}\".format(score, lda_model.print_topic(index, 10)))"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Score: 0.5503628253936768\t \n",
            "Topic: 0.008*\"time \" + 0.008*\"story \" + 0.007*\"character \" + 0.006*\"like \" + 0.006*\"people \" + 0.005*\"world \" + 0.005*\"watch \" + 0.005*\"great \" + 0.005*\"life \" + 0.004*\"work \"\n",
            "\n",
            "Score: 0.4229656159877777\t \n",
            "Topic: 0.012*\"story \" + 0.009*\"life \" + 0.009*\"character \" + 0.007*\"love \" + 0.006*\"performance \" + 0.005*\"time \" + 0.004*\"young \" + 0.004*\"like \" + 0.004*\"woman \" + 0.004*\"best \"\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JOfgAPf72TW1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "7681d951-9473-4daa-ed1d-b01a9ad01edd"
      },
      "source": [
        "for index, score in sorted(lda_model_tfidf[bow_corpus[4310]], key=lambda tup: -1*tup[1]):\n",
        "    print(\"\\nScore: {}\\t \\nTopic: {}\".format(score, lda_model_tfidf.print_topic(index, 10)))"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Score: 0.9699935913085938\t \n",
            "Topic: 0.002*\"character \" + 0.002*\"good \" + 0.002*\"like \" + 0.002*\"watch \" + 0.002*\"story \" + 0.002*\"time \" + 0.002*\"\" + 0.002*\"think \" + 0.002*\"people \" + 0.002*\"great \"\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aqdRUtsN2frA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 185
        },
        "outputId": "38a3585c-18c0-4ff9-8f27-ff9cecc4cb5b"
      },
      "source": [
        "unseen_document = \"The movie was very touching and heart whelming\"\n",
        "bow_vector = dictionary.doc2bow(preprocess(unseen_document))\n",
        "for index, score in sorted(lda_model[bow_vector], key=lambda tup: -1*tup[1]):\n",
        "    print(\"Score: {}\\t Topic: {}\".format(score, lda_model.print_topic(index, 5)))"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Score: 0.6999570727348328\t Topic: 0.012*\"story \" + 0.009*\"life \" + 0.009*\"character \" + 0.007*\"love \" + 0.006*\"performance \"\n",
            "Score: 0.033342648297548294\t Topic: 0.010*\"great \" + 0.010*\"love \" + 0.010*\"like \" + 0.009*\"series \" + 0.009*\"time \"\n",
            "Score: 0.03334103524684906\t Topic: 0.008*\"time \" + 0.008*\"story \" + 0.007*\"character \" + 0.006*\"like \" + 0.006*\"people \"\n",
            "Score: 0.033340658992528915\t Topic: 0.009*\"like \" + 0.008*\"life \" + 0.006*\"come \" + 0.006*\"time \" + 0.005*\"good \"\n",
            "Score: 0.03333787992596626\t Topic: 0.006*\"great \" + 0.006*\"horror \" + 0.006*\"time \" + 0.006*\"star \" + 0.005*\"scene \"\n",
            "Score: 0.033336907625198364\t Topic: 0.013*\"character \" + 0.008*\"like \" + 0.006*\"go \" + 0.006*\"time \" + 0.005*\"kill \"\n",
            "Score: 0.03333652392029762\t Topic: 0.028*\"like \" + 0.022*\"good \" + 0.019*\"watch \" + 0.012*\"think \" + 0.012*\"time \"\n",
            "Score: 0.03333616629242897\t Topic: 0.006*\"like \" + 0.005*\"scene \" + 0.005*\"western \" + 0.004*\"great \" + 0.004*\"director \"\n",
            "Score: 0.03333616629242897\t Topic: 0.011*\"like \" + 0.010*\"character \" + 0.010*\"scene \" + 0.007*\"time \" + 0.007*\"game \"\n",
            "Score: 0.03333493694663048\t Topic: 0.010*\"like \" + 0.009*\"bad \" + 0.009*\"act \" + 0.008*\"watch \" + 0.008*\"look \"\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}