Topic2Vec_20newsgroups.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TOPIC2VEC algorithm by using gensim and according to the second hint given by Gordon Mohr.  \n",
    "(https://groups.google.com/forum/#!topic/gensim/BVu5-pD6910)\n",
    "\n",
    "\n",
    "1. Vectorization of docs by using CountVectorizer (with or without tfidf) with no lemmatization\n",
    "2. Latent Dirichlet Allocation \n",
    "3. Topic2Vec of the entire dataset (20 NewsGroups)   \n",
    "\n",
    "It saves:\n",
    "* the topic2vec model obtained"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np; import pandas as pd; import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import codecs \n",
    "from glob import glob\n",
    "import os\n",
    "import pickle\n",
    "import copy\n",
    "import pyorient\n",
    "import ast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "from time import time\n",
    "import string\n",
    "import re\n",
    "# random\n",
    "from random import shuffle\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS\n",
    "from sklearn.decomposition import LatentDirichletAllocation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from gensim import corpora, models, similarities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_top_words = 20"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. IMPORTING DOCS FROM 20 NEWSGROUPS DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_20newsgroups\n",
    "categories = ['comp.sys.ibm.pc.hardware',\n",
    "'comp.sys.mac.hardware',\n",
    "'comp.windows.x',\n",
    "'rec.sport.baseball',\n",
    "'rec.sport.hockey',\n",
    "'sci.med',\n",
    "'sci.space',\n",
    "'soc.religion.christian']\n",
    "\n",
    "n_topics = len(categories)\n",
    "\n",
    "categories_source = {}\n",
    "\n",
    "for cat in categories:\n",
    "    categories_source[cat] = cat.replace('.', '_')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "newsgroups_train = fetch_20newsgroups(subset='train',\n",
    "                                      remove=('headers', 'footers', 'quotes'),\n",
    "                                      categories=categories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i,j in categories_source.items():\n",
    "    print(i,j)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "#### TOTAL NUMBER OF DOC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n_docs = newsgroups_train.filenames.shape[0]\n",
    "n_docs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. LDA to find the topic most-associated with each word"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 From Strings to Vectors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### WITH Lemmatization"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# Return the casting of the original tag in a single\n",
    "# character which is accepted by the lemmatizer\n",
    "import nltk.corpus  # splits on punctuactions   \n",
    "stop_words = nltk.corpus.stopwords.words('english')\n",
    "\n",
    "import re\n",
    "def get_wordnet_pos(treebank_tag):\n",
    "\n",
    "    # I recognize the initial character of the word, identifying the type\n",
    "    if treebank_tag.startswith('J'):\n",
    "        return nltk.corpus.reader.wordnet.ADJ\n",
    "    elif treebank_tag.startswith('V'):\n",
    "        return nltk.corpus.reader.wordnet.VERB\n",
    "    elif treebank_tag.startswith('N'):\n",
    "        return nltk.corpus.reader.wordnet.NOUN\n",
    "    elif treebank_tag.startswith('R'):\n",
    "        return nltk.corpus.reader.wordnet.ADV\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "from nltk import word_tokenize, pos_tag        \n",
    "from nltk.stem import WordNetLemmatizer \n",
    "class LemmaTokenizer(object):\n",
    "    def __init__(self):\n",
    "        self.wnl = WordNetLemmatizer()\n",
    "    def __call__(self, doc):\n",
    "        tokenized_doc = word_tokenize(doc) # splits on punctuactions  \n",
    "        tagged_doc = pos_tag(tokenized_doc)\n",
    "        \n",
    "        lemmatized_doc = []\n",
    "        # Scan the (word, tag) tuples which are the elements of tagged_tweet1\n",
    "        for word, tag in tagged_doc:\n",
    "            ret_value = get_wordnet_pos(tag)\n",
    "            # If the function does not return None I provide the ret_value\n",
    "            if ret_value != None:\n",
    "                lemmatized_doc.append(self.wnl.lemmatize(word, get_wordnet_pos(tag)))\n",
    "            # If the function returns None I do not provide the ret_value\n",
    "            else:\n",
    "                lemmatized_doc.append(self.wnl.lemmatize(word))\n",
    "        nonumbers_nopunct_lemmatized_doc = [word for word in lemmatized_doc if re.search('[a-zA-Z]{2,}', word)]\n",
    "#        nonumbers_nopunct_lemmatized_doc = [word for word in nopunct_lemmatized_doc if not re.search('\\d+', word)]\n",
    "        lemmatized_doc_stopw = [word for word in nonumbers_nopunct_lemmatized_doc if word not in stop_words]\n",
    "        \n",
    "        return lemmatized_doc_stopw #[self.wnl.lemmatize(t) for t in word_tokenize(doc)]"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "t0 = time()\n",
    "tf_vectorizer = CountVectorizer(tokenizer=LemmaTokenizer(), encoding='utf-8', analyzer='word',\n",
    "                                stop_words=[\"'s\", \"fx\"], ngram_range = (1,1), min_df = 2).fit(df_norm2.text)\n",
    "print(\"fit vectorizer with lemmatization done in %0.3fs.\" % (time() - t0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### WITHOUT Lemmatization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "t0 = time()\n",
    "tf_vectorizer = CountVectorizer(encoding='utf-8', analyzer='word', stop_words='english',\n",
    "                                ngram_range = (1,1), min_df = 2, token_pattern = '[a-zA-Z]{2,}').fit(newsgroups_train.data)\n",
    "print(\"fit vectorizer without lemmatization done in %0.3fs.\" % (time() - t0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vectorization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n_features = len(tf_vectorizer.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "newsgroups_train.data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tf_docs = tf_vectorizer.transform(newsgroups_train.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### WITH TFIDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "tfidf_vectorizer = TfidfTransformer(sublinear_tf=False, use_idf = True).fit(tf_docs)\n",
    "tfidf_docs = tfidf_vectorizer.transform(tf_docs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 LDA implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_top_words(model, feature_names, n_top_words):\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "        print(\"Topic #%d:\" % topic_idx)\n",
    "        print(\" \".join([feature_names[i] for i in topic.argsort()[:-n_top_words - 1:-1]]))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(\"Fitting LDA models with tf features, n_samples=%d and n_features=%d...\"\n",
    "      % (n_docs, n_features))\n",
    "lda = LatentDirichletAllocation(n_topics=n_topics, max_iter=5,\n",
    "                                learning_method='online', learning_offset=50.,\n",
    "                                random_state=0)\n",
    "t0 = time()\n",
    "lda.fit(tf_docs)\n",
    "print(\"done in %0.3fs.\" % (time() - t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(\"\\nTopics in LDA model:\")\n",
    "tf_feature_names = tf_vectorizer.get_feature_names()\n",
    "print_top_words(lda, tf_feature_names, n_top_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "per_topic_distr_LDA = lda.components_\n",
    "per_topic_distr_LDA.shape\n",
    "#per_topic_distr_LDA.sum(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. TOPIC2VEC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "most_p_topic = np.argmax(per_topic_distr_LDA, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "word_and_topic = zip(tf_feature_names, most_p_topic)\n",
    "\n",
    "word2topic_dict = {word : 'topic_' + np.array_str(topic) for word, topic in word_and_topic}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1 Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def tokenizer(document):\n",
    "    \n",
    "    text = \"\".join([ch for ch in document if ch not in string.punctuation])\n",
    "    text_list = text.split()\n",
    "    normalized_text = [x.lower() for x in text_list]\n",
    "    # Define an empty list\n",
    "    nostopwords_text = []\n",
    "    # Scan the words\n",
    "    for word in normalized_text:\n",
    "        # Determine if the word is contained in the stop words list\n",
    "        if word not in ENGLISH_STOP_WORDS:\n",
    "            # If the word is not contained I append it\n",
    "            nostopwords_text.append(word)\n",
    "    tokenized_text = [word for word in nostopwords_text if re.search('[a-zA-Z]{2,}', word)]\n",
    "            \n",
    "    return tokenized_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def map_doc_to_topic(tokenized_text, prefix, doc_id_number, word2topic_dict):\n",
    "    doc_to_topic_list = [prefix + '_' + str(doc_id_number)]\n",
    "    for word in tokenized_text:\n",
    "        if word in word2topic_dict.keys():\n",
    "            doc_to_topic_list.append(word2topic_dict[word])\n",
    "            \n",
    "    return doc_to_topic_list"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "class LabeledLineSentence(object):\n",
    "    def __init__(self, docs_list, word2topic_dict):\n",
    "        self.labels_list = word2topic_dict\n",
    "        self.docs_list = docs_list\n",
    "    def __iter__(self):\n",
    "        for idx, doc in enumerate(self.docs_list):\n",
    "            words_doc=tokenizer(doc)\n",
    "            tags_doc = map_doc_to_topic(words_doc, idx, word2topic_dict)\n",
    "            yield models.doc2vec.LabeledSentence(words = words_doc,\n",
    "                                                 tags = tags_doc)\n",
    "            \n",
    "    def sentences_perm(self):\n",
    "        shuffle(models.doc2vec.LabeledSentence)\n",
    "        return models.doc2vec.LabeledSentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class LabeledLineSentence_training(object):\n",
    "    def __init__(self, sources, word2topic_dict):\n",
    "        self.labels_list = word2topic_dict\n",
    "        self.sources = sources\n",
    "        flipped = {}\n",
    "        # make sure that keys are unique\n",
    "        for key, value in sources.items():\n",
    "            if value not in flipped:\n",
    "                flipped[value] = [key]\n",
    "            else:\n",
    "                raise Exception('Non-unique prefix encountered')\n",
    "    \n",
    "    def __iter__(self):\n",
    "        for source, prefix in self.sources.items():\n",
    "            print(source)\n",
    "            newsgroups_train_cat = fetch_20newsgroups(subset='train',\n",
    "                                                      remove=('headers', 'footers', 'quotes'),\n",
    "                                                      categories=[source])\n",
    "            for idx, doc in enumerate(newsgroups_train_cat.data):\n",
    "                words_doc=tokenizer(doc)\n",
    "                tags_doc = map_doc_to_topic(words_doc, prefix, idx, word2topic_dict)\n",
    "                yield models.doc2vec.LabeledSentence(words = words_doc,\n",
    "                                                     tags = tags_doc)\n",
    "                \n",
    "    def to_array(self):\n",
    "        self.sentences = []\n",
    "        for source, prefix in self.sources.items():\n",
    "            print(source)\n",
    "            newsgroups_train_cat = fetch_20newsgroups(subset='train',\n",
    "                                                      remove=('headers', 'footers', 'quotes'),\n",
    "                                                      categories=[source])\n",
    "            for idx, doc in enumerate(newsgroups_train_cat.data):\n",
    "                words_doc=tokenizer(doc)\n",
    "                tags_doc = map_doc_to_topic(words_doc, prefix, idx, word2topic_dict)\n",
    "                self.sentences.append(models.doc2vec.LabeledSentence(words = words_doc,\n",
    "                                                     tags = tags_doc))\n",
    "        return self.sentences\n",
    "            \n",
    "    def sentences_perm(self):\n",
    "        shuffle(self.sentences)\n",
    "        return self.sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 3.1 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "it = LabeledLineSentence_training(categories_source, word2topic_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = models.Doc2Vec(size=100, window=10, min_count=4, dm=1, dbow_words=1,\n",
    "                              workers=50, alpha=0.025, min_alpha=0.025) # use fixed learning rate\n",
    "model.build_vocab(it.to_array())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for epoch in range(10):\n",
    "    model.train(it.sentences_perm())\n",
    "    model.alpha -= 0.002 # decrease the learning rate\n",
    "    model.min_alpha = model.alpha # fix the learning rate, no decay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fname =  os.getcwd() # Prints the working directory\n",
    "fname = fname + '/topic2vec_20NG_2_ndoc' + str(n_docs) + 'n_topic' + str(n_topics) + '.model'\n",
    "model.save(fname)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}