lemmatization_20newsgroups.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Code to perform the lemmatization/tokenization of the 20 NewsGroups dataset\n",
    "* It saves a dictionary with tokenized docs, category of each doc and number of docs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import codecs \n",
    "from glob import glob\n",
    "import os\n",
    "import pickle\n",
    "import copy\n",
    "from time import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. IMPORTING DOCS FROM 20 NEWSGROUPS DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_20newsgroups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "categories = ['alt.atheism',\n",
    " 'comp.graphics',\n",
    " 'comp.os.ms-windows.misc',\n",
    " 'comp.sys.ibm.pc.hardware',\n",
    " 'comp.sys.mac.hardware',\n",
    " 'comp.windows.x',\n",
    " 'misc.forsale',\n",
    " 'rec.autos',\n",
    " 'rec.motorcycles',\n",
    " 'rec.sport.baseball',\n",
    " 'rec.sport.hockey',\n",
    " 'sci.crypt',\n",
    " 'sci.electronics',\n",
    " 'sci.med',\n",
    " 'sci.space',\n",
    " 'soc.religion.christian',\n",
    " 'talk.politics.guns',\n",
    " 'talk.politics.mideast',\n",
    " 'talk.politics.misc',\n",
    " 'talk.religion.misc']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## INPUT PARAMETERS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "newsgroups_train = fetch_20newsgroups(subset='train', \n",
    "                                      remove=('headers', 'footers', 'quotes'),\n",
    "                                      categories=categories, shuffle=True, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "#### TOTAL NUMBER OF DOC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n_docs = newsgroups_train.filenames.shape[0]\n",
    "n_docs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## COMPLETE LEMMATIZATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Return the casting of the original tag in a single\n",
    "# character which is accepted by the lemmatizer\n",
    "import nltk.corpus  # splits on punctuactions   \n",
    "stop_words = nltk.corpus.stopwords.words('english')\n",
    "\n",
    "import re\n",
    "def get_wordnet_pos(treebank_tag):\n",
    "\n",
    "    # I recognize the initial character of the word, identifying the type\n",
    "    if treebank_tag.startswith('J'):\n",
    "        return nltk.corpus.reader.wordnet.ADJ\n",
    "    elif treebank_tag.startswith('V'):\n",
    "        return nltk.corpus.reader.wordnet.VERB\n",
    "    elif treebank_tag.startswith('N'):\n",
    "        return nltk.corpus.reader.wordnet.NOUN\n",
    "    elif treebank_tag.startswith('R'):\n",
    "        return nltk.corpus.reader.wordnet.ADV\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "from nltk import word_tokenize, pos_tag        \n",
    "from nltk.stem import WordNetLemmatizer \n",
    "def LemmaTokenizer_func(doc,cat):\n",
    "    wnl = WordNetLemmatizer()\n",
    "    doc = doc.lower()\n",
    "    tokenized_doc = word_tokenize(doc) # splits on punctuactions  \n",
    "    tagged_doc = pos_tag(tokenized_doc)\n",
    "        \n",
    "    lemmatized_doc = []\n",
    "    # Scan the (word, tag) tuples which are the elements of tagged_tweet1\n",
    "    for word, tag in tagged_doc:\n",
    "        ret_value = get_wordnet_pos(tag)\n",
    "        # If the function does not return None I provide the ret_value\n",
    "        if ret_value != None:\n",
    "            lemmatized_doc.append(wnl.lemmatize(word, get_wordnet_pos(tag)))\n",
    "        # If the function returns None I do not provide the ret_value\n",
    "        else:\n",
    "            lemmatized_doc.append(wnl.lemmatize(word))\n",
    "    \n",
    "    nonumbers_nopunct_lemmatized_doc = [word for word in lemmatized_doc if re.search('[a-zA-Z]{2,}', word)]\n",
    "    lemmatized_doc_stopw = [word for word in nonumbers_nopunct_lemmatized_doc if word not in stop_words]\n",
    "    lemmatized_doc_stopw = ' '.join(lemmatized_doc_stopw)\n",
    "\n",
    "    return (lemmatized_doc_stopw, cat) #[self.wnl.lemmatize(t) for t in word_tokenize(doc)]"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "from nltk import word_tokenize          \n",
    "from nltk.stem import WordNetLemmatizer \n",
    "\n",
    "def LemmaTokenizer_func(doc):\n",
    "    wnl = WordNetLemmatizer()\n",
    "    return [wnl.lemmatize(t) for t in word_tokenize(doc)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cat_docs_original = map(lambda x: newsgroups_train.target_names[x], newsgroups_train.target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from joblib import Parallel, delayed  \n",
    "import multiprocessing\n",
    "t0 = time()\n",
    "num_cores = multiprocessing.cpu_count()\n",
    "\n",
    "tokenized_docs = Parallel(n_jobs=num_cores)(delayed(LemmaTokenizer_func)(doc,cat_doc) for doc,cat_doc in zip(newsgroups_train.data,cat_docs_original))\n",
    "print(\"Full lemmatization done in %0.3fs.\" % (time() - t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cwd =  os.getcwd() # Prints the working directory\n",
    "results_dir_path = cwd + '/results/'\n",
    "\n",
    "output = open(results_dir_path + '/lemmatized_text_n_docs' + str(n_docs) + '.pkl', 'w')\n",
    "\n",
    "pickle.dump({'tokenized_docs':tokenized_docs,\n",
    "             'n_docs':n_docs}, output) #space of the parameters spanned with the grid search\n",
    "output.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}