unigram_text_generate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import os\n",
    "import re\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "words = set(nltk.corpus.words.words())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stopwords=['','(',')','{','}','\\\\','--',':']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'contributors': None,\n",
       " 'coordinates': None,\n",
       " 'created_at': 'Sun Jan 23 00:00:01 +0000 2011',\n",
       " 'favorited': False,\n",
       " 'geo': None,\n",
       " 'id': 28965135238303744,\n",
       " 'id_str': '28965135238303744',\n",
       " 'in_reply_to_screen_name': 'BeasBookNook',\n",
       " 'in_reply_to_status_id': 28962147899805696,\n",
       " 'in_reply_to_status_id_str': '28962147899805696',\n",
       " 'in_reply_to_user_id': 47618028,\n",
       " 'in_reply_to_user_id_str': '47618028',\n",
       " 'place': None,\n",
       " 'retweet_count': 0,\n",
       " 'retweeted': False,\n",
       " 'source': '<a href=\"http://www.tweetdeck.com\" rel=\"nofollow\">TweetDeck</a>',\n",
       " 'text': \"@BeasBookNook every ARC I've ever read hasn't had final copy edits. Isn't that the nature of the beast? I guess some are better than others.\",\n",
       " 'truncated': False,\n",
       " 'user': {'contributors_enabled': False,\n",
       "  'created_at': 'Tue Apr 03 18:05:30 +0000 2007',\n",
       "  'default_profile': False,\n",
       "  'default_profile_image': False,\n",
       "  'description': 'Truth be told, I think I thrive under a lack of accountability. /Michael Scott',\n",
       "  'favourites_count': 0,\n",
       "  'follow_request_sent': None,\n",
       "  'followers_count': 95,\n",
       "  'following': None,\n",
       "  'friends_count': 168,\n",
       "  'geo_enabled': False,\n",
       "  'id': 3345521,\n",
       "  'id_str': '3345521',\n",
       "  'is_translator': False,\n",
       "  'lang': 'en',\n",
       "  'listed_count': 11,\n",
       "  'location': 'Vancouver Island, BC',\n",
       "  'name': 'Heller',\n",
       "  'notifications': None,\n",
       "  'profile_background_color': 'ffffff',\n",
       "  'profile_background_image_url': 'http://a0.twimg.com/profile_background_images/21939138/Roboto3_vectorized.png',\n",
       "  'profile_background_tile': False,\n",
       "  'profile_image_url': 'http://a3.twimg.com/profile_images/142506464/robot_normal.jpg',\n",
       "  'profile_link_color': '000000',\n",
       "  'profile_sidebar_border_color': '000000',\n",
       "  'profile_sidebar_fill_color': '215ca2',\n",
       "  'profile_text_color': '000000',\n",
       "  'profile_use_background_image': True,\n",
       "  'protected': False,\n",
       "  'screen_name': 'Heller',\n",
       "  'show_all_inline_media': False,\n",
       "  'statuses_count': 2245,\n",
       "  'time_zone': 'Pacific Time (US & Canada)',\n",
       "  'url': None,\n",
       "  'utc_offset': -28800,\n",
       "  'verified': False}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets=[]\n",
    "for line in open(\"20110123-000.json\", 'r'):\n",
    "    try:\n",
    "        tweets.append(json.loads(line))\n",
    "    except:\n",
    "        continue\n",
    "        \n",
    "tweets[14]        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_tweets= []\n",
    "for line in open(\"20110123-006.json\", 'r'):\n",
    "    try:\n",
    "        test_tweets.append(json.loads(line))\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Making language model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.util import ngrams\n",
    "import string\n",
    "from nltk import SimpleGoodTuringProbDist\n",
    "from math import log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<PlaintextCorpusReader in 'C:\\\\Users\\\\Sudhanshu\\\\Downloads\\\\twitter_bot'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<PlaintextCorpusReader in 'C:\\\\Users\\\\Sudhanshu\\\\Downloads\\\\twitter_bot'>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus = nltk.corpus.reader.plaintext.PlaintextCorpusReader(\".\", \"sample.txt\")\n",
    "test = nltk.corpus.reader.plaintext.PlaintextCorpusReader(\".\", \"test_data.txt\")\n",
    "print(test)\n",
    "corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unigram perplexity with add 1 smoothing\\ Laplacian smoothing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43784"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unigram= [w.strip(string.punctuation).lower() for w in corpus.words()]\n",
    "len(unigram)\n",
    "#unigram[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "spl= int(95*len(unigram)/100)\n",
    "train_corpus= unigram[:spl]\n",
    "test_corpus= unigram[spl:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "freq_1gram= nltk.FreqDist(train_corpus)\n",
    "len_train= len(train_corpus)\n",
    "vocab= len(set(train_corpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def unigram_prob_with_add1smoothing(word):\n",
    "    return (freq_1gram[word]+1)/(len_train+vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def entropy(n, text):\n",
    "        e = 0.0\n",
    "        text = [\"<s>\"] + text + [\"</s>\"]\n",
    "        for i in range(n - 1, len(text)):\n",
    "            context = text[i - n + 1:i]\n",
    "            token = text[i]\n",
    "            #print(str(context)+\"    \"+token)\n",
    "            e += logprob(token, context)\n",
    "        return e / float(len(text) - (n - 1))\n",
    "\n",
    "\n",
    "def logprob(word, context):\n",
    "    if len(context)==0:\n",
    "        p=unigram_prob_with_add1smoothing(word)\n",
    "    elif len(context)==1:\n",
    "        p=bigram_prob_with_add1smoothing(context[0], word)\n",
    "    else:\n",
    "        p=trigram_prob_with_add1smoothing(context[0], context[1], word)\n",
    "    return -p*log(p , 2)\n",
    "\n",
    "\n",
    "def perplexity(n, text):\n",
    "      return pow(2.0, entropy(n, text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.024138089875898"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(1, test_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.025133170322091"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(1, train_corpus)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bigram Perplexity with add 1 smoothing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43785"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigram= [a for a in ngrams(unigram, 2, pad_left=True,pad_right=True,left_pad_symbol='<s>', right_pad_symbol=\"</s>\")]\n",
    "len(bigram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cfreq_2gram = nltk.ConditionalFreqDist(bigram)\n",
    "cprob_2gram = nltk.ConditionalProbDist(cfreq_2gram, nltk.MLEProbDist)\n",
    "def bigram_prob_with_add1smoothing(word1, word2):\n",
    "    cprob_2gram_add1=float((((1+cfreq_2gram[word1][word2])/(len(cfreq_2gram)+sum(cfreq_2gram[word1].values())))))\n",
    "    return cprob_2gram_add1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0081485615099057"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(2, test_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0085341354101802"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(2, train_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bigram_prob_with_gtsmoothing(word1, word2):\n",
    "    return SimpleGoodTuringProbDist(word1, word2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def entropy(n, text):\n",
    "        e = 0.0\n",
    "        text = [\"<s>\"] + text + [\"</s>\"]\n",
    "        for i in range(n - 1, len(text)):\n",
    "            context = text[i - n + 1:i]\n",
    "            token = text[i]\n",
    "            #print(str(context)+\"    \"+token)\n",
    "            e += logprob_gt(token, context)\n",
    "        return e / float(len(text) - (n - 1))\n",
    "\n",
    "\n",
    "def logprob_gt(word, context):\n",
    "    if len(context)==0:\n",
    "        p=unigram_prob_with_gtsmoothing(word)\n",
    "    elif len(context)==1:\n",
    "        p=bigram_prob_with_gtsmoothing(context[0], word)\n",
    "    else:\n",
    "        p=trigram_prob_with_gtsmoothing(context[0], context[1], word)\n",
    "    return -p*log(p , 2)\n",
    "\n",
    "\n",
    "def perplexity(n, text):\n",
    "      return pow(2.0, entropy(n, text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trigram perplexity with add 1 smoothing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43786"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trigram= [a for a in ngrams(unigram, 3, pad_left= True, pad_right=True, left_pad_symbol= \"<s>\", right_pad_symbol=\"</s>\")]\n",
    "len(trigram)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trigrams_as_bigrams=[]\n",
    "trigrams_as_bigrams.extend([((t[0],t[1]), t[2]) for t in trigram])\n",
    "cfreq_3gram = nltk.ConditionalFreqDist(trigrams_as_bigrams)\n",
    "cprob_3gram = nltk.ConditionalProbDist(cfreq_3gram, nltk.MLEProbDist)\n",
    "def trigram_prob_with_add1smoothing(w1, w2, w3):\n",
    "    cprob_3gram_add1=float((((1+cfreq_3gram[(w1,w2)][w3])/(len(cfreq_3gram)+sum(cfreq_3gram[(w1,w2)].values())))))\n",
    "    return cprob_3gram_add1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0007427952147212"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(3, test_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.000768607503031"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perplexity(3, train_corpus)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trying to generate text from corpus:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter, OrderedDict, defaultdict\n",
    "from operator import itemgetter\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_unigram = [w.strip(string.punctuation).lower() for w in test.words()]\n",
    "\n",
    "test_bigram= [a for a in ngrams(test_unigram, 2)]\n",
    "len(test_bigram)\n",
    "\n",
    "test_trigram= [a for a in ngrams(test_unigram, 3)]\n",
    "len(test_trigram)\n",
    "\n",
    "test_frequency= nltk.FreqDist(test_bigram)\n",
    "test_sorted_freq= OrderedDict(reversed(sorted(test_frequency.items(), key= itemgetter(1))))\n",
    "\n",
    "test_cfreq_2gram= nltk.ConditionalFreqDist(test_bigram)\n",
    "\n",
    "test_cprob_2gram= nltk.ConditionalProbDist(test_cfreq_2gram, nltk.MLEProbDist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "counts = Counter(corpus.words())\n",
    "total_count = len(corpus.words())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('I', 1173),\n",
       " ('a', 1152),\n",
       " ('to', 940),\n",
       " ('the', 913),\n",
       " ('you', 652),\n",
       " ('de', 560),\n",
       " ('me', 537),\n",
       " ('in', 463),\n",
       " ('is', 427),\n",
       " ('s', 420),\n",
       " ('it', 415),\n",
       " ('my', 413),\n",
       " ('and', 407),\n",
       " ('of', 374),\n",
       " ('on', 373),\n",
       " ('no', 348),\n",
       " ('for', 344),\n",
       " ('t', 305),\n",
       " ('that', 291),\n",
       " ('o', 290)]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counts.most_common(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for word in counts:\n",
    "    counts[word] /= float(total_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bien to Beck many in Messing same you t don China t candy day the here want ser color everywhere I chain anda tan only the follow de swag min june ko knock monte now to and de dis can Proper bien de an mano is we think the have Raise get stupid you DaDa think o match by of die eu This have that ask best our Q em I SOLO shopping re but t that weekly as now pic selling Yo Broken amor I fam BONG de take best de D para outside r fail feel a file\n"
     ]
    }
   ],
   "source": [
    "# generating 100 words at random of the language\n",
    "text=[]\n",
    "for _ in range(100):\n",
    "    r = random.random()\n",
    "    accumulator = .0\n",
    " \n",
    "    for word, freq in counts.items():\n",
    "        accumulator += freq\n",
    " \n",
    "        if accumulator >= r:\n",
    "            text.append(word)\n",
    "            break\n",
    "print(' '.join(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = defaultdict(lambda: defaultdict(lambda : 0))\n",
    "for sentence in corpus.sents():\n",
    "    for w1, w2, w3 in ngrams(sentence, 3, pad_left= True, pad_right=True):\n",
    "        model[(w1, w2)][w3] +=1\n",
    "\n",
    "#transform counts to probabilities\n",
    "for w1_w2 in model:\n",
    "    total_count = float(sum(model[w1_w2].values()))\n",
    "    for w3 in model[w1_w2]:\n",
    "        model[w1_w2][w3] /= total_count\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text = [None, None]\n",
    "sentence_finished = False\n",
    "while not sentence_finished:\n",
    "    r= random.random()\n",
    "    accumulator = 0.0\n",
    "    for word in model[tuple(text[-2:])].keys():\n",
    "        accumulator += model[tuple(text[-2:])][word]\n",
    "        if(accumulator > r):\n",
    "            text.append(word)\n",
    "            break\n",
    "        if(text[-2:] == [None, None]):\n",
    "            sentence_finished = True\n",
    "            \n",
    "print(\" \".join([t for t in text if t]))            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}