ngram_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ab8c302d-65eb-4a99-b7b4-fac1333986bd",
   "metadata": {},
   "source": [
    "# Course 2 - N-grams\n",
    "\n",
    "## Import librairies "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "4a9b0544-e1af-43c0-beaf-a19b047f9895",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "from collections import Counter, defaultdict\n",
    "import math\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "902dd652-441b-439e-90fd-3c9c3ad0ef93",
   "metadata": {},
   "source": [
    "## Import dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "c9688d96-218a-4bee-b80a-a87ec677dafb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of dream reports in the training set: 27489\n",
      "Number of dream reports in the test set: 463\n"
     ]
    }
   ],
   "source": [
    "dataset = load_dataset(\"gustavecortal/DreamBank-annotated\")\n",
    "df = dataset[\"train\"].to_pandas()\n",
    "df[\"report\"] = df[\"report\"].astype(str)\n",
    "\n",
    "train_df = df.query(\"id != 'vietnam_vet3'\")\n",
    "train_list = list(train_df[\"report\"])\n",
    "print(\"Number of dream reports in the training set:\",len(train_list))\n",
    "\n",
    "test_df = df.query(\"id == 'vietnam_vet3'\")\n",
    "test_list = list(test_df[\"report\"])\n",
    "print(\"Number of dream reports in the test set:\",len(test_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "4474ec65-c86c-404a-9203-d16bfe06f79f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of tokens in the training set: 5538015\n",
      "Vocabulary size: 52195\n",
      "Number of tokens in the test set: 78254\n"
     ]
    }
   ],
   "source": [
    "train_strings = \" \".join(train_list)\n",
    "train_strings = train_strings.lower()\n",
    "train_tokens = word_tokenize(train_strings)\n",
    "print(\"Number of tokens in the training set:\",len(train_tokens))\n",
    "\n",
    "vocab = set(train_tokens)\n",
    "print(\"Vocabulary size:\",len(vocab))\n",
    "\n",
    "test_strings = \" \".join(test_list)\n",
    "test_strings = test_strings.lower()\n",
    "test_tokens = word_tokenize(test_strings)\n",
    "print(\"Number of tokens in the test set:\",len(test_tokens))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f654303f-e7e4-47bc-8c39-1ce6c21704be",
   "metadata": {},
   "source": [
    "## Train n-grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "3f8cac30-fbdb-4aa0-9560-fe4a52df569a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(text):\n",
    "    \"\"\"Tokenize the input text.\"\"\"\n",
    "    \n",
    "    return word_tokenize(text)\n",
    "\n",
    "def count_ngrams(tokens, n):\n",
    "    \"\"\"Counts n-grams.\"\"\"\n",
    "    \n",
    "    ngrams = [tuple(tokens[i:i+n]) for i in range(len(tokens)-n+1)]\n",
    "    \n",
    "    return Counter(ngrams)\n",
    "\n",
    "def calculate_ngram_probabilities(train_tokens, n, test_tokens, k=0.00001):\n",
    "    \"\"\"Calculates n-gram probabilities.\"\"\"\n",
    "    \n",
    "    vocab = set(train_tokens)\n",
    "    V = len(vocab)\n",
    "    ngram_counts = count_ngrams(train_tokens, n)\n",
    "    n_minus_one_gram_counts = count_ngrams(train_tokens, n-1)\n",
    "    ngram_probabilities = defaultdict(float)\n",
    "    \n",
    "    for ngram in ngram_counts:\n",
    "        prefix = ngram[:-1]\n",
    "        ngram_counts[ngram] += k\n",
    "        n_minus_one_gram_counts[prefix] += k\n",
    "        ngram_probabilities[ngram] = (ngram_counts[ngram] + k) / (n_minus_one_gram_counts[prefix] + k*V)\n",
    "\n",
    "    for i in range(len(test_tokens)-n+1):\n",
    "        ngram = tuple(test_tokens[i:i+n])\n",
    "        if ngram not in ngram_counts:\n",
    "            ngram_counts[ngram] = k\n",
    "            prefix = ngram[:-1]\n",
    "            if prefix not in n_minus_one_gram_counts:\n",
    "                n_minus_one_gram_counts[prefix] = k\n",
    "            ngram_probabilities[ngram] = (ngram_counts[ngram] + k) / (n_minus_one_gram_counts[prefix] + k*V)\n",
    "    \n",
    "    return ngram_probabilities"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3179fa6a-8f25-40fa-8f39-77b0c9126d05",
   "metadata": {},
   "source": [
    "## Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "c8b98ee6-1265-497c-bbb0-4765bd474946",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of 5-grams: 4864073\n"
     ]
    }
   ],
   "source": [
    "n = 5\n",
    "\n",
    "ngram_probabilities = calculate_ngram_probabilities(train_tokens, n, test_tokens)\n",
    "print(f\"Number of {n}-grams:\",len(ngram_probabilities))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "a4880c9d-0247-4ef1-86e6-80e836150339",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(float,\n",
       "            {('the', 'one', 'at', 'the', 'meads'): 0.1329467319687954,\n",
       "             ('one', 'at', 'the', 'meads', \"'s\"): 0.6570606323425058,\n",
       "             ('at', 'the', 'meads', \"'s\", 'house'): 0.6570606323425058,\n",
       "             ('the', 'meads', \"'s\", 'house', ','): 0.6570606323425058,\n",
       "             ('meads', \"'s\", 'house', ',', 'where'): 0.6570606323425058,\n",
       "             (\"'s\", 'house', ',', 'where', 'it'): 0.1050224953686006,\n",
       "             ('house', ',', 'where', 'it', \"'s\"): 0.6570606323425058,\n",
       "             (',', 'where', 'it', \"'s\", 'bigger'): 0.11734624429121941,\n",
       "             ('where', 'it', \"'s\", 'bigger', 'inside'): 0.6570606323425058,\n",
       "             ('it', \"'s\", 'bigger', 'inside', 'than'): 0.6570606323425058,\n",
       "             (\"'s\", 'bigger', 'inside', 'than', 'out'): 0.6570606323425058,\n",
       "             ('bigger', 'inside', 'than', 'out', ';'): 0.6570606323425058,\n",
       "             ('inside', 'than', 'out', ';', 'there'): 0.6570606323425058,\n",
       "             ('than', 'out', ';', 'there', \"'s\"): 0.6570606323425058,\n",
       "             ('out', ';', 'there', \"'s\", 'a'): 0.6570606323425058,\n",
       "             (';', 'there', \"'s\", 'a', 'european'): 0.39652492505828807,\n",
       "             ('there', \"'s\", 'a', 'european', 'village'): 0.6570606323425058,\n",
       "             (\"'s\", 'a', 'european', 'village', 'just'): 0.6570606323425058,\n",
       "             ('a',\n",
       "              'european',\n",
       "              'village',\n",
       "              'just',\n",
       "              'inside'): 0.6570606323425058,\n",
       "             ('european',\n",
       "              'village',\n",
       "              'just',\n",
       "              'inside',\n",
       "              ','): 0.6570606323425058,\n",
       "             ('village', 'just', 'inside', ',', 'with'): 0.6570606323425058,\n",
       "             ('just', 'inside', ',', 'with', 'a'): 0.6570606323425058,\n",
       "             ('inside', ',', 'with', 'a', 'cobblestone'): 0.28393848879601136,\n",
       "             (',', 'with', 'a', 'cobblestone', 'street'): 0.6570606323425058,\n",
       "             ('with', 'a', 'cobblestone', 'street', 'and'): 0.6570606323425058,\n",
       "             ('a', 'cobblestone', 'street', 'and', 'a'): 0.6570606323425058,\n",
       "             ('cobblestone',\n",
       "              'street',\n",
       "              'and',\n",
       "              'a',\n",
       "              'pied-piper'): 0.6570606323425058,\n",
       "             ('street', 'and', 'a', 'pied-piper', 'sort'): 0.6570606323425058,\n",
       "             ('and', 'a', 'pied-piper', 'sort', 'of'): 0.6570606323425058,\n",
       "             ('a', 'pied-piper', 'sort', 'of', 'man'): 0.6570606323425058,\n",
       "             ('pied-piper', 'sort', 'of', 'man', 'with'): 0.6570606323425058,\n",
       "             ('sort', 'of', 'man', 'with', 'curly'): 0.6570606323425058,\n",
       "             ('of', 'man', 'with', 'curly', 'hair'): 0.6570606323425058,\n",
       "             ('man', 'with', 'curly', 'hair', ','): 0.6570606323425058,\n",
       "             ('with', 'curly', 'hair', ',', 'he'): 0.39652492505828807,\n",
       "             ('curly', 'hair', ',', 'he', 'can'): 0.6570606323425058,\n",
       "             ('hair', ',', 'he', 'can', 'do'): 0.6570606323425058,\n",
       "             (',', 'he', 'can', 'do', 'things'): 0.6570606323425058,\n",
       "             ('he', 'can', 'do', 'things', 'like'): 0.39652492505828807,\n",
       "             ('can', 'do', 'things', 'like', 'juggle'): 0.6570606323425058,\n",
       "             ('do', 'things', 'like', 'juggle', '-'): 0.6570606323425058,\n",
       "             ('things', 'like', 'juggle', '-', 'i'): 0.6570606323425058,\n",
       "             ('like', 'juggle', '-', 'i', 'go'): 0.6570606323425058,\n",
       "             ('juggle', '-', 'i', 'go', 'up'): 0.6570606323425058,\n",
       "             ('-', 'i', 'go', 'up', 'the'): 0.6570606323425058,\n",
       "             ('i', 'go', 'up', 'the', 'back'): 0.19008055533379714,\n",
       "             ('go', 'up', 'the', 'back', 'stairs'): 0.44229051119426094,\n",
       "             ('up', 'the', 'back', 'stairs', '['): 0.39652492505828807,\n",
       "             ('the', 'back', 'stairs', '[', 'there'): 0.6570606323425058,\n",
       "             ('back', 'stairs', '[', 'there', 'are'): 0.6570606323425058,\n",
       "             ('stairs', '[', 'there', 'are', \"n't\"): 0.6570606323425058,\n",
       "             ('[', 'there', 'are', \"n't\", 'any'): 0.6570606323425058,\n",
       "             ('there', 'are', \"n't\", 'any', 'in'): 0.08679252488291923,\n",
       "             ('are', \"n't\", 'any', 'in', 'the'): 0.6570606323425058,\n",
       "             (\"n't\", 'any', 'in', 'the', 'real'): 0.39652492505828807,\n",
       "             ('any', 'in', 'the', 'real', 'house'): 0.6570606323425058,\n",
       "             ('in', 'the', 'real', 'house', ']'): 0.39652492505828807,\n",
       "             ('the', 'real', 'house', ']', 'and'): 0.6570606323425058,\n",
       "             ('real', 'house', ']', 'and', 'then'): 0.6570606323425058,\n",
       "             ('house', ']', 'and', 'then', 'down'): 0.6570606323425058,\n",
       "             (']', 'and', 'then', 'down', 'the'): 0.6570606323425058,\n",
       "             ('and', 'then', 'down', 'the', 'other'): 0.39652492505828807,\n",
       "             ('then', 'down', 'the', 'other', 'side'): 0.6570606323425058,\n",
       "             ('down', 'the', 'other', 'side', '['): 0.15333120718311674,\n",
       "             ('the', 'other', 'side', '[', 'since'): 0.6570606323425058,\n",
       "             ('other', 'side', '[', 'since', 'there'): 0.6570606323425058,\n",
       "             ('side', '[', 'since', 'there', \"'s\"): 0.6570606323425058,\n",
       "             ('[', 'since', 'there', \"'s\", 'a'): 0.6570606323425058,\n",
       "             ('since', 'there', \"'s\", 'a', 'second'): 0.6570606323425058,\n",
       "             ('there', \"'s\", 'a', 'second', 'set'): 0.6570606323425058,\n",
       "             (\"'s\", 'a', 'second', 'set', ','): 0.6570606323425058,\n",
       "             ('a', 'second', 'set', ',', 'immediately'): 0.6570606323425058,\n",
       "             ('second', 'set', ',', 'immediately', ']'): 0.6570606323425058,\n",
       "             ('set', ',', 'immediately', ']', 'then'): 0.6570606323425058,\n",
       "             (',', 'immediately', ']', 'then', 'down'): 0.6570606323425058,\n",
       "             ('immediately', ']', 'then', 'down', 'a'): 0.6570606323425058,\n",
       "             (']', 'then', 'down', 'a', 'short'): 0.6570606323425058,\n",
       "             ('then', 'down', 'a', 'short', 'empty'): 0.6570606323425058,\n",
       "             ('down', 'a', 'short', 'empty', 'hallway'): 0.6570606323425058,\n",
       "             ('a', 'short', 'empty', 'hallway', 'that'): 0.6570606323425058,\n",
       "             ('short',\n",
       "              'empty',\n",
       "              'hallway',\n",
       "              'that',\n",
       "              'turns'): 0.6570606323425058,\n",
       "             ('empty', 'hallway', 'that', 'turns', 'a'): 0.6570606323425058,\n",
       "             ('hallway', 'that', 'turns', 'a', 'corner'): 0.6570606323425058,\n",
       "             ('that', 'turns', 'a', 'corner', ','): 0.6570606323425058,\n",
       "             ('turns', 'a', 'corner', ',', 'where'): 0.6570606323425058,\n",
       "             ('a', 'corner', ',', 'where', 'i'): 0.5678712989358199,\n",
       "             ('corner', ',', 'where', 'i', 'find'): 0.39652492505828807,\n",
       "             (',', 'where', 'i', 'find', 'a'): 0.22114746702757215,\n",
       "             ('where', 'i', 'find', 'a', 'tiny'): 0.22114746702757215,\n",
       "             ('i', 'find', 'a', 'tiny', 'room'): 0.36219385870234483,\n",
       "             ('find', 'a', 'tiny', 'room', '...'): 0.39652492505828807,\n",
       "             ('a', 'tiny', 'room', '...', 'a'): 0.6570606323425058,\n",
       "             ('tiny', 'room', '...', 'a', 'young'): 0.6570606323425058,\n",
       "             ('room', '...', 'a', 'young', 'woman'): 0.6570606323425058,\n",
       "             ('...', 'a', 'young', 'woman', 'with'): 0.6570606323425058,\n",
       "             ('a',\n",
       "              'young',\n",
       "              'woman',\n",
       "              'with',\n",
       "              'shoulder-length'): 0.08679252488291923,\n",
       "             ('young',\n",
       "              'woman',\n",
       "              'with',\n",
       "              'shoulder-length',\n",
       "              'blonde'): 0.6570606323425058,\n",
       "             ('woman',\n",
       "              'with',\n",
       "              'shoulder-length',\n",
       "              'blonde',\n",
       "              'hair'): 0.6570606323425058,\n",
       "             ('with',\n",
       "              'shoulder-length',\n",
       "              'blonde',\n",
       "              'hair',\n",
       "              'in'): 0.39652492505828807,\n",
       "             ('shoulder-length',\n",
       "              'blonde',\n",
       "              'hair',\n",
       "              'in',\n",
       "              'a'): 0.6570606323425058,\n",
       "             ('blonde', 'hair', 'in', 'a', 'pageboy'): 0.22114746702757215,\n",
       "             ('hair', 'in', 'a', 'pageboy', 'is'): 0.6570606323425058,\n",
       "             ('in', 'a', 'pageboy', 'is', 'there'): 0.6570606323425058,\n",
       "             ('a', 'pageboy', 'is', 'there', ','): 0.6570606323425058,\n",
       "             ('pageboy', 'is', 'there', ',', 'cooking'): 0.6570606323425058,\n",
       "             ('is', 'there', ',', 'cooking', 'at'): 0.39652492505828807,\n",
       "             ('there', ',', 'cooking', 'at', 'a'): 0.6570606323425058,\n",
       "             (',', 'cooking', 'at', 'a', 'stove'): 0.6570606323425058,\n",
       "             ('cooking', 'at', 'a', 'stove', 'that'): 0.6570606323425058,\n",
       "             ('at', 'a', 'stove', 'that', 'almost'): 0.6570606323425058,\n",
       "             ('a', 'stove', 'that', 'almost', 'fills'): 0.6570606323425058,\n",
       "             ('stove', 'that', 'almost', 'fills', 'the'): 0.6570606323425058,\n",
       "             ('that', 'almost', 'fills', 'the', 'room'): 0.6570606323425058,\n",
       "             ('almost', 'fills', 'the', 'room', '...'): 0.6570606323425058,\n",
       "             ('fills', 'the', 'room', '...', 'she'): 0.6570606323425058,\n",
       "             ('the', 'room', '...', 'she', \"'s\"): 0.6570606323425058,\n",
       "             ('room', '...', 'she', \"'s\", 'nice'): 0.6570606323425058,\n",
       "             ('...', 'she', \"'s\", 'nice', 'to'): 0.6570606323425058,\n",
       "             ('she', \"'s\", 'nice', 'to', 'me'): 0.6570606323425058,\n",
       "             (\"'s\", 'nice', 'to', 'me', '.'): 0.6570606323425058,\n",
       "             ('nice', 'to', 'me', '.', 'now'): 0.07395525500741018,\n",
       "             ('to', 'me', '.', 'now', 'outside'): 0.18109874030235643,\n",
       "             ('me', '.', 'now', 'outside', ','): 0.6570606323425058,\n",
       "             ('.', 'now', 'outside', ',', 'i'): 0.6570606323425058,\n",
       "             ('now', 'outside', ',', 'i', \"'m\"): 0.6570606323425058,\n",
       "             ('outside', ',', 'i', \"'m\", 'waiting'): 0.28393848879601136,\n",
       "             (',', 'i', \"'m\", 'waiting', 'for'): 0.39652492505828807,\n",
       "             ('i', \"'m\", 'waiting', 'for', 'my'): 0.23509244587798112,\n",
       "             (\"'m\", 'waiting', 'for', 'my', 'aunt'): 0.15333120718311674,\n",
       "             ('waiting', 'for', 'my', 'aunt', 'to'): 0.6570606323425058,\n",
       "             ('for', 'my', 'aunt', 'to', 'pick'): 0.6570606323425058,\n",
       "             ('my', 'aunt', 'to', 'pick', 'me'): 0.6570606323425058,\n",
       "             ('aunt', 'to', 'pick', 'me', 'up'): 0.6570606323425058,\n",
       "             ('to', 'pick', 'me', 'up', '-'): 0.012899828642103477,\n",
       "             ('pick', 'me', 'up', '-', 'she'): 0.6570606323425058,\n",
       "             ('me', 'up', '-', 'she', 'arrives'): 0.6570606323425058,\n",
       "             ('up', '-', 'she', 'arrives', 'in'): 0.6570606323425058,\n",
       "             ('-', 'she', 'arrives', 'in', 'a'): 0.6570606323425058,\n",
       "             ('she', 'arrives', 'in', 'a', 'little'): 0.6570606323425058,\n",
       "             ('arrives', 'in', 'a', 'little', 'round'): 0.6570606323425058,\n",
       "             ('in', 'a', 'little', 'round', 'convertible'): 0.6570606323425058,\n",
       "             ('a',\n",
       "              'little',\n",
       "              'round',\n",
       "              'convertible',\n",
       "              'and'): 0.6570606323425058,\n",
       "             ('little',\n",
       "              'round',\n",
       "              'convertible',\n",
       "              'and',\n",
       "              'we'): 0.6570606323425058,\n",
       "             ('round', 'convertible', 'and', 'we', 'go'): 0.6570606323425058,\n",
       "             ('convertible', 'and', 'we', 'go', 'for'): 0.6570606323425058,\n",
       "             ('and', 'we', 'go', 'for', 'a'): 0.8845765995276383,\n",
       "             ('we', 'go', 'for', 'a', 'drive'): 0.22186280687119325,\n",
       "             ('go', 'for', 'a', 'drive', ','): 0.11734624429121941,\n",
       "             ('for', 'a', 'drive', ',', 'not'): 0.22114746702757215,\n",
       "             ('a', 'drive', ',', 'not', 'very'): 0.6570606323425058,\n",
       "             ('drive', ',', 'not', 'very', 'far'): 0.6570606323425058,\n",
       "             (',', 'not', 'very', 'far', '-'): 0.28393848879601136,\n",
       "             ('not', 'very', 'far', '-', 'we'): 0.6570606323425058,\n",
       "             ('very', 'far', '-', 'we', 'cross'): 0.6570606323425058,\n",
       "             ('far', '-', 'we', 'cross', 'a'): 0.6570606323425058,\n",
       "             ('-', 'we', 'cross', 'a', 'little'): 0.6570606323425058,\n",
       "             ('we', 'cross', 'a', 'little', 'bridge'): 0.7930419197766817,\n",
       "             ('cross', 'a', 'little', 'bridge', 'over'): 0.39652492505828807,\n",
       "             ('a', 'little', 'bridge', 'over', 'a'): 0.6570606323425058,\n",
       "             ('little', 'bridge', 'over', 'a', 'creek'): 0.6570606323425058,\n",
       "             ('bridge', 'over', 'a', 'creek', ','): 0.6570606323425058,\n",
       "             ('over', 'a', 'creek', ',', 'then'): 0.6570606323425058,\n",
       "             ('a', 'creek', ',', 'then', 'double'): 0.6570606323425058,\n",
       "             ('creek', ',', 'then', 'double', 'back'): 0.6570606323425058,\n",
       "             (',', 'then', 'double', 'back', 'and'): 0.6570606323425058,\n",
       "             ('then', 'double', 'back', 'and', 'she'): 0.6570606323425058,\n",
       "             ('double', 'back', 'and', 'she', 'drops'): 0.6570606323425058,\n",
       "             ('back', 'and', 'she', 'drops', 'me'): 0.6570606323425058,\n",
       "             ('and', 'she', 'drops', 'me', 'off'): 0.6570606323425058,\n",
       "             ('she', 'drops', 'me', 'off', 'at'): 0.6570606323425058,\n",
       "             ('drops', 'me', 'off', 'at', 'the'): 0.7930419197766817,\n",
       "             ('me', 'off', 'at', 'the', 'house'): 0.22114746702757215,\n",
       "             ('off', 'at', 'the', 'house', 'again'): 0.6570606323425058,\n",
       "             ('at', 'the', 'house', 'again', '.'): 0.6570606323425058,\n",
       "             ('the', 'house', 'again', '.', 'inside'): 0.22114746702757215,\n",
       "             ('house', 'again', '.', 'inside', '('): 0.6570606323425058,\n",
       "             ('again', '.', 'inside', '(', '?'): 0.6570606323425058,\n",
       "             ('.', 'inside', '(', '?', ')'): 0.6570606323425058,\n",
       "             ('inside', '(', '?', ')', 'i'): 0.6570606323425058,\n",
       "             ('(', '?', ')', 'i', 'sit'): 0.18109874030235643,\n",
       "             ('?', ')', 'i', 'sit', 'with'): 0.6570606323425058,\n",
       "             (')', 'i', 'sit', 'with', 'a'): 0.6570606323425058,\n",
       "             ('i', 'sit', 'with', 'a', 'couple'): 0.1329467319687954,\n",
       "             ('sit', 'with', 'a', 'couple', 'of'): 0.6570606323425058,\n",
       "             ('with', 'a', 'couple', 'of', 'people'): 0.045954272280016806,\n",
       "             ('a', 'couple', 'of', 'people', ','): 0.11932536164353157,\n",
       "             ('couple', 'of', 'people', ',', 'playing'): 0.22114746702757215,\n",
       "             ('of', 'people', ',', 'playing', 'with'): 0.6570606323425058,\n",
       "             ('people', ',', 'playing', 'with', 'a'): 0.6570606323425058,\n",
       "             (',', 'playing', 'with', 'a', 'string'): 0.39652492505828807,\n",
       "             ('playing', 'with', 'a', 'string', 'of'): 0.6570606323425058,\n",
       "             ('with', 'a', 'string', 'of', 'blue'): 0.6570606323425058,\n",
       "             ('a', 'string', 'of', 'blue', 'balloons'): 0.6570606323425058,\n",
       "             ('string', 'of', 'blue', 'balloons', '.'): 0.6570606323425058,\n",
       "             ('of', 'blue', 'balloons', '.', 'i'): 0.6570606323425058,\n",
       "             ('blue', 'balloons', '.', 'i', \"'m\"): 0.6570606323425058,\n",
       "             ('balloons', '.', 'i', \"'m\", 'at'): 0.6570606323425058,\n",
       "             ('.', 'i', \"'m\", 'at', 'a'): 0.35154063497208077,\n",
       "             ('i', \"'m\", 'at', 'a', 'family'): 0.019948039606235642,\n",
       "             (\"'m\", 'at', 'a', 'family', 'reunion'): 0.22114746702757215,\n",
       "             ('at', 'a', 'family', 'reunion', 'in'): 0.22114746702757215,\n",
       "             ('a', 'family', 'reunion', 'in', 'a'): 0.6570606323425058,\n",
       "             ('family', 'reunion', 'in', 'a', 'large'): 0.39652492505828807,\n",
       "             ('reunion', 'in', 'a', 'large', 'fine'): 0.6570606323425058,\n",
       "             ('in', 'a', 'large', 'fine', 'house'): 0.6570606323425058,\n",
       "             ('a', 'large', 'fine', 'house', 'with'): 0.6570606323425058,\n",
       "             ('large', 'fine', 'house', 'with', 'grounds'): 0.6570606323425058,\n",
       "             ('fine', 'house', 'with', 'grounds', '('): 0.6570606323425058,\n",
       "             ('house', 'with', 'grounds', '(', 'or'): 0.6570606323425058,\n",
       "             ('with', 'grounds', '(', 'or', 'a'): 0.6570606323425058,\n",
       "             ('grounds', '(', 'or', 'a', 'maybe'): 0.6570606323425058,\n",
       "             ('(', 'or', 'a', 'maybe', 'a'): 0.6570606323425058,\n",
       "             ('or', 'a', 'maybe', 'a', 'rented'): 0.6570606323425058,\n",
       "             ('a', 'maybe', 'a', 'rented', 'public'): 0.6570606323425058,\n",
       "             ('maybe',\n",
       "              'a',\n",
       "              'rented',\n",
       "              'public',\n",
       "              'building'): 0.6570606323425058,\n",
       "             ('a', 'rented', 'public', 'building', 'like'): 0.6570606323425058,\n",
       "             ('rented',\n",
       "              'public',\n",
       "              'building',\n",
       "              'like',\n",
       "              'in'): 0.6570606323425058,\n",
       "             ('public', 'building', 'like', 'in', 'a'): 0.6570606323425058,\n",
       "             ('building', 'like', 'in', 'a', 'park'): 0.6570606323425058,\n",
       "             ('like', 'in', 'a', 'park', ')'): 0.6570606323425058,\n",
       "             ('in', 'a', 'park', ')', '-'): 0.6570606323425058,\n",
       "             ('a', 'park', ')', '-', 'maybe'): 0.6570606323425058,\n",
       "             ('park', ')', '-', 'maybe', 'it'): 0.6570606323425058,\n",
       "             (')', '-', 'maybe', 'it', \"'s\"): 0.6570606323425058,\n",
       "             ('-', 'maybe', 'it', \"'s\", 'granite'): 0.28393848879601136,\n",
       "             ('maybe', 'it', \"'s\", 'granite', 'or'): 0.6570606323425058,\n",
       "             ('it', \"'s\", 'granite', 'or', 'something'): 0.6570606323425058,\n",
       "             (\"'s\", 'granite', 'or', 'something', ','): 0.6570606323425058,\n",
       "             ('granite', 'or', 'something', ',', 'very'): 0.6570606323425058,\n",
       "             ('or', 'something', ',', 'very', 'imposing'): 0.18109874030235643,\n",
       "             ('something', ',', 'very', 'imposing', '.'): 0.6570606323425058,\n",
       "             (',', 'very', 'imposing', '.', 'i'): 0.6570606323425058,\n",
       "             ('very', 'imposing', '.', 'i', \"'m\"): 0.6570606323425058,\n",
       "             ('imposing', '.', 'i', \"'m\", 'on'): 0.6570606323425058,\n",
       "             ('.', 'i', \"'m\", 'on', 'a'): 0.4306227936486813,\n",
       "             ('i', \"'m\", 'on', 'a', 'little'): 0.025393411813819478,\n",
       "             (\"'m\", 'on', 'a', 'little', 'round'): 0.22114746702757215,\n",
       "             ('on', 'a', 'little', 'round', 'balcony'): 0.39652492505828807,\n",
       "             ('a', 'little', 'round', 'balcony', 'with'): 0.6570606323425058,\n",
       "             ('little', 'round', 'balcony', 'with', 'a'): 0.6570606323425058,\n",
       "             ('round', 'balcony', 'with', 'a', 'few'): 0.6570606323425058,\n",
       "             ('balcony', 'with', 'a', 'few', 'others'): 0.6570606323425058,\n",
       "             ('with', 'a', 'few', 'others', '('): 0.15333120718311674,\n",
       "             ('a', 'few', 'others', '(', 'i'): 0.5678712989358199,\n",
       "             ('few', 'others', '(', 'i', 'think'): 0.39652492505828807,\n",
       "             ('others', '(', 'i', 'think', 'men'): 0.6570606323425058,\n",
       "             ('(', 'i', 'think', 'men', ')'): 0.6570606323425058,\n",
       "             ('i', 'think', 'men', ')', ','): 0.6570606323425058,\n",
       "             ('think', 'men', ')', ',', 'there'): 0.6570606323425058,\n",
       "             ('men', ')', ',', 'there', 'are'): 0.6570606323425058,\n",
       "             (')', ',', 'there', 'are', 'lots'): 0.6570606323425058,\n",
       "             (',', 'there', 'are', 'lots', 'more'): 0.1329467319687954,\n",
       "             ('there', 'are', 'lots', 'more', 'people'): 0.39652492505828807,\n",
       "             ('are', 'lots', 'more', 'people', 'inside'): 0.6570606323425058,\n",
       "             ('lots', 'more', 'people', 'inside', '.'): 0.6570606323425058,\n",
       "             ('more', 'people', 'inside', '.', 'i'): 0.39652492505828807,\n",
       "             ('people', 'inside', '.', 'i', 'feel'): 0.18109874030235643,\n",
       "             ('inside', '.', 'i', 'feel', 'faint'): 0.18109874030235643,\n",
       "             ('.', 'i', 'feel', 'faint', 'and'): 0.6570606323425058,\n",
       "             ('i', 'feel', 'faint', 'and', 'collapse'): 0.6570606323425058,\n",
       "             ('feel', 'faint', 'and', 'collapse', '.'): 0.6570606323425058,\n",
       "             ('faint', 'and', 'collapse', '.', 'they'): 0.6570606323425058,\n",
       "             ('and', 'collapse', '.', 'they', 'see'): 0.6570606323425058,\n",
       "             ('collapse', '.', 'they', 'see', 'that'): 0.6570606323425058,\n",
       "             ('.', 'they', 'see', 'that', 'i'): 0.6570606323425058,\n",
       "             ('they', 'see', 'that', 'i', \"'m\"): 0.6570606323425058,\n",
       "             ('see', 'that', 'i', \"'m\", 'very'): 0.22114746702757215,\n",
       "             ('that', 'i', \"'m\", 'very', 'sick'): 0.15333120718311674,\n",
       "             ('i', \"'m\", 'very', 'sick', ','): 0.6570606323425058,\n",
       "             (\"'m\", 'very', 'sick', ',', 'and'): 0.6570606323425058,\n",
       "             ('very', 'sick', ',', 'and', 'quickly'): 0.6570606323425058,\n",
       "             ('sick', ',', 'and', 'quickly', 'take'): 0.6570606323425058,\n",
       "             (',', 'and', 'quickly', 'take', 'me'): 0.6570606323425058,\n",
       "             ('and', 'quickly', 'take', 'me', 'to'): 0.6570606323425058,\n",
       "             ('quickly', 'take', 'me', 'to', 'a'): 0.6570606323425058,\n",
       "             ('take', 'me', 'to', 'a', 'car'): 0.23469014170449054,\n",
       "             ('me', 'to', 'a', 'car', 'to'): 0.39652492505828807,\n",
       "             ('to', 'a', 'car', 'to', 'go'): 0.28393848879601136,\n",
       "             ('a', 'car', 'to', 'go', 'to'): 0.5432889771023333,\n",
       "             ('car', 'to', 'go', 'to', 'the'): 0.36219385870234483,\n",
       "             ('to', 'go', 'to', 'the', 'hospital'): 0.024412176354192044,\n",
       "             ('go', 'to', 'the', 'hospital', '.'): 0.12105222382816568,\n",
       "             ('to', 'the', 'hospital', '.', '('): 0.029831787878751723,\n",
       "             ('the', 'hospital', '.', '(', 'i'): 0.7930419197766817,\n",
       "             ('hospital', '.', '(', 'i', 'do'): 0.39652492505828807,\n",
       "             ('.', '(', 'i', 'do', \"n't\"): 0.8436219583020306,\n",
       "             ('(', 'i', 'do', \"n't\", 'see'): 0.00947688992888305,\n",
       "             ('i', 'do', \"n't\", 'see', 'any'): 0.18989776363252098,\n",
       "             ('do', \"n't\", 'see', 'any', 'of'): 0.051225389253947866,\n",
       "             (\"n't\", 'see', 'any', 'of', 'that'): 0.39652492505828807,\n",
       "             ('see', 'any', 'of', 'that', ','): 0.6570606323425058,\n",
       "             ('any', 'of', 'that', ',', 'the'): 0.22114746702757215,\n",
       "             ('of', 'that', ',', 'the', 'next'): 0.28393848879601136,\n",
       "             ('that', ',', 'the', 'next', 'i'): 0.6570606323425058,\n",
       "             (',', 'the', 'next', 'i', 'know'): 0.39652492505828807,\n",
       "             ('the', 'next', 'i', 'know', 'is'): 0.6570606323425058,\n",
       "             ('next', 'i', 'know', 'is', 'that'): 0.6570606323425058,\n",
       "             ('i', 'know', 'is', 'that', 'i'): 0.5207464702186085,\n",
       "             ('know', 'is', 'that', 'i', \"'m\"): 0.23469014170449054,\n",
       "             ('is', 'that', 'i', \"'m\", 'lying'): 0.05707238231339417,\n",
       "             ('that', 'i', \"'m\", 'lying', 'in'): 0.6570606323425058,\n",
       "             ('i', \"'m\", 'lying', 'in', 'the'): 0.15333120718311674,\n",
       "             (\"'m\", 'lying', 'in', 'the', 'back'): 0.6570606323425058,\n",
       "             ('lying', 'in', 'the', 'back', 'seat'): 0.39652492505828807,\n",
       "             ('in', 'the', 'back', 'seat', 'of'): 0.2890482153462596,\n",
       "             ('the', 'back', 'seat', 'of', 'the'): 0.29894308803339786,\n",
       "             ('back', 'seat', 'of', 'the', 'car'): 0.7868327970773443,\n",
       "             ('seat', 'of', 'the', 'car', '-'): 0.05399104630395488,\n",
       "             ('of', 'the', 'car', '-', 'a'): 0.6570606323425058,\n",
       "             ('the', 'car', '-', 'a', 'sedan'): 0.39652492505828807,\n",
       "             ('car', '-', 'a', 'sedan', '-'): 0.6570606323425058,\n",
       "             ('-', 'a', 'sedan', '-', 'going'): 0.6570606323425058,\n",
       "             ('a', 'sedan', '-', 'going', 'like'): 0.6570606323425058,\n",
       "             ('sedan',\n",
       "              '-',\n",
       "              'going',\n",
       "              'like',\n",
       "              'gangbusters'): 0.6570606323425058,\n",
       "             ('-', 'going', 'like', 'gangbusters', 'on'): 0.6570606323425058,\n",
       "             ('going', 'like', 'gangbusters', 'on', 'a'): 0.6570606323425058,\n",
       "             ('like', 'gangbusters', 'on', 'a', 'huge'): 0.6570606323425058,\n",
       "             ('gangbusters', 'on', 'a', 'huge', 'freeway'): 0.6570606323425058,\n",
       "             ('on', 'a', 'huge', 'freeway', '.'): 0.6570606323425058,\n",
       "             ('a', 'huge', 'freeway', '.', 'it'): 0.6570606323425058,\n",
       "             ('huge', 'freeway', '.', 'it', 'must'): 0.6570606323425058,\n",
       "             ('freeway', '.', 'it', 'must', 'be'): 0.6570606323425058,\n",
       "             ('.', 'it', 'must', 'be', '5'): 0.03918272734539197,\n",
       "             ('it', 'must', 'be', '5', 'lanes'): 0.6570606323425058,\n",
       "             ('must', 'be', '5', 'lanes', 'in'): 0.6570606323425058,\n",
       "             ('be', '5', 'lanes', 'in', 'each'): 0.6570606323425058,\n",
       "             ('5', 'lanes', 'in', 'each', 'direction'): 0.6570606323425058,\n",
       "             ('lanes', 'in', 'each', 'direction', ','): 0.6570606323425058,\n",
       "             ('in', 'each', 'direction', ',', 'lots'): 0.6570606323425058,\n",
       "             ('each', 'direction', ',', 'lots', 'of'): 0.6570606323425058,\n",
       "             ('direction', ',', 'lots', 'of', 'cars'): 0.6570606323425058,\n",
       "             (',', 'lots', 'of', 'cars', 'but'): 0.6570606323425058,\n",
       "             ('lots', 'of', 'cars', 'but', 'not'): 0.6570606323425058,\n",
       "             ('of', 'cars', 'but', 'not', 'crowded'): 0.6570606323425058,\n",
       "             ('cars', 'but', 'not', 'crowded', ','): 0.6570606323425058,\n",
       "             ('but', 'not', 'crowded', ',', 'and'): 0.6570606323425058,\n",
       "             ('not', 'crowded', ',', 'and', 'all'): 0.6570606323425058,\n",
       "             ('crowded', ',', 'and', 'all', 'going'): 0.39652492505828807,\n",
       "             (',', 'and', 'all', 'going', 'really'): 0.6570606323425058,\n",
       "             ('and', 'all', 'going', 'really', 'fast'): 0.6570606323425058,\n",
       "             ('all', 'going', 'really', 'fast', '.'): 0.6570606323425058,\n",
       "             ('going', 'really', 'fast', '.', 'i'): 0.8518041090756284,\n",
       "             ('really', 'fast', '.', 'i', 'ca'): 0.1050224953686006,\n",
       "             ('fast', '.', 'i', 'ca', \"n't\"): 0.6570606323425058,\n",
       "             ('.', 'i', 'ca', \"n't\", 'see'): 0.053085809850681696,\n",
       "             ('i', 'ca', \"n't\", 'see', 'anything'): 0.0497836244946564,\n",
       "             ('ca', \"n't\", 'see', 'anything', 'else'): 0.14790903093930172,\n",
       "             (\"n't\", 'see', 'anything', 'else', 'but'): 0.39652492505828807,\n",
       "             ('see', 'anything', 'else', 'but', 'freeway'): 0.6570606323425058,\n",
       "             ('anything', 'else', 'but', 'freeway', 'and'): 0.6570606323425058,\n",
       "             ('else', 'but', 'freeway', 'and', 'blue'): 0.6570606323425058,\n",
       "             ('but', 'freeway', 'and', 'blue', 'sky'): 0.6570606323425058,\n",
       "             ('freeway', 'and', 'blue', 'sky', ','): 0.6570606323425058,\n",
       "             ('and', 'blue', 'sky', ',', 'with'): 0.6570606323425058,\n",
       "             ('blue', 'sky', ',', 'with', 'a'): 0.6570606323425058,\n",
       "             ('sky', ',', 'with', 'a', 'few'): 0.6570606323425058,\n",
       "             (',', 'with', 'a', 'few', 'white'): 0.1050224953686006,\n",
       "             ('with', 'a', 'few', 'white', 'clouds'): 0.6570606323425058,\n",
       "             ('a', 'few', 'white', 'clouds', '.'): 0.6570606323425058,\n",
       "             ('few', 'white', 'clouds', '.', 'as'): 0.6570606323425058,\n",
       "             ('white', 'clouds', '.', 'as', 'we'): 0.6570606323425058,\n",
       "             ('clouds', '.', 'as', 'we', 'barrel'): 0.6570606323425058,\n",
       "             ('.', 'as', 'we', 'barrel', 'along'): 0.6570606323425058,\n",
       "             ('as', 'we', 'barrel', 'along', ','): 0.6570606323425058,\n",
       "             ('we', 'barrel', 'along', ',', 'the'): 0.6570606323425058,\n",
       "             ('barrel', 'along', ',', 'the', 'road'): 0.6570606323425058,\n",
       "             ('along', ',', 'the', 'road', 'rises'): 0.6570606323425058,\n",
       "             (',', 'the', 'road', 'rises', ','): 0.6570606323425058,\n",
       "             ('the', 'road', 'rises', ',', 'and'): 0.6570606323425058,\n",
       "             ('road', 'rises', ',', 'and', 'quickly'): 0.6570606323425058,\n",
       "             ('rises', ',', 'and', 'quickly', '-'): 0.6570606323425058,\n",
       "             (',', 'and', 'quickly', '-', 'soon'): 0.6570606323425058,\n",
       "             ('and', 'quickly', '-', 'soon', 'it'): 0.6570606323425058,\n",
       "             ('quickly', '-', 'soon', 'it', 'changes'): 0.6570606323425058,\n",
       "             ('-', 'soon', 'it', 'changes', 'to'): 0.6570606323425058,\n",
       "             ('soon', 'it', 'changes', 'to', 'steps'): 0.6570606323425058,\n",
       "             ('it', 'changes', 'to', 'steps', ','): 0.6570606323425058,\n",
       "             ('changes', 'to', 'steps', ',', 'which'): 0.6570606323425058,\n",
       "             ('to', 'steps', ',', 'which', 'does'): 0.6570606323425058,\n",
       "             ('steps', ',', 'which', 'does', \"n't\"): 0.6570606323425058,\n",
       "             (',', 'which', 'does', \"n't\", 'slow'): 0.05707238231339417,\n",
       "             ('which', 'does', \"n't\", 'slow', 'anybody'): 0.6570606323425058,\n",
       "             ('does', \"n't\", 'slow', 'anybody', 'down'): 0.6570606323425058,\n",
       "             (\"n't\", 'slow', 'anybody', 'down', ','): 0.6570606323425058,\n",
       "             ('slow', 'anybody', 'down', ',', 'and'): 0.6570606323425058,\n",
       "             ('anybody', 'down', ',', 'and', 'it'): 0.6570606323425058,\n",
       "             ('down', ',', 'and', 'it', 'keeps'): 0.22114746702757215,\n",
       "             (',', 'and', 'it', 'keeps', 'going'): 0.39652492505828807,\n",
       "             ('and', 'it', 'keeps', 'going', 'up'): 0.6570606323425058,\n",
       "             ('it', 'keeps', 'going', 'up', ','): 0.39652492505828807,\n",
       "             ('keeps', 'going', 'up', ',', 'the'): 0.6570606323425058,\n",
       "             ('going', 'up', ',', 'the', 'incline'): 0.6570606323425058,\n",
       "             ('up', ',', 'the', 'incline', 'increasing'): 0.6570606323425058,\n",
       "             (',', 'the', 'incline', 'increasing', '.'): 0.6570606323425058,\n",
       "             ('the', 'incline', 'increasing', '.', 'now'): 0.6570606323425058,\n",
       "             ('incline', 'increasing', '.', 'now', 'my'): 0.6570606323425058,\n",
       "             ('increasing', '.', 'now', 'my', 'line'): 0.6570606323425058,\n",
       "             ('.', 'now', 'my', 'line', 'of'): 0.6570606323425058,\n",
       "             ('now', 'my', 'line', 'of', 'sight'): 0.6570606323425058,\n",
       "             ('my', 'line', 'of', 'sight', 'changes'): 0.39652492505828807,\n",
       "             ('line', 'of', 'sight', 'changes', 'to'): 0.6570606323425058,\n",
       "             ('of', 'sight', 'changes', 'to', 'a'): 0.6570606323425058,\n",
       "             ('sight', 'changes', 'to', 'a', 'spot'): 0.6570606323425058,\n",
       "             ('changes', 'to', 'a', 'spot', 'at'): 0.6570606323425058,\n",
       "             ('to', 'a', 'spot', 'at', 'the'): 0.6570606323425058,\n",
       "             ('a', 'spot', 'at', 'the', 'side'): 0.6570606323425058,\n",
       "             ('spot', 'at', 'the', 'side', 'of'): 0.6570606323425058,\n",
       "             ('at', 'the', 'side', 'of', 'the'): 0.8971558554047067,\n",
       "             ('the', 'side', 'of', 'the', 'freeway'): 0.011816131514776765,\n",
       "             ('side', 'of', 'the', 'freeway', '('): 0.11734624429121941,\n",
       "             ('of', 'the', 'freeway', '(', 'outside'): 0.39652492505828807,\n",
       "             ('the', 'freeway', '(', 'outside', 'of'): 0.6570606323425058,\n",
       "             ('freeway', '(', 'outside', 'of', 'the'): 0.6570606323425058,\n",
       "             ('(', 'outside', 'of', 'the', 'car'): 0.6570606323425058,\n",
       "             ('outside', 'of', 'the', 'car', ')'): 0.18109874030235643,\n",
       "             ('of', 'the', 'car', ')', ','): 0.39652492505828807,\n",
       "             ('the', 'car', ')', ',', 'looking'): 0.39652492505828807,\n",
       "             ('car', ')', ',', 'looking', 'across'): 0.6570606323425058,\n",
       "             (')', ',', 'looking', 'across', 'it'): 0.6570606323425058,\n",
       "             (',', 'looking', 'across', 'it', '-'): 0.6570606323425058,\n",
       "             ('looking', 'across', 'it', '-', 'and'): 0.6570606323425058,\n",
       "             ('across', 'it', '-', 'and', 'the'): 0.6570606323425058,\n",
       "             ('it', '-', 'and', 'the', 'incline'): 0.28393848879601136,\n",
       "             ('-', 'and', 'the', 'incline', '('): 0.6570606323425058,\n",
       "             ('and', 'the', 'incline', '(', 'without'): 0.6570606323425058,\n",
       "             ('the', 'incline', '(', 'without', 'the'): 0.6570606323425058,\n",
       "             ('incline', '(', 'without', 'the', 'steps'): 0.6570606323425058,\n",
       "             ('(', 'without', 'the', 'steps', ')'): 0.7930419197766817,\n",
       "             ('without', 'the', 'steps', ')', 'takes'): 0.39652492505828807,\n",
       "             ('the', 'steps', ')', 'takes', 'a'): 0.6570606323425058,\n",
       "             ('steps', ')', 'takes', 'a', 'sharp'): 0.6570606323425058,\n",
       "             (')', 'takes', 'a', 'sharp', 'turn'): 0.6570606323425058,\n",
       "             ('takes', 'a', 'sharp', 'turn', 'up'): 0.6570606323425058,\n",
       "             ('a', 'sharp', 'turn', 'up', 'to'): 0.6570606323425058,\n",
       "             ('sharp', 'turn', 'up', 'to', 'vertical'): 0.6570606323425058,\n",
       "             ('turn', 'up', 'to', 'vertical', '.'): 0.6570606323425058,\n",
       "             ('up', 'to', 'vertical', '.', 'the'): 0.6570606323425058,\n",
       "             ('to', 'vertical', '.', 'the', 'cars'): 0.6570606323425058,\n",
       "             ('vertical', '.', 'the', 'cars', 'are'): 0.6570606323425058,\n",
       "             ('.', 'the', 'cars', 'are', 'not'): 0.28393848879601136,\n",
       "             ('the', 'cars', 'are', 'not', 'stopping'): 0.6570606323425058,\n",
       "             ('cars', 'are', 'not', 'stopping', 'or'): 0.6570606323425058,\n",
       "             ('are', 'not', 'stopping', 'or', 'even'): 0.6570606323425058,\n",
       "             ('not', 'stopping', 'or', 'even', 'slowing'): 0.6570606323425058,\n",
       "             ('stopping', 'or', 'even', 'slowing', 'down'): 0.6570606323425058,\n",
       "             ('or', 'even', 'slowing', 'down', ','): 0.6570606323425058,\n",
       "             ('even', 'slowing', 'down', ',', 'and'): 0.6570606323425058,\n",
       "             ('slowing', 'down', ',', 'and', 'they'): 0.6570606323425058,\n",
       "             ('down', ',', 'and', 'they', 'whoosh'): 0.28393848879601136,\n",
       "             (',', 'and', 'they', 'whoosh', 'straight'): 0.6570606323425058,\n",
       "             ('and', 'they', 'whoosh', 'straight', 'up'): 0.6570606323425058,\n",
       "             ('they', 'whoosh', 'straight', 'up', '.'): 0.6570606323425058,\n",
       "             ('whoosh', 'straight', 'up', '.', 'they'): 0.6570606323425058,\n",
       "             ('straight', 'up', '.', 'they', 'make'): 0.6570606323425058,\n",
       "             ('up', '.', 'they', 'make', 'it'): 0.6570606323425058,\n",
       "             ('.', 'they', 'make', 'it', 'about'): 0.6570606323425058,\n",
       "             ('they', 'make', 'it', 'about', 'halfway'): 0.6570606323425058,\n",
       "             ('make', 'it', 'about', 'halfway', 'up'): 0.6570606323425058,\n",
       "             ('it', 'about', 'halfway', 'up', 'on'): 0.6570606323425058,\n",
       "             ('about', 'halfway', 'up', 'on', 'momentum'): 0.6570606323425058,\n",
       "             ('halfway', 'up', 'on', 'momentum', ','): 0.6570606323425058,\n",
       "             ('up', 'on', 'momentum', ',', 'and'): 0.6570606323425058,\n",
       "             ('on', 'momentum', ',', 'and', 'then'): 0.6570606323425058,\n",
       "             ('momentum', ',', 'and', 'then', 'gravity'): 0.6570606323425058,\n",
       "             (',', 'and', 'then', 'gravity', 'takes'): 0.6570606323425058,\n",
       "             ('and', 'then', 'gravity', 'takes', 'over'): 0.6570606323425058,\n",
       "             ('then', 'gravity', 'takes', 'over', 'and'): 0.6570606323425058,\n",
       "             ('gravity', 'takes', 'over', 'and', 'they'): 0.6570606323425058,\n",
       "             ('takes', 'over', 'and', 'they', 'fall'): 0.6570606323425058,\n",
       "             ('over', 'and', 'they', 'fall', 'down'): 0.6570606323425058,\n",
       "             ('and', 'they', 'fall', 'down', '.'): 0.6570606323425058,\n",
       "             ('they', 'fall', 'down', '.', 'i'): 0.6570606323425058,\n",
       "             ('fall', 'down', '.', 'i', 'can'): 0.28393848879601136,\n",
       "             ('down', '.', 'i', 'can', 'see'): 0.6570606323425058,\n",
       "             ('.', 'i', 'can', 'see', 'them'): 0.024533512197204288,\n",
       "             ('i', 'can', 'see', 'them', 'sort'): 0.1050224953686006,\n",
       "             ('can', 'see', 'them', 'sort', 'of'): 0.6570606323425058,\n",
       "             ('see', 'them', 'sort', 'of', 'drop'): 0.6570606323425058,\n",
       "             ('them', 'sort', 'of', 'drop', 'off'): 0.6570606323425058,\n",
       "             ('sort', 'of', 'drop', 'off', '['): 0.6570606323425058,\n",
       "             ('of', 'drop', 'off', '[', 'rather'): 0.6570606323425058,\n",
       "             ('drop', 'off', '[', 'rather', 'than'): 0.6570606323425058,\n",
       "             ('off', '[', 'rather', 'than', 'sliding'): 0.6570606323425058,\n",
       "             ('[', 'rather', 'than', 'sliding', 'down'): 0.6570606323425058,\n",
       "             ('rather',\n",
       "              'than',\n",
       "              'sliding',\n",
       "              'down',\n",
       "              'still'): 0.6570606323425058,\n",
       "             ('than', 'sliding', 'down', 'still', 'in'): 0.6570606323425058,\n",
       "             ('sliding', 'down', 'still', 'in', 'contact'): 0.6570606323425058,\n",
       "             ('down', 'still', 'in', 'contact', 'with'): 0.6570606323425058,\n",
       "             ('still', 'in', 'contact', 'with', 'the'): 0.6570606323425058,\n",
       "             ('in', 'contact', 'with', 'the', 'pavement'): 0.22114746702757215,\n",
       "             ('contact', 'with', 'the', 'pavement', ']'): 0.6570606323425058,\n",
       "             ('with', 'the', 'pavement', ']', 'and'): 0.6570606323425058,\n",
       "             ('the', 'pavement', ']', 'and', 'fall'): 0.6570606323425058,\n",
       "             ('pavement', ']', 'and', 'fall', 'down'): 0.6570606323425058,\n",
       "             (']', 'and', 'fall', 'down', 'way'): 0.6570606323425058,\n",
       "             ('and', 'fall', 'down', 'way', 'below'): 0.6570606323425058,\n",
       "             ('fall', 'down', 'way', 'below', 'me'): 0.6570606323425058,\n",
       "             ('down', 'way', 'below', 'me', '.'): 0.6570606323425058,\n",
       "             ('way', 'below', 'me', '.', 'i'): 0.6570606323425058,\n",
       "             ('below', 'me', '.', 'i', 'watch'): 0.22114746702757215,\n",
       "             ('me', '.', 'i', 'watch', 'a'): 0.1329467319687954,\n",
       "             ('.', 'i', 'watch', 'a', 'plane'): 0.1050224953686006,\n",
       "             ('i', 'watch', 'a', 'plane', 'fly'): 0.6570606323425058,\n",
       "             ('watch', 'a', 'plane', 'fly', 'past'): 0.6570606323425058,\n",
       "             ('a', 'plane', 'fly', 'past', 'and'): 0.6570606323425058,\n",
       "             ('plane', 'fly', 'past', 'and', 'shortly'): 0.6570606323425058,\n",
       "             ('fly', 'past', 'and', 'shortly', 'realize'): 0.6570606323425058,\n",
       "             ('past', 'and', 'shortly', 'realize', 'it'): 0.6570606323425058,\n",
       "             ('and', 'shortly', 'realize', 'it', 'was'): 0.6570606323425058,\n",
       "             ('shortly', 'realize', 'it', 'was', 'too'): 0.6570606323425058,\n",
       "             ('realize', 'it', 'was', 'too', 'low'): 0.6570606323425058,\n",
       "             ('it', 'was', 'too', 'low', '.'): 0.28393848879601136,\n",
       "             ('was', 'too', 'low', '.', 'it'): 0.39652492505828807,\n",
       "             ('too', 'low', '.', 'it', 'crashes'): 0.6570606323425058,\n",
       "             ('low', '.', 'it', 'crashes', 'not'): 0.6570606323425058,\n",
       "             ('.', 'it', 'crashes', 'not', 'half'): 0.6570606323425058,\n",
       "             ('it', 'crashes', 'not', 'half', 'a'): 0.6570606323425058,\n",
       "             ('crashes', 'not', 'half', 'a', 'block'): 0.6570606323425058,\n",
       "             ('not', 'half', 'a', 'block', 'away'): 0.6570606323425058,\n",
       "             ('half', 'a', 'block', 'away', '('): 0.39652492505828807,\n",
       "             ('a', 'block', 'away', '(', 'rather'): 0.6570606323425058,\n",
       "             ('block', 'away', '(', 'rather', 'as'): 0.6570606323425058,\n",
       "             ('away', '(', 'rather', 'as', 'though'): 0.6570606323425058,\n",
       "             ('(', 'rather', 'as', 'though', 'it'): 0.6570606323425058,\n",
       "             ('rather', 'as', 'though', 'it', 'fell'): 0.6570606323425058,\n",
       "             ('as', 'though', 'it', 'fell', 'from'): 0.6570606323425058,\n",
       "             ('though', 'it', 'fell', 'from', 'a'): 0.6570606323425058,\n",
       "             ('it', 'fell', 'from', 'a', 'height'): 0.6570606323425058,\n",
       "             ('fell', 'from', 'a', 'height', 'of'): 0.6570606323425058,\n",
       "             ('from', 'a', 'height', 'of', 'a'): 0.6570606323425058,\n",
       "             ('a', 'height', 'of', 'a', 'few'): 0.6570606323425058,\n",
       "             ('height', 'of', 'a', 'few', 'yards'): 0.6570606323425058,\n",
       "             ('of', 'a', 'few', 'yards', 'than'): 0.6570606323425058,\n",
       "             ('a', 'few', 'yards', 'than', 'flying'): 0.6570606323425058,\n",
       "             ('few',\n",
       "              'yards',\n",
       "              'than',\n",
       "              'flying',\n",
       "              'straight'): 0.6570606323425058,\n",
       "             ('yards',\n",
       "              'than',\n",
       "              'flying',\n",
       "              'straight',\n",
       "              'into'): 0.6570606323425058,\n",
       "             ('than', 'flying', 'straight', 'into', 'the'): 0.6570606323425058,\n",
       "             ('flying',\n",
       "              'straight',\n",
       "              'into',\n",
       "              'the',\n",
       "              'ground'): 0.6570606323425058,\n",
       "             ('straight', 'into', 'the', 'ground', ')'): 0.6570606323425058,\n",
       "             ('into', 'the', 'ground', ')', '.'): 0.6570606323425058,\n",
       "             ('the', 'ground', ')', '.', 'and'): 0.39652492505828807,\n",
       "             ('ground', ')', '.', 'and', 'i'): 0.6570606323425058,\n",
       "             (')', '.', 'and', 'i', 'think'): 0.28393848879601136,\n",
       "             ('.', 'and', 'i', 'think', 'there'): 0.07723883120400651,\n",
       "             ('and', 'i', 'think', 'there', 'might'): 0.03770535812587004,\n",
       "             ('i', 'think', 'there', 'might', 'be'): 0.6570606323425058,\n",
       "             ('think',\n",
       "              'there',\n",
       "              'might',\n",
       "              'be',\n",
       "              'survivors'): 0.39652492505828807,\n",
       "             ('there', 'might', 'be', 'survivors', ','): 0.6570606323425058,\n",
       "             ('might', 'be', 'survivors', ',', 'and'): 0.6570606323425058,\n",
       "             ('be', 'survivors', ',', 'and', 'that'): 0.6570606323425058,\n",
       "             ('survivors', ',', 'and', 'that', 'i'): 0.6570606323425058,\n",
       "             (',', 'and', 'that', 'i', 'should'): 0.07105035263204262,\n",
       "             ('and', 'that', 'i', 'should', 'call'): 0.048729263676568914,\n",
       "             ('that', 'i', 'should', 'call', 'the'): 0.18109874030235643,\n",
       "             ('i', 'should', 'call', 'the', 'police'): 0.44229051119426094,\n",
       "             ('should', 'call', 'the', 'police', '.'): 0.5678712989358199,\n",
       "             ('call', 'the', 'police', '.', 'before'): 0.028152162774801846,\n",
       "             ('the', 'police', '.', 'before', 'i'): 0.6570606323425058,\n",
       "             ('police', '.', 'before', 'i', 'get'): 0.6570606323425058,\n",
       "             ('.', 'before', 'i', 'get', 'the'): 0.18109874030235643,\n",
       "             ('before', 'i', 'get', 'the', 'number'): 0.39652492505828807,\n",
       "             ('i', 'get', 'the', 'number', 'dialed'): 0.22114746702757215,\n",
       "             ('get', 'the', 'number', 'dialed', ','): 0.6570606323425058,\n",
       "             ('the', 'number', 'dialed', ',', 'i'): 0.6570606323425058,\n",
       "             ('number', 'dialed', ',', 'i', 'realize'): 0.6570606323425058,\n",
       "             ('dialed', ',', 'i', 'realize', 'i'): 0.6570606323425058,\n",
       "             (',', 'i', 'realize', 'i', 'want'): 0.0974575527873556,\n",
       "             ('i', 'realize', 'i', 'want', 'the'): 0.36219385870234483,\n",
       "             ('realize', 'i', 'want', 'the', 'fire'): 0.39652492505828807,\n",
       "             ('i', 'want', 'the', 'fire', 'department'): 0.6570606323425058,\n",
       "             ('want',\n",
       "              'the',\n",
       "              'fire',\n",
       "              'department',\n",
       "              'instead'): 0.6570606323425058,\n",
       "             ('the', 'fire', 'department', 'instead', '.'): 0.6570606323425058,\n",
       "             ('fire', 'department', 'instead', '.', 'i'): 0.6570606323425058,\n",
       "             ('department', 'instead', '.', 'i', 'get'): 0.6570606323425058,\n",
       "             ('instead', '.', 'i', 'get', 'no'): 0.39652492505828807,\n",
       "             ('.', 'i', 'get', 'no', 'ring'): 0.39652492505828807,\n",
       "             ('i', 'get', 'no', 'ring', ','): 0.6570606323425058,\n",
       "             ('get', 'no', 'ring', ',', 'try'): 0.6570606323425058,\n",
       "             ('no', 'ring', ',', 'try', 'again'): 0.6570606323425058,\n",
       "             ('ring', ',', 'try', 'again', 'and'): 0.6570606323425058,\n",
       "             (',', 'try', 'again', 'and', 'find'): 0.6570606323425058,\n",
       "             ('try', 'again', 'and', 'find', 'the'): 0.6570606323425058,\n",
       "             ('again', 'and', 'find', 'the', 'line'): 0.6570606323425058,\n",
       "             ('and', 'find', 'the', 'line', 'is'): 0.39652492505828807,\n",
       "             ('find', 'the', 'line', 'is', 'dead'): 0.6570606323425058,\n",
       "             ('the', 'line', 'is', 'dead', '.'): 0.39652492505828807,\n",
       "             ('line', 'is', 'dead', '.', 'some'): 0.39652492505828807,\n",
       "             ('is', 'dead', '.', 'some', 'people'): 0.6570606323425058,\n",
       "             ('dead', '.', 'some', 'people', '('): 0.6570606323425058,\n",
       "             ('.', 'some', 'people', '(', '3'): 0.6570606323425058,\n",
       "             ('some', 'people', '(', '3', '?'): 0.6570606323425058,\n",
       "             ('people', '(', '3', '?', '-'): 0.6570606323425058,\n",
       "             ('(', '3', '?', '-', 'at'): 0.6570606323425058,\n",
       "             ('3', '?', '-', 'at', 'least'): 0.6570606323425058,\n",
       "             ('?', '-', 'at', 'least', 'one'): 0.6570606323425058,\n",
       "             ('-', 'at', 'least', 'one', 'a'): 0.6570606323425058,\n",
       "             ('at', 'least', 'one', 'a', 'woman'): 0.6570606323425058,\n",
       "             ('least', 'one', 'a', 'woman', ')'): 0.6570606323425058,\n",
       "             ('one', 'a', 'woman', ')', 'are'): 0.6570606323425058,\n",
       "             ('a', 'woman', ')', 'are', 'coming'): 0.6570606323425058,\n",
       "             ('woman', ')', 'are', 'coming', 'toward'): 0.6570606323425058,\n",
       "             (')', 'are', 'coming', 'toward', 'the'): 0.6570606323425058,\n",
       "             ('are', 'coming', 'toward', 'the', 'house'): 0.6570606323425058,\n",
       "             ('coming', 'toward', 'the', 'house', 'and'): 0.22114746702757215,\n",
       "             ('toward', 'the', 'house', 'and', 'i'): 0.28393848879601136,\n",
       "             ('the', 'house', 'and', 'i', 'understand'): 0.017385012610835932,\n",
       "             ('house', 'and', 'i', 'understand', 'they'): 0.6570606323425058,\n",
       "             ('and', 'i', 'understand', 'they', 'are'): 0.6570606323425058,\n",
       "             ('i',\n",
       "              'understand',\n",
       "              'they',\n",
       "              'are',\n",
       "              'responsible'): 0.6570606323425058,\n",
       "             ('understand',\n",
       "              'they',\n",
       "              'are',\n",
       "              'responsible',\n",
       "              'for'): 0.6570606323425058,\n",
       "             ('they', 'are', 'responsible', 'for', 'the'): 0.6570606323425058,\n",
       "             ('are', 'responsible', 'for', 'the', 'dead'): 0.6570606323425058,\n",
       "             ('responsible',\n",
       "              'for',\n",
       "              'the',\n",
       "              'dead',\n",
       "              'phone'): 0.6570606323425058,\n",
       "             ('for', 'the', 'dead', 'phone', 'and'): 0.6570606323425058,\n",
       "             ('the', 'dead', 'phone', 'and', 'i'): 0.6570606323425058,\n",
       "             ('dead', 'phone', 'and', 'i', 'should'): 0.6570606323425058,\n",
       "             ('phone', 'and', 'i', 'should', 'be'): 0.6570606323425058,\n",
       "             ('and', 'i', 'should', 'be', 'suspicious'): 0.15333120718311674,\n",
       "             ('i', 'should', 'be', 'suspicious', 'of'): 0.6570606323425058,\n",
       "             ('should', 'be', 'suspicious', 'of', 'them'): 0.6570606323425058,\n",
       "             ('be', 'suspicious', 'of', 'them', '.'): 0.6570606323425058,\n",
       "             ('suspicious', 'of', 'them', '.', 'i'): 0.6570606323425058,\n",
       "             ('of', 'them', '.', 'i', 'try'): 0.0052765389298422205,\n",
       "             ('them', '.', 'i', 'try', 'to'): 0.6388792169915891,\n",
       "             ('.', 'i', 'try', 'to', 'leave'): 0.0032335472777716744,\n",
       "             ('i', 'try', 'to', 'leave', 'the'): 0.28393848879601136,\n",
       "             ('try', 'to', 'leave', 'the', 'house'): 0.39652492505828807,\n",
       "             ('to', 'leave', 'the', 'house', '-'): 0.07395525500741018,\n",
       "             ('leave', 'the', 'house', '-', 'run'): 0.6570606323425058,\n",
       "             ('the', 'house', '-', 'run', 'into'): 0.6570606323425058,\n",
       "             ('house', '-', 'run', 'into', 'a'): 0.6570606323425058,\n",
       "             ('-', 'run', 'into', 'a', 'pregnant'): 0.6570606323425058,\n",
       "             ('run', 'into', 'a', 'pregnant', 'girl'): 0.6570606323425058,\n",
       "             ('into', 'a', 'pregnant', 'girl', 'at'): 0.6570606323425058,\n",
       "             ('a', 'pregnant', 'girl', 'at', 'the'): 0.6570606323425058,\n",
       "             ('pregnant', 'girl', 'at', 'the', 'back'): 0.6570606323425058,\n",
       "             ('girl', 'at', 'the', 'back', ';'): 0.39652492505828807,\n",
       "             ('at', 'the', 'back', ';', 'she'): 0.6570606323425058,\n",
       "             ('the', 'back', ';', 'she', 'seems'): 0.6570606323425058,\n",
       "             ('back', ';', 'she', 'seems', 'to'): 0.6570606323425058,\n",
       "             (';', 'she', 'seems', 'to', 'be'): 0.28393848879601136,\n",
       "             ('she', 'seems', 'to', 'be', 'in'): 0.0655272466119476,\n",
       "             ('seems', 'to', 'be', 'in', 'some'): 0.0754099621596594,\n",
       "             ('to', 'be', 'in', 'some', 'sort'): 0.46937793653103266,\n",
       "             ('be', 'in', 'some', 'sort', 'of'): 0.8845765995276383,\n",
       "             ('in', 'some', 'sort', 'of', 'a'): 0.17593850224785018,\n",
       "             ('some', 'sort', 'of', 'a', 'bad'): 0.012266878764936467,\n",
       "             ('sort', 'of', 'a', 'bad', 'way'): 0.6570606323425058,\n",
       "             ('of', 'a', 'bad', 'way', 'and'): 0.6570606323425058,\n",
       "             ('a', 'bad', 'way', 'and', 'has'): 0.6570606323425058,\n",
       "             ('bad', 'way', 'and', 'has', 'maybe'): 0.6570606323425058,\n",
       "             ('way', 'and', 'has', 'maybe', 'started'): 0.6570606323425058,\n",
       "             ('and', 'has', 'maybe', 'started', 'labor'): 0.6570606323425058,\n",
       "             ('has', 'maybe', 'started', 'labor', '.'): 0.6570606323425058,\n",
       "             ('maybe', 'started', 'labor', '.', 'the'): 0.6570606323425058,\n",
       "             ('started', 'labor', '.', 'the', 'woman'): 0.6570606323425058,\n",
       "             ('labor', '.', 'the', 'woman', 'from'): 0.6570606323425058,\n",
       "             ('.', 'the', 'woman', 'from', 'the'): 0.3066593478034211,\n",
       "             ('the', 'woman', 'from', 'the', 'plane'): 0.39652492505828807,\n",
       "             ('woman', 'from', 'the', 'plane', 'comes'): 0.6570606323425058,\n",
       "             ('from', 'the', 'plane', 'comes', 'this'): 0.6570606323425058,\n",
       "             ('the', 'plane', 'comes', 'this', 'way'): 0.6570606323425058,\n",
       "             ('plane', 'comes', 'this', 'way', ','): 0.6570606323425058,\n",
       "             ('comes', 'this', 'way', ',', 'calls'): 0.6570606323425058,\n",
       "             ('this', 'way', ',', 'calls', 'the'): 0.6570606323425058,\n",
       "             ('way', ',', 'calls', 'the', 'others'): 0.6570606323425058,\n",
       "             (',', 'calls', 'the', 'others', 'to'): 0.6570606323425058,\n",
       "             ('calls', 'the', 'others', 'to', 'come'): 0.6570606323425058,\n",
       "             ('the', 'others', 'to', 'come', 'get'): 0.22114746702757215,\n",
       "             ('others', 'to', 'come', 'get', 'the'): 0.6570606323425058,\n",
       "             ('to', 'come', 'get', 'the', 'pregnant'): 0.6570606323425058,\n",
       "             ('come', 'get', 'the', 'pregnant', 'girl'): 0.6570606323425058,\n",
       "             ('get', 'the', 'pregnant', 'girl', ','): 0.6570606323425058,\n",
       "             ('the', 'pregnant', 'girl', ',', 'and'): 0.6570606323425058,\n",
       "             ('pregnant', 'girl', ',', 'and', 'i'): 0.6570606323425058,\n",
       "             ('girl', ',', 'and', 'i', 'pretend'): 0.05399104630395488,\n",
       "             (',', 'and', 'i', 'pretend', 'to'): 0.6570606323425058,\n",
       "             ('and', 'i', 'pretend', 'to', 'be'): 0.7930419197766817,\n",
       "             ('i', 'pretend', 'to', 'be', 'asleep'): 0.27544628961930756,\n",
       "             ('pretend', 'to', 'be', 'asleep', 'or'): 0.36219385870234483,\n",
       "             ('to', 'be', 'asleep', 'or', 'dead'): 0.7930419197766817,\n",
       "             ('be', 'asleep', 'or', 'dead', '.'): 0.7930419197766817,\n",
       "             ('asleep', 'or', 'dead', '.', 'they'): 0.39652492505828807,\n",
       "             ('or', 'dead', '.', 'they', 'do'): 0.6570606323425058,\n",
       "             ('dead', '.', 'they', 'do', \"n't\"): 0.6570606323425058,\n",
       "             ('.', 'they', 'do', \"n't\", 'buy'): 0.011831481428021783,\n",
       "             ('they', 'do', \"n't\", 'buy', 'it'): 0.6570606323425058,\n",
       "             ('do', \"n't\", 'buy', 'it', ','): 0.6570606323425058,\n",
       "             (\"n't\", 'buy', 'it', ',', 'i'): 0.6570606323425058,\n",
       "             ('buy', 'it', ',', 'i', 'guess'): 0.6570606323425058,\n",
       "             ('it', ',', 'i', 'guess', ','): 0.3194404070928193,\n",
       "             (',', 'i', 'guess', ',', 'and'): 0.28870246089795376,\n",
       "             ('i', 'guess', ',', 'and', 'we'): 0.09224597779469626,\n",
       "             ('guess', ',', 'and', 'we', 'all'): 0.28393848879601136,\n",
       "             (',', 'and', 'we', 'all', 'go'): 0.10161994664310907,\n",
       "             ('and', 'we', 'all', 'go', 'inside'): 0.22114746702757215,\n",
       "             ('we', 'all', 'go', 'inside', '.'): 0.6570606323425058,\n",
       "             ('all', 'go', 'inside', '.', '<'): 0.6570606323425058,\n",
       "             ('go', 'inside', '.', '<', 'br'): 0.39652492505828807,\n",
       "             ('inside', '.', '<', 'br', '/'): 0.8845765995276383,\n",
       "             ('.', '<', 'br', '/', '>'): 0.9995612186932639,\n",
       "             ('<', 'br', '/', '>', '&'): 0.029835668766938807,\n",
       "             ('br', '/', '>', '&', '#'): 0.9974969542776214,\n",
       "             ('/', '>', '&', '#', '160'): 0.9974969542776214,\n",
       "             ('>', '&', '#', '160', ';'): 0.9975089007376602,\n",
       "             ('&', '#', '160', ';', '&'): 0.825644934149741,\n",
       "             ('#', '160', ';', '&', '#'): 0.9995026878713428,\n",
       "             ('160', ';', '&', '#', '160'): 0.9995026878713428,\n",
       "             (';', '&', '#', '160', ';'): 0.9995026878713428,\n",
       "             ('&', '#', '160', ';', 'i'): 0.02439943639856932,\n",
       "             ('#', '160', ';', 'i', \"'m\"): 0.12689629705767025,\n",
       "             ('160', ';', 'i', \"'m\", 'not'): 0.22114746702757215,\n",
       "             (';', 'i', \"'m\", 'not', 'a'): 0.1329467319687954,\n",
       "             ('i', \"'m\", 'not', 'a', 'prisoner'): 0.036335348209211846,\n",
       "             (\"'m\", 'not', 'a', 'prisoner', ','): 0.6570606323425058,\n",
       "             ('not', 'a', 'prisoner', ',', 'i'): 0.6570606323425058,\n",
       "             ('a', 'prisoner', ',', 'i', 'do'): 0.6570606323425058,\n",
       "             ('prisoner', ',', 'i', 'do', \"n't\"): 0.6570606323425058,\n",
       "             (',', 'i', 'do', \"n't\", 'think'): 0.07189865866689818,\n",
       "             ('i', 'do', \"n't\", 'think', ','): 0.010429702643363627,\n",
       "             ('do', \"n't\", 'think', ',', 'so'): 0.18109874030235643,\n",
       "             (\"n't\", 'think', ',', 'so', 'much'): 0.6570606323425058,\n",
       "             ('think', ',', 'so', 'much', 'as'): 0.6570606323425058,\n",
       "             (',', 'so', 'much', 'as', 'being'): 0.6570606323425058,\n",
       "             ('so', 'much', 'as', 'being', 'detained'): 0.6570606323425058,\n",
       "             ('much', 'as', 'being', 'detained', 'for'): 0.6570606323425058,\n",
       "             ('as', 'being', 'detained', 'for', 'some'): 0.6570606323425058,\n",
       "             ('being',\n",
       "              'detained',\n",
       "              'for',\n",
       "              'some',\n",
       "              'reason'): 0.6570606323425058,\n",
       "             ('detained', 'for', 'some', 'reason', ','): 0.6570606323425058,\n",
       "             ('for', 'some', 'reason', ',', 'and'): 0.23556314302445708,\n",
       "             ('some', 'reason', ',', 'and', 'i'): 0.33668687065937514,\n",
       "             ('reason', ',', 'and', 'i', 'go'): 0.051225389253947866,\n",
       "             (',', 'and', 'i', 'go', 'to'): 0.20505157380238118,\n",
       "             ('and', 'i', 'go', 'to', 'my'): 0.051598540594174847,\n",
       "             ('i', 'go', 'to', 'my', 'brother'): 0.04393527870944046,\n",
       "             ('go', 'to', 'my', 'brother', \"'s\"): 0.7930419197766817,\n",
       "             ('to', 'my', 'brother', \"'s\", 'old'): 0.1329467319687954,\n",
       "             ('my', 'brother', \"'s\", 'old', 'room'): 0.6634335553609497,\n",
       "             ('brother', \"'s\", 'old', 'room', '.'): 0.28393848879601136,\n",
       "             (\"'s\", 'old', 'room', '.', 'he'): 0.7930419197766817,\n",
       "             ('old', 'room', '.', 'he', 'moved'): 0.39652492505828807,\n",
       "             ('room', '.', 'he', 'moved', 'out'): 0.6570606323425058,\n",
       "             ('.', 'he', 'moved', 'out', 'of'): 0.6570606323425058,\n",
       "             ('he', 'moved', 'out', 'of', 'it'): 0.6570606323425058,\n",
       "             ('moved', 'out', 'of', 'it', 'a'): 0.28393848879601136,\n",
       "             ('out', 'of', 'it', 'a', 'couple'): 0.39652492505828807,\n",
       "             ('of', 'it', 'a', 'couple', 'of'): 0.6570606323425058,\n",
       "             ('it', 'a', 'couple', 'of', 'years'): 0.09504122806017132,\n",
       "             ('a', 'couple', 'of', 'years', 'ago'): 0.285119882607423,\n",
       "             ('couple', 'of', 'years', 'ago', 'but'): 0.28393848879601136,\n",
       "             ('of', 'years', 'ago', 'but', 'there'): 0.6570606323425058,\n",
       "             ('years', 'ago', 'but', 'there', 'is'): 0.6570606323425058,\n",
       "             ('ago', 'but', 'there', 'is', 'furniture'): 0.6570606323425058,\n",
       "             ('but', 'there', 'is', 'furniture', 'still'): 0.6570606323425058,\n",
       "             ('there',\n",
       "              'is',\n",
       "              'furniture',\n",
       "              'still',\n",
       "              'there'): 0.6570606323425058,\n",
       "             ('is', 'furniture', 'still', 'there', 'and'): 0.6570606323425058,\n",
       "             ('furniture', 'still', 'there', 'and', 'i'): 0.6570606323425058,\n",
       "             ('still', 'there', 'and', 'i', 'have'): 0.28393848879601136,\n",
       "             ('there', 'and', 'i', 'have', \"n't\"): 0.36219385870234483,\n",
       "             ('and', 'i', 'have', \"n't\", 'looked'): 0.04078059013227328,\n",
       "             ('i', 'have', \"n't\", 'looked', 'around'): 0.28393848879601136,\n",
       "             ('have', \"n't\", 'looked', 'around', 'in'): 0.6570606323425058,\n",
       "             (\"n't\", 'looked', 'around', 'in', 'a'): 0.6570606323425058,\n",
       "             ('looked', 'around', 'in', 'a', 'long'): 0.39652492505828807,\n",
       "             ('around', 'in', 'a', 'long', 'time'): 0.39652492505828807,\n",
       "             ('in', 'a', 'long', 'time', '.'): 0.7153084738699784,\n",
       "             ('a', 'long', 'time', '.', '['): 0.012083109697347712,\n",
       "             ('long', 'time', '.', '[', 'this'): 0.39652492505828807,\n",
       "             ('time', '.', '[', 'this', 'does'): 0.6570606323425058,\n",
       "             ('.', '[', 'this', 'does', 'not'): 0.6570606323425058,\n",
       "             ('[', 'this', 'does', 'not', 'seem'): 0.6570606323425058,\n",
       "             ('this', 'does', 'not', 'seem', 'to'): 0.6570606323425058,\n",
       "             ('does', 'not', 'seem', 'to', 'be'): 0.6647230243181298,\n",
       "             ('not', 'seem', 'to', 'be', 'the'): 0.08679252488291923,\n",
       "             ('seem', 'to', 'be', 'the', 'family'): 0.05707238231339417,\n",
       "             ('to', 'be', 'the', 'family', 'home'): 0.28393848879601136,\n",
       "             ('be', 'the', 'family', 'home', '-'): 0.6570606323425058,\n",
       "             ('the', 'family', 'home', '-', 'i'): 0.6570606323425058,\n",
       "             ('family', 'home', '-', 'i', 'felt'): 0.6570606323425058,\n",
       "             ('home', '-', 'i', 'felt', 'like'): 0.6570606323425058,\n",
       "             ('-', 'i', 'felt', 'like', 'it'): 0.39652492505828807,\n",
       "             ('i', 'felt', 'like', 'it', 'was'): 0.8134930143263254,\n",
       "             ('felt', 'like', 'it', 'was', 'rented'): 0.05399104630395488,\n",
       "             ('like', 'it', 'was', 'rented', '.'): 0.6570606323425058,\n",
       "             ('it', 'was', 'rented', '.', ']'): 0.6570606323425058,\n",
       "             ('was', 'rented', '.', ']', 'there'): 0.6570606323425058,\n",
       "             ('rented', '.', ']', 'there', 'are'): 0.6570606323425058,\n",
       "             ('.', ']', 'there', 'are', 'a'): 0.44229051119426094,\n",
       "             (']', 'there', 'are', 'a', 'couple'): 0.28393848879601136,\n",
       "             ('there', 'are', 'a', 'couple', 'of'): 0.6409790226664998,\n",
       "             ('are', 'a', 'couple', 'of', 'chests'): 0.04440199698427669,\n",
       "             ('a', 'couple', 'of', 'chests', 'of'): 0.6570606323425058,\n",
       "             ('couple', 'of', 'chests', 'of', 'drawers'): 0.6570606323425058,\n",
       "             ('of', 'chests', 'of', 'drawers', 'that'): 0.6570606323425058,\n",
       "             ('chests', 'of', 'drawers', 'that', 'i'): 0.6570606323425058,\n",
       "             ('of', 'drawers', 'that', 'i', 'realize'): 0.6570606323425058,\n",
       "             ('drawers', 'that', 'i', 'realize', 'i'): 0.6570606323425058,\n",
       "             ('that', 'i', 'realize', 'i', 'could'): 0.6570606323425058,\n",
       "             ('i', 'realize', 'i', 'could', 'have'): 0.35203403911776165,\n",
       "             ('realize', 'i', 'could', 'have', 'used'): 0.22114746702757215,\n",
       "             ('i', 'could', 'have', 'used', 'downstairs'): 0.22114746702757215,\n",
       "             ('could',\n",
       "              'have',\n",
       "              'used',\n",
       "              'downstairs',\n",
       "              'long'): 0.6570606323425058,\n",
       "             ('have', 'used', 'downstairs', 'long', 'ago'): 0.6570606323425058,\n",
       "             ('used', 'downstairs', 'long', 'ago', ','): 0.6570606323425058,\n",
       "             ('downstairs', 'long', 'ago', ',', 'yet'): 0.6570606323425058,\n",
       "             ('long', 'ago', ',', 'yet', 'i'): 0.6570606323425058,\n",
       "             ('ago', ',', 'yet', 'i', \"'ve\"): 0.6570606323425058,\n",
       "             (',', 'yet', 'i', \"'ve\", 'been'): 0.6570606323425058,\n",
       "             ('yet', 'i', \"'ve\", 'been', 'looking'): 0.6570606323425058,\n",
       "             ('i', \"'ve\", 'been', 'looking', 'for'): 0.6634335553609497,\n",
       "             (\"'ve\",\n",
       "              'been',\n",
       "              'looking',\n",
       "              'for',\n",
       "              'acceptable'): 0.28393848879601136,\n",
       "             ('been',\n",
       "              'looking',\n",
       "              'for',\n",
       "              'acceptable',\n",
       "              'furniture'): 0.6570606323425058,\n",
       "             ('looking',\n",
       "              'for',\n",
       "              'acceptable',\n",
       "              'furniture',\n",
       "              'all'): 0.6570606323425058,\n",
       "             ('for',\n",
       "              'acceptable',\n",
       "              'furniture',\n",
       "              'all',\n",
       "              'this'): 0.6570606323425058,\n",
       "             ('acceptable',\n",
       "              'furniture',\n",
       "              'all',\n",
       "              'this',\n",
       "              'time'): 0.6570606323425058,\n",
       "             ('furniture', 'all', 'this', 'time', '.'): 0.6570606323425058,\n",
       "             ('all', 'this', 'time', '.', 'also'): 0.1050224953686006,\n",
       "             ('this', 'time', '.', 'also', 'there'): 0.6570606323425058,\n",
       "             ('time', '.', 'also', 'there', 'are'): 0.6570606323425058,\n",
       "             ('.', 'also', 'there', 'are', 'blankets'): 0.28393848879601136,\n",
       "             ('also', 'there', 'are', 'blankets', '('): 0.6570606323425058,\n",
       "             ('there', 'are', 'blankets', '(', 'as'): 0.6570606323425058,\n",
       "             ('are', 'blankets', '(', 'as', 'well'): 0.6570606323425058,\n",
       "             ('blankets', '(', 'as', 'well', 'as'): 0.6570606323425058,\n",
       "             ('(', 'as', 'well', 'as', 'clothes'): 0.6570606323425058,\n",
       "             ('as', 'well', 'as', 'clothes', 'and'): 0.6570606323425058,\n",
       "             ('well', 'as', 'clothes', 'and', 'shoes'): 0.6570606323425058,\n",
       "             ('as', 'clothes', 'and', 'shoes', '-'): 0.6570606323425058,\n",
       "             ('clothes', 'and', 'shoes', '-', 'women'): 0.6570606323425058,\n",
       "             ('and', 'shoes', '-', 'women', \"'s\"): 0.6570606323425058,\n",
       "             ('shoes', '-', 'women', \"'s\", 'shoes'): 0.6570606323425058,\n",
       "             ('-', 'women', \"'s\", 'shoes', '-'): 0.6570606323425058,\n",
       "             ('women', \"'s\", 'shoes', '-', 'but'): 0.6570606323425058,\n",
       "             (\"'s\", 'shoes', '-', 'but', 'they'): 0.6570606323425058,\n",
       "             ('shoes', '-', 'but', 'they', 'are'): 0.6570606323425058,\n",
       "             ('-', 'but', 'they', 'are', \"n't\"): 0.39652492505828807,\n",
       "             ('but', 'they', 'are', \"n't\", 'important'): 0.06886260532324838,\n",
       "             ('they', 'are', \"n't\", 'important', ')'): 0.6570606323425058,\n",
       "             ('are', \"n't\", 'important', ')', ','): 0.6570606323425058,\n",
       "             (\"n't\", 'important', ')', ',', 'a'): 0.6570606323425058,\n",
       "             ('important', ')', ',', 'a', 'lot'): 0.6570606323425058,\n",
       "             (')', ',', 'a', 'lot', 'of'): 0.6570606323425058,\n",
       "             (',', 'a', 'lot', 'of', 'blankets'): 0.035061405317166144,\n",
       "             ('a', 'lot', 'of', 'blankets', ','): 0.6570606323425058,\n",
       "             ('lot', 'of', 'blankets', ',', 'and'): 0.6570606323425058,\n",
       "             ('of', 'blankets', ',', 'and', 'i'): 0.6570606323425058,\n",
       "             ('blankets', ',', 'and', 'i', 'badly'): 0.6570606323425058,\n",
       "             (',', 'and', 'i', 'badly', 'needed'): 0.6570606323425058,\n",
       "             ('and', 'i', 'badly', 'needed', 'those'): 0.6570606323425058,\n",
       "             ('i', 'badly', 'needed', 'those', 'last'): 0.6570606323425058,\n",
       "             ('badly',\n",
       "              'needed',\n",
       "              'those',\n",
       "              'last',\n",
       "              'winter'): 0.6570606323425058,\n",
       "             ('needed', 'those', 'last', 'winter', 'but'): 0.6570606323425058,\n",
       "             ('those', 'last', 'winter', 'but', 'did'): 0.6570606323425058,\n",
       "             ('last', 'winter', 'but', 'did', \"n't\"): 0.6570606323425058,\n",
       "             ('winter', 'but', 'did', \"n't\", 'realize'): 0.6570606323425058,\n",
       "             ('but', 'did', \"n't\", 'realize', 'they'): 0.22114746702757215,\n",
       "             ('did', \"n't\", 'realize', 'they', 'were'): 0.6133156290440296,\n",
       "             (\"n't\", 'realize', 'they', 'were', 'here'): 0.22114746702757215,\n",
       "             ('realize', 'they', 'were', 'here', '.'): 0.6570606323425058,\n",
       "             ('they', 'were', 'here', '.', 'also'): 0.1329467319687954,\n",
       "             ('were', 'here', '.', 'also', 'a'): 0.6570606323425058,\n",
       "             ('here', '.', 'also', 'a', 'very'): 0.6570606323425058,\n",
       "             ('.', 'also', 'a', 'very', 'nice'): 0.6570606323425058,\n",
       "             ('also', 'a', 'very', 'nice', 'shoe'): 0.6570606323425058,\n",
       "             ('a', 'very', 'nice', 'shoe', 'rack'): 0.6570606323425058,\n",
       "             ('very', 'nice', 'shoe', 'rack', '.'): 0.6570606323425058,\n",
       "             ('nice', 'shoe', 'rack', '.', 'the'): 0.6570606323425058,\n",
       "             ('shoe', 'rack', '.', 'the', 'woman'): 0.6570606323425058,\n",
       "             ('rack', '.', 'the', 'woman', 'comes'): 0.6570606323425058,\n",
       "             ('.', 'the', 'woman', 'comes', 'back'): 0.18109874030235643,\n",
       "             ('the', 'woman', 'comes', 'back', 'to'): 0.6570606323425058,\n",
       "             ('woman', 'comes', 'back', 'to', 'say'): 0.6570606323425058,\n",
       "             ('comes', 'back', 'to', 'say', 'or'): 0.6570606323425058,\n",
       "             ('back', 'to', 'say', 'or', 'do'): 0.6570606323425058,\n",
       "             ('to', 'say', 'or', 'do', 'something'): 0.39652492505828807,\n",
       "             ('say', 'or', 'do', 'something', '.'): 0.6570606323425058,\n",
       "             ('or', 'do', 'something', '.', '['): 0.22114746702757215,\n",
       "             ('do', 'something', '.', '[', 'i'): 0.7930419197766817,\n",
       "             ('something', '.', '[', 'i', 'do'): 0.39652492505828807,\n",
       "             ('.', '[', 'i', 'do', \"n't\"): 0.8880230672641279,\n",
       "             ('[', 'i', 'do', \"n't\", 'know'): 0.7687119718491752,\n",
       "             ('i', 'do', \"n't\", 'know', 'what'): 0.1922356425544765,\n",
       "             ('do', \"n't\", 'know', 'what', ','): 0.016774746741028086,\n",
       "             (\"n't\", 'know', 'what', ',', 'and'): 0.3150632852900033,\n",
       "             ('know', 'what', ',', 'and', 'the'): 0.28393848879601136,\n",
       "             ('what', ',', 'and', 'the', 'dream'): 0.6570606323425058,\n",
       "             (',', 'and', 'the', 'dream', 'ended'): 0.39883487814346263,\n",
       "             ('and', 'the', 'dream', 'ended', 'about'): 0.07395525500741018,\n",
       "             ('the', 'dream', 'ended', 'about', 'there'): 0.6570606323425058,\n",
       "             ('dream', 'ended', 'about', 'there', ','): 0.6570606323425058,\n",
       "             ('ended', 'about', 'there', ',', 'i'): 0.6570606323425058,\n",
       "             ('about', 'there', ',', 'i', 'think'): 0.6570606323425058,\n",
       "             ('there', ',', 'i', 'think', '.'): 0.3865504098709183,\n",
       "             (',', 'i', 'think', '.', ']'): 0.010281718321160243,\n",
       "             ('i', 'think', '.', ']', 'me'): 0.39652492505828807,\n",
       "             ('think', '.', ']', 'me', 'pulling'): 0.6570606323425058,\n",
       "             ('.', ']', 'me', 'pulling', 'the'): 0.6570606323425058,\n",
       "             (']', 'me', 'pulling', 'the', 'green'): 0.6570606323425058,\n",
       "             ('me', 'pulling', 'the', 'green', 'leaves'): 0.6570606323425058,\n",
       "             ('pulling', 'the', 'green', 'leaves', 'and'): 0.6570606323425058,\n",
       "             ('the', 'green', 'leaves', 'and', 'berries'): 0.6570606323425058,\n",
       "             ('green', 'leaves', 'and', 'berries', 'off'): 0.6570606323425058,\n",
       "             ('leaves', 'and', 'berries', 'off', 'some'): 0.6570606323425058,\n",
       "             ('and', 'berries', 'off', 'some', 'branches'): 0.6570606323425058,\n",
       "             ('berries', 'off', 'some', 'branches', 'or'): 0.6570606323425058,\n",
       "             ('off', 'some', 'branches', 'or', 'a'): 0.6570606323425058,\n",
       "             ('some', 'branches', 'or', 'a', 'bush'): 0.6570606323425058,\n",
       "             ('branches', 'or', 'a', 'bush', '('): 0.6570606323425058,\n",
       "             ('or', 'a', 'bush', '(', 'not'): 0.6570606323425058,\n",
       "             ('a', 'bush', '(', 'not', 'from'): 0.6570606323425058,\n",
       "             ('bush', '(', 'not', 'from', 'a'): 0.6570606323425058,\n",
       "             ('(', 'not', 'from', 'a', 'live'): 0.6570606323425058,\n",
       "             ('not', 'from', 'a', 'live', 'one'): 0.6570606323425058,\n",
       "             ('from', 'a', 'live', 'one', 'growing'): 0.39652492505828807,\n",
       "             ('a', 'live', 'one', 'growing', 'in'): 0.6570606323425058,\n",
       "             ('live', 'one', 'growing', 'in', 'the'): 0.6570606323425058,\n",
       "             ('one', 'growing', 'in', 'the', 'ground'): 0.6570606323425058,\n",
       "             ('growing', 'in', 'the', 'ground', ','): 0.6570606323425058,\n",
       "             ('in', 'the', 'ground', ',', 'but'): 0.28393848879601136,\n",
       "             ('the', 'ground', ',', 'but', 'inside'): 0.0644261420593791,\n",
       "             ('ground', ',', 'but', 'inside', 'the'): 0.6570606323425058,\n",
       "             (',', 'but', 'inside', 'the', 'house'): 0.28393848879601136,\n",
       "             ('but', 'inside', 'the', 'house', ')'): 0.6570606323425058,\n",
       "             ('inside', 'the', 'house', ')', '.'): 0.6570606323425058,\n",
       "             ('the', 'house', ')', '.', 'a'): 0.1329467319687954,\n",
       "             ('house', ')', '.', 'a', 'boy'): 0.6570606323425058,\n",
       "             (')', '.', 'a', 'boy', 'of'): 0.6570606323425058,\n",
       "             ('.', 'a', 'boy', 'of', '10'): 0.39652492505828807,\n",
       "             ('a', 'boy', 'of', '10', 'or'): 0.6570606323425058,\n",
       "             ('boy', 'of', '10', 'or', '12'): 0.6570606323425058,\n",
       "             ('of', '10', 'or', '12', 'is'): 0.6570606323425058,\n",
       "             ('10', 'or', '12', 'is', 'impatiently'): 0.6570606323425058,\n",
       "             ('or', '12', 'is', 'impatiently', 'telling'): 0.6570606323425058,\n",
       "             ('12', 'is', 'impatiently', 'telling', 'me'): 0.6570606323425058,\n",
       "             ('is', 'impatiently', 'telling', 'me', ','): 0.6570606323425058,\n",
       "             ('impatiently', 'telling', 'me', ',', '``'): 0.6570606323425058,\n",
       "             ('telling', 'me', ',', '``', 'mother'): 0.1329467319687954,\n",
       "             ('me', ',', '``', 'mother', \"'s\"): 0.6570606323425058,\n",
       "             (',', '``', 'mother', \"'s\", 'asleep'): 0.6570606323425058,\n",
       "             ('``', 'mother', \"'s\", 'asleep', \"''\"): 0.6570606323425058,\n",
       "             ('mother', \"'s\", 'asleep', \"''\", '('): 0.6570606323425058,\n",
       "             (\"'s\", 'asleep', \"''\", '(', 'i'): 0.6570606323425058,\n",
       "             ('asleep', \"''\", '(', 'i', 'think'): 0.6570606323425058,\n",
       "             (\"''\", '(', 'i', 'think', ')'): 0.36219385870234483,\n",
       "             ('(', 'i', 'think', ')', '-'): 0.036335348209211846,\n",
       "             ('i', 'think', ')', '-', 'as'): 0.6570606323425058,\n",
       "             ('think', ')', '-', 'as', 'if'): 0.6570606323425058,\n",
       "             (')', '-', 'as', 'if', 'i'): 0.6570606323425058,\n",
       "             ('-', 'as', 'if', 'i', \"'m\"): 0.6570606323425058,\n",
       "             ('as', 'if', 'i', \"'m\", 'not'): 0.0974575527873556,\n",
       "             ('if', 'i', \"'m\", 'not', 'supposed'): 0.06052671716914943,\n",
       "             ('i', \"'m\", 'not', 'supposed', 'to'): 0.9503951735228037,\n",
       "             (\"'m\", 'not', 'supposed', 'to', 'be'): 0.4751985371546745,\n",
       "             ('not', 'supposed', 'to', 'be', 'doing'): 0.1461858418981423,\n",
       "             ('supposed', 'to', 'be', 'doing', 'what'): 0.03276395093893053,\n",
       "             ('to', 'be', 'doing', 'what', 'i'): 0.7930419197766817,\n",
       "             ('be', 'doing', 'what', 'i', \"'m\"): 0.39652492505828807,\n",
       "             ('doing', 'what', 'i', \"'m\", 'doing'): 0.7930419197766817,\n",
       "             ('what', 'i', \"'m\", 'doing', 'instead'): 0.015262363946377673,\n",
       "             ('i', \"'m\", 'doing', 'instead', 'of'): 0.6570606323425058,\n",
       "             (\"'m\", 'doing', 'instead', 'of', 'her'): 0.6570606323425058,\n",
       "             ('doing', 'instead', 'of', 'her', 'or'): 0.6570606323425058,\n",
       "             ('instead', 'of', 'her', 'or', 'without'): 0.6570606323425058,\n",
       "             ('of', 'her', 'or', 'without', 'her'): 0.6570606323425058,\n",
       "             ('her', 'or', 'without', 'her', 'knowledge'): 0.6570606323425058,\n",
       "             ('or', 'without', 'her', 'knowledge', 'or'): 0.6570606323425058,\n",
       "             ('without',\n",
       "              'her',\n",
       "              'knowledge',\n",
       "              'or',\n",
       "              'something'): 0.6570606323425058,\n",
       "             ('her', 'knowledge', 'or', 'something', '.'): 0.6570606323425058,\n",
       "             ('knowledge', 'or', 'something', '.', 'this'): 0.6570606323425058,\n",
       "             ('or', 'something', '.', 'this', 'is'): 0.19327584918399482,\n",
       "             ('something', '.', 'this', 'is', 'in'): 0.22114746702757215,\n",
       "             ('.', 'this', 'is', 'in', 'an'): 0.1329467319687954,\n",
       "             ('this', 'is', 'in', 'an', 'old'): 0.6570606323425058,\n",
       "             ('is', 'in', 'an', 'old', 'rambling'): 0.28393848879601136,\n",
       "             ('in', 'an', 'old', 'rambling', 'house'): 0.6570606323425058,\n",
       "             ('an', 'old', 'rambling', 'house', 'with'): 0.6570606323425058,\n",
       "             ('old', 'rambling', 'house', 'with', 'lots'): 0.6570606323425058,\n",
       "             ('rambling', 'house', 'with', 'lots', 'of'): 0.6570606323425058,\n",
       "             ('house', 'with', 'lots', 'of', 'old'): 0.19008055533379714,\n",
       "             ('with', 'lots', 'of', 'old', 'furniture'): 0.28393848879601136,\n",
       "             ('lots', 'of', 'old', 'furniture', 'and'): 0.39652492505828807,\n",
       "             ('of',\n",
       "              'old',\n",
       "              'furniture',\n",
       "              'and',\n",
       "              'interconnected'): 0.39652492505828807,\n",
       "             ('old',\n",
       "              'furniture',\n",
       "              'and',\n",
       "              'interconnected',\n",
       "              'rooms'): 0.6570606323425058,\n",
       "             ('furniture',\n",
       "              'and',\n",
       "              'interconnected',\n",
       "              'rooms',\n",
       "              '.'): 0.6570606323425058,\n",
       "             ('and',\n",
       "              'interconnected',\n",
       "              'rooms',\n",
       "              '.',\n",
       "              'lace'): 0.6570606323425058,\n",
       "             ('interconnected',\n",
       "              'rooms',\n",
       "              '.',\n",
       "              'lace',\n",
       "              'doilies'): 0.6570606323425058,\n",
       "             ('rooms', '.', 'lace', 'doilies', 'on'): 0.6570606323425058,\n",
       "             ('.', 'lace', 'doilies', 'on', 'little'): 0.6570606323425058,\n",
       "             ('lace', 'doilies', 'on', 'little', 'tables'): 0.6570606323425058,\n",
       "             ('doilies', 'on', 'little', 'tables', ','): 0.6570606323425058,\n",
       "             ('on', 'little', 'tables', ',', 'throw'): 0.6570606323425058,\n",
       "             ('little', 'tables', ',', 'throw', 'rugs'): 0.6570606323425058,\n",
       "             ('tables', ',', 'throw', 'rugs', ','): 0.6570606323425058,\n",
       "             (',', 'throw', 'rugs', ',', 'etc'): 0.6570606323425058,\n",
       "             ('throw', 'rugs', ',', 'etc', '.'): 0.6570606323425058,\n",
       "             ('rugs', ',', 'etc', '.', 'some'): 0.6570606323425058,\n",
       "             (',', 'etc', '.', 'some', 'of'): 0.44229051119426094,\n",
       "             ('etc', '.', 'some', 'of', '``'): 0.39652492505828807,\n",
       "             ('.', 'some', 'of', '``', 'us'): 0.6570606323425058,\n",
       "             ('some', 'of', '``', 'us', \"''\"): 0.6570606323425058,\n",
       "             ...})"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ngram_probabilities"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4bf8c90d-ddb9-468d-9eb5-8721bf6c5682",
   "metadata": {},
   "source": [
    "## Evaluate perplexity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "ca01a108-3e86-4607-af1f-e92359e3cf4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_perplexity(test_tokens, ngram_probabilities, n):\n",
    "    \"\"\"Calculates the perplexity of a test corpus given n-gram probabilities.\"\"\"\n",
    "    log_probability_sum = 0\n",
    "    ngram_count = 0\n",
    "    \n",
    "    for i in range(len(test_tokens)-n+1):\n",
    "        ngram = tuple(test_tokens[i:i+n])\n",
    "        log_probability_sum += math.log2(ngram_probabilities[ngram])\n",
    "        ngram_count += 1\n",
    "    \n",
    "    average_log_probability = -log_probability_sum / ngram_count\n",
    "    perplexity = math.pow(2, average_log_probability)\n",
    "    \n",
    "    return perplexity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "f71da5c4-9541-4d47-8fdd-284cf609b52f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.4339180118336756"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_perplexity(train_tokens, ngram_probabilities, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "8ea72998-97ed-448b-a379-6f22085c62c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17168.83336216221"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_perplexity(test_tokens, ngram_probabilities, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "d46ea163-f28c-4d53-be0e-1abfc3264a12",
   "metadata": {},
   "outputs": [],
   "source": [
    "def greedy_sampling(context, vocab, ngram_probabilities, n, max_length = 50):\n",
    "    \n",
    "    sentence = []\n",
    "\n",
    "    if len(context) < (n-1):\n",
    "        print(\"len(context) < n\")\n",
    "        return sentence\n",
    "\n",
    "    context = context[-(n-1):]\n",
    "    \n",
    "    for i in range(max_length):\n",
    "\n",
    "        probs = dict()\n",
    "        \n",
    "        for v in vocab:\n",
    "\n",
    "            ngram = list(context)\n",
    "            ngram.append(v)\n",
    "            ngram = tuple(ngram)\n",
    "            probs[v] = ngram_probabilities[ngram]\n",
    "\n",
    "        best_token = max(probs, key=probs.get) # greedy \n",
    "        #print(best_v)\n",
    "        #print(probs[best_v])\n",
    "        \n",
    "        if probs[best_token] == 0:\n",
    "            print(\"prob = 0\")\n",
    "            return sentence\n",
    "            \n",
    "        sentence.append(best_token)\n",
    "        context = list(context)[1:]\n",
    "        context.append(best_token)\n",
    "        context = tuple(context)\n",
    "            \n",
    "    return sentence  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "73f14e1b-82c1-4520-89d2-f245778009fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the one at the meads 's house , where i was going . i was walking down the street and i was on the phone . i am constantly interrupted by the antics of his 2-year-old boy who i am apparently babysitting . the boy runs and jumps from a high place . i yell , `` get him in the car ! i 'll be right back . '' i said , `` i 'm going to be late for work and i ca n't remember what it was . i was in a room with my friends niles zelling , matt yams , queen billman , and another woman . i am in a house . i was in a room with my friends niles zelling , matt yams , queen billman , and another woman . i am in a house . i was in a room with my friends niles zelling , matt yams , queen billman , and another woman . i am in a house . i was in a room with my friends niles zelling , matt yams , queen billman , and another woman . i am in a house . i was in a room\n"
     ]
    }
   ],
   "source": [
    "context = ['i', \"'m\", 'in', 'honduras']\n",
    "context = ['the', 'one', 'at', 'the']\n",
    "sentence = greedy_sampling(context, vocab, ngram_probabilities, n, max_length = 200)\n",
    "print(\" \".join(context) + \" \" +  \" \".join(sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "f3578700-cc29-42a6-a15a-7593ffcf6a17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"the one at the meads's house, where it's bigger inside than out; there's a european village just inside, with a cobblestone street and a pied-piper sort of man with curly hair, he can do things like juggle - i go up the back stairs [there aren't any \""
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_strings[:250]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd223e31-51fe-4d12-b151-44b7ca44ea5d",
   "metadata": {},
   "source": [
    "## TODO\n",
    "\n",
    "padding, backoff, interpolation, better sampling methods (top-k, top-p), etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44bbb323-a890-4280-b25c-5d686f631209",
   "metadata": {},
   "outputs": [],
   "source": [
    "def padding(sequence, n, pad_left=False, pad_right=False, left_pad_symbol=None, right_pad_symbol=None):\n",
    "    if pad_left:\n",
    "        sequence = [left_pad_symbol] * (n-1) + sequence\n",
    "    if pad_right:\n",
    "        sequence += [right_pad_symbol] * (n-1)\n",
    "    return sequence"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f2e37c4-cea3-4e86-b197-e096fef751eb",
   "metadata": {},
   "source": [
    "# Using NLTK.lm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c8f26cb-20c6-4dc5-96b7-4e7740e1a27d",
   "metadata": {},
   "source": [
    "## Import dataset and prepare training and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "5cbd22bd-1454-46ea-8467-99476ebfac53",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.lm.preprocessing import padded_everygram_pipeline\n",
    "from nltk.lm import MLE, StupidBackoff, Laplace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "feed5c71-ec83-4e0c-8468-860c95381140",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "351146\n",
      "5627\n"
     ]
    }
   ],
   "source": [
    "train_sents = sent_tokenize(train_strings)\n",
    "print(len(train_sents))\n",
    "train_sents_tokens = [word_tokenize(s) for s in train_sents]\n",
    "\n",
    "test_sents = sent_tokenize(test_strings)\n",
    "print(len(test_sents))\n",
    "test_sents_tokens = [word_tokenize(s) for s in test_sents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "0c05025e-0192-4457-b826-88be9af0b482",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['i',\n",
       "  \"'m\",\n",
       "  'in',\n",
       "  'honduras',\n",
       "  'at',\n",
       "  'some',\n",
       "  'kind',\n",
       "  'of',\n",
       "  'border',\n",
       "  'crossing',\n",
       "  '.'],\n",
       " ['a',\n",
       "  'wide',\n",
       "  'corridor',\n",
       "  'of',\n",
       "  'thick',\n",
       "  'fenced',\n",
       "  'scrub',\n",
       "  'is',\n",
       "  'off',\n",
       "  'on',\n",
       "  'either',\n",
       "  'side',\n",
       "  '..',\n",
       "  'everything',\n",
       "  'is',\n",
       "  'grim',\n",
       "  ',',\n",
       "  'sinister',\n",
       "  ',',\n",
       "  'ominous',\n",
       "  '.']]"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_sents_tokens[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "d75bcc6c-02ac-452b-b5a0-59b6f093b248",
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 3\n",
    "\n",
    "train, vocab = padded_everygram_pipeline(n, train_sents_tokens)\n",
    "\n",
    "#lm = MLE(n) # Maximum Likelihood Estimate\n",
    "#lm = StupidBackoff(order = n) # Stupid Backoff\n",
    "lm = Laplace(n) # Laplace smoothing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "cacd176d-f4a0-4f2c-8d70-5ffc0804ba24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Vocabulary with cutoff=1 unk_label='<UNK>' and 52182 items>\n",
      "52182\n"
     ]
    }
   ],
   "source": [
    "lm.fit(train, vocab)\n",
    "print(lm.vocab)\n",
    "print(len(lm.vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "ef014e11-ceb0-4785-a9b3-3af500b03413",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('france', '<UNK>')"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.vocab.lookup([\"france\", \"gustave\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "8308ed30-2eb0-4987-966a-32fc56dd7c3f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<NgramCounter with 3 ngram orders and 19774470 ngrams>\n"
     ]
    }
   ],
   "source": [
    "print(lm.counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "fcd46cf0-dae9-4d7c-9825-86ba2f0506eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.counts['gustave']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "9f01f954-1ba0-49fe-97cc-594a3aa0e000",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "62"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.counts['france']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "f716ef3a-16e8-4ea7-9956-80fed4fd7f9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7106"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.counts['dream']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "4405f101-c325-48ac-9ccb-0a483103c09a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "288"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.counts[['i']]['love']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "83ad8699-46b0-484e-aa12-619e0154126c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1624"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.counts[['i']]['want']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "c7ad39c9-4768-4ef2-b6b5-d6be0894f954",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.038742394727067954"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.score(\"i\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "5c4618dd-0c34-4556-9a96-037e32855828",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9.006667507288967e-06"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.score(\"france\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "61a74a0b-4889-424e-b771-56a2788282dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.429629763061741e-07"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.score(\"gustave\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "517dc0b5-c865-4fd8-8d37-65e757f55fe8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7.431076763022962e-05"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.score(\"i\", [\"love\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "23210af9-686c-4007-9882-17afa4e59395",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50465.94290553111"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.perplexity(train_sents_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "3067bfc8-0d28-4edd-8597-f469c7958cac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "49653.97430774097"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.perplexity(test_sents_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "3ff8d6ad-f125-4ac6-8680-1275af7218c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['land',\n",
       " ')',\n",
       " 'blue',\n",
       " ',',\n",
       " 'showing',\n",
       " 'my',\n",
       " 'stamina',\n",
       " '.',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>']"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.generate(20, random_seed=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "6fde7ae5-5d7b-4c55-b1e3-9604b15edae5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['say',\n",
       " ',',\n",
       " '``',\n",
       " 'hon',\n",
       " ',',\n",
       " 'relax',\n",
       " '.',\n",
       " \"''\",\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>',\n",
       " '</s>']"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm.generate(20, text_seed=['i'], random_seed=42)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}