06_nlp_snowball_dbscan_k_means_clustering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLP: Examining TF-IDF on Snowball Stemmed Text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.preprocessing import Normalizer\n",
    "from sklearn.cluster import KMeans, DBSCAN, MeanShift, AgglomerativeClustering, Birch, SpectralClustering\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('news_articles_stemmed_lemmatized_no_extra_stops.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 84155 entries, 0 to 84154\n",
      "Data columns (total 10 columns):\n",
      "article_text            84155 non-null object\n",
      "datetime                84155 non-null object\n",
      "source                  84155 non-null object\n",
      "title                   84155 non-null object\n",
      "url                     84155 non-null object\n",
      "cleaned_article_text    84155 non-null object\n",
      "text_lancaster          84155 non-null object\n",
      "text_porter             84155 non-null object\n",
      "text_snowball           84155 non-null object\n",
      "text_lemmatize          84155 non-null object\n",
      "dtypes: object(10)\n",
      "memory usage: 6.4+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "snowball_df = df.drop(['article_text','cleaned_article_text','text_lancaster',\n",
    "         'text_porter','text_lemmatize'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 84155 entries, 0 to 84154\n",
      "Data columns (total 5 columns):\n",
      "datetime         84155 non-null object\n",
      "source           84155 non-null object\n",
      "title            84155 non-null object\n",
      "url              84155 non-null object\n",
      "text_snowball    84155 non-null object\n",
      "dtypes: object(5)\n",
      "memory usage: 3.2+ MB\n"
     ]
    }
   ],
   "source": [
    "snowball_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "snowball = snowball_df['text_snowball'].tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vectorize / LSA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "tf_idf = TfidfVectorizer(stop_words='english', ngram_range=(1, 4), min_df=25, max_df=0.98)\n",
    "tf_idf_vecs = tf_idf.fit_transform(snowball)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lsa = TruncatedSVD(400, algorithm='arpack')\n",
    "lsa_vecs = lsa.fit_transform(tf_idf_vecs)\n",
    "lsa_vecs = Normalizer(copy=False).fit_transform(lsa_vecs)\n",
    "feature_names = tf_idf.get_feature_names()\n",
    "lsa_df = pd.DataFrame(lsa.components_.round(5), columns=feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save('tf_idf_vecs', tf_idf_vecs)\n",
    "np.save('lsa_vecs', lsa_vecs)\n",
    "np.save('feature_names', feature_names)\n",
    "lsa_df.to_pickle('lsa_df.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cluster"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DBSCAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "db = DBSCAN(n_jobs=-1, eps=0.5, min_samples=25)\n",
    "db_clusters = db.fit_predict(lsa_vecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "db_2 = DBSCAN(n_jobs=-1, eps=0.6, min_samples=25)\n",
    "db_2_clusters = db_2.fit_predict(lsa_vecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "db_3 = DBSCAN(n_jobs=-1, eps=0.7, min_samples=15)\n",
    "db_3_clusters = db_3.fit_predict(lsa_vecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "84155\n",
      "84155\n",
      "84155\n"
     ]
    }
   ],
   "source": [
    "print(len(lsa_vecs))\n",
    "print(len(db_clusters))\n",
    "print(len(snowball_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "d = {'db_1_clusters':db_clusters, 'db_2_clusters':db_2_clusters, 'db_3_clusters':db_3_clusters}\n",
    "db_clusters_df = pd.DataFrame(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>db_1_clusters</th>\n",
       "      <th>db_2_clusters</th>\n",
       "      <th>db_3_clusters</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>77857</th>\n",
       "      <td>-1</td>\n",
       "      <td>110</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11399</th>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4437</th>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38588</th>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59934</th>\n",
       "      <td>7</td>\n",
       "      <td>35</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24903</th>\n",
       "      <td>16</td>\n",
       "      <td>48</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29971</th>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14244</th>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77312</th>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26957</th>\n",
       "      <td>-1</td>\n",
       "      <td>67</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       db_1_clusters  db_2_clusters  db_3_clusters\n",
       "77857             -1            110              1\n",
       "11399             -1             -1             -1\n",
       "4437              -1             -1             -1\n",
       "38588             -1             -1             -1\n",
       "59934              7             35             45\n",
       "24903             16             48             57\n",
       "29971             -1             -1             80\n",
       "14244             -1             -1             -1\n",
       "77312             -1             -1             -1\n",
       "26957             -1             67             61"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "db_clusters_df.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explore cluster sizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x205057bdc18>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGe5JREFUeJzt3X+M1Pd95/Hnq5A4hNQE7NxotaBbKiNXGM5OWVFyiapt\nuNSbpAr+A1sbOTU+UVPJtE1OSD24ky7XP5Bs6Vw31p2RUJyC3ZwxpfGB4pKWrDM63ekMwT9SDJiy\nDVB2b4EEO+bWPXNZ+r4/5jPNd+czdGdnZ3aH8HpIo/3Me76fz76/K+zXfH/MriICMzOzol+Y7QbM\nzKzzOBzMzCzjcDAzs4zDwczMMg4HMzPLOBzMzCzjcDAzs0xD4SDp30g6LulNSc9L+pCkRZIOSTqd\nvi4sbL9N0pCkU5LuLdRXSTqWXntKklL9FkkvpPphST2t3lEzM2vcpOEgqRv4faA3IlYAc4ABYCsw\nGBHLgMH0HEnL0+t3Af3A05LmpOV2AI8Ay9KjP9U3Au9ExB3Ak8DjLdk7MzNrytwpbDdP0k+BDwP/\nG9gG9KXXdwNl4N8C64A9EXEVOCNpCFgt6Sxwa0S8AiDpWeA+4GCa8x/TWvuA/yxJ8U98fPv222+P\nnp6eBtuf3Hvvvcf8+fNbtl673Ch9gnttF/faHjdLr6+++uqPI+Jjk203aThExIik/wT8HfB/gb+K\niL+SVIqI0bTZBaCUxt3AK4UlhlPtp2lcW6/OOZ++37ikd4HbgB9fr6+enh6OHj06WfsNK5fL9PX1\ntWy9drlR+gT32i7utT1ull4lnWtku0nDIV1LWAcsBX4C/JmkLxW3iYiQ1PZf0iRpE7AJoFQqUS6X\nW7b22NhYS9drlxulT3Cv7eJe28O9TtTIaaV/BZyJiB8BSPoW8C+Bi5K6ImJUUhdwKW0/AiwpzF+c\naiNpXFsvzhmWNBdYAFyubSQidgI7AXp7e6OVKX+jvGu4UfoE99ou7rU93OtEjdyt9HfAGkkfTncX\nrQVOAgeADWmbDcD+ND4ADKQ7kJZSufB8JJ2CuiJpTVrnoZo51bXWAy//U9cbzMysvRq55nBY0j7g\nNWAceJ3Ku/ePAHslbQTOAQ+k7Y9L2gucSNtvjohrablHgV3APCoXog+m+jPAc+ni9dtU7nYyM7NZ\n0tDdShHxVeCrNeWrVI4i6m2/Hdhep34UWFGn/j5wfyO9mJlZ+/kT0mZmlnE4mJlZxuFgZmYZh4OZ\nmWUa/fUZP1d6tr6U1basHOfhOvVWO/vY59v+PczMpstHDmZmlnE4mJlZxuFgZmYZh4OZmWUcDmZm\nlnE4mJlZxuFgZmYZh4OZmWUcDmZmlnE4mJlZxuFgZmYZh4OZmWUcDmZmlpk0HCTdKemNwuOKpK9I\nWiTpkKTT6evCwpxtkoYknZJ0b6G+StKx9NpTkpTqt0h6IdUPS+ppx86amVljJg2HiDgVEfdExD3A\nKuDvgReBrcBgRCwDBtNzJC0HBoC7gH7gaUlz0nI7gEeAZenRn+obgXci4g7gSeDx1uyemZk1Y6qn\nldYCfxsR54B1wO5U3w3cl8brgD0RcTUizgBDwGpJXcCtEfFKRATwbM2c6lr7gLXVowozM5t5Uw2H\nAeD5NC5FxGgaXwBKadwNnC/MGU617jSurU+YExHjwLvAbVPszczMWqThvwQn6YPAF4Btta9FREiK\nVjZ2nR42AZsASqUS5XK5qXW2rBzPaqV59eut1mzPVWNjY9NeY6a41/Zwr+3hXieayp8J/SzwWkRc\nTM8vSuqKiNF0yuhSqo8ASwrzFqfaSBrX1otzhiXNBRYAl2sbiIidwE6A3t7e6Ovrm0L7P1Pvz4Fu\nWTnOE8fa/1dTzz7YN6355XKZZvd7prnX9nCv7eFeJ5rKaaUv8rNTSgAHgA1pvAHYX6gPpDuQllK5\n8HwknYK6ImlNup7wUM2c6lrrgZfTdQkzM5sFDb1VljQf+AzwO4XyY8BeSRuBc8ADABFxXNJe4AQw\nDmyOiGtpzqPALmAecDA9AJ4BnpM0BLxN5dqGmZnNkobCISLeo+YCcURcpnL3Ur3ttwPb69SPAivq\n1N8H7m+kFzMzaz9/QtrMzDIOBzMzyzgczMws43AwM7OMw8HMzDIOBzMzyzgczMws43AwM7OMw8HM\nzDIOBzMzyzgczMws43AwM7OMw8HMzDIOBzMzyzgczMws43AwM7OMw8HMzDIOBzMzyzQUDpI+Kmmf\npLcknZT0CUmLJB2SdDp9XVjYfpukIUmnJN1bqK+SdCy99pQkpfotkl5I9cOSelq9o2Zm1rhGjxy+\nBnwnIn4ZuBs4CWwFBiNiGTCYniNpOTAA3AX0A09LmpPW2QE8AixLj/5U3wi8ExF3AE8Cj09zv8zM\nbBomDQdJC4BfA54BiIj/FxE/AdYBu9Nmu4H70ngdsCcirkbEGWAIWC2pC7g1Il6JiACerZlTXWsf\nsLZ6VGFmZjOvkSOHpcCPgD+R9Lqkr0uaD5QiYjRtcwEopXE3cL4wfzjVutO4tj5hTkSMA+8Ct019\nd8zMrBXmNrjNrwC/FxGHJX2NdAqpKiJCUrSjwSJJm4BNAKVSiXK53NQ6W1aOZ7XSvPr1Vmu256qx\nsbFprzFT3Gt7uNf2cK8TNRIOw8BwRBxOz/dRCYeLkroiYjSdMrqUXh8BlhTmL061kTSurRfnDEua\nCywALtc2EhE7gZ0Avb290dfX10D7uYe3vpTVtqwc54ljjfw4pufsg33Tml8ul2l2v2eae20P99oe\n7nWiSU8rRcQF4LykO1NpLXACOABsSLUNwP40PgAMpDuQllK58HwknYK6ImlNup7wUM2c6lrrgZfT\ndQkzM5sFjb5V/j3gm5I+CPwQ+NdUgmWvpI3AOeABgIg4LmkvlQAZBzZHxLW0zqPALmAecDA9oHKx\n+zlJQ8DbVO52MjOzWdJQOETEG0BvnZfWXmf77cD2OvWjwIo69feB+xvpxczM2s+fkDYzs4zDwczM\nMg4HMzPLOBzMzCzjcDAzs4zDwczMMg4HMzPLOBzMzCzjcDAzs4zDwczMMg4HMzPLOBzMzCzjcDAz\ns4zDwczMMg4HMzPLOBzMzCzjcDAzs4zDwczMMg2Fg6Szko5JekPS0VRbJOmQpNPp68LC9tskDUk6\nJeneQn1VWmdI0lOSlOq3SHoh1Q9L6mntbpqZ2VRM5cjh1yPinoio/i3prcBgRCwDBtNzJC0HBoC7\ngH7gaUlz0pwdwCPAsvToT/WNwDsRcQfwJPB487tkZmbTNZ3TSuuA3Wm8G7ivUN8TEVcj4gwwBKyW\n1AXcGhGvREQAz9bMqa61D1hbPaowM7OZ12g4BPBdSa9K2pRqpYgYTeMLQCmNu4HzhbnDqdadxrX1\nCXMiYhx4F7htCvthZmYtNLfB7T4VESOS/hlwSNJbxRcjIiRF69ubKAXTJoBSqUS5XG5qnS0rx7Na\naV79eqs123PV2NjYtNeYKe61Pdxre7jXiRoKh4gYSV8vSXoRWA1clNQVEaPplNGltPkIsKQwfXGq\njaRxbb04Z1jSXGABcLlOHzuBnQC9vb3R19fXSPuZh7e+lNW2rBzniWONZmXzzj7YN6355XKZZvd7\nprnX9nCv7eFeJ5r0tJKk+ZJ+sToGfgN4EzgAbEibbQD2p/EBYCDdgbSUyoXnI+kU1BVJa9L1hIdq\n5lTXWg+8nK5LmJnZLGjkrXIJeDFdH54L/NeI+I6k7wN7JW0EzgEPAETEcUl7gRPAOLA5Iq6ltR4F\ndgHzgIPpAfAM8JykIeBtKnc7mZnZLJk0HCLih8DddeqXgbXXmbMd2F6nfhRYUaf+PnB/A/2amdkM\n8Cekzcws43AwM7OMw8HMzDIOBzMzyzgczMws43AwM7OMw8HMzDIOBzMzyzgczMws43AwM7OMw8HM\nzDIOBzMzyzgczMws43AwM7OMw8HMzDIOBzMzyzgczMws43AwM7NMw+EgaY6k1yV9Oz1fJOmQpNPp\n68LCttskDUk6JeneQn2VpGPptaeU/jC1pFskvZDqhyX1tG4XzcxsqqZy5PBl4GTh+VZgMCKWAYPp\nOZKWAwPAXUA/8LSkOWnODuARYFl69Kf6RuCdiLgDeBJ4vKm9MTOzlmgoHCQtBj4PfL1QXgfsTuPd\nwH2F+p6IuBoRZ4AhYLWkLuDWiHglIgJ4tmZOda19wNrqUYWZmc28Ro8c/hj4A+AfCrVSRIym8QWg\nlMbdwPnCdsOp1p3GtfUJcyJiHHgXuK3B3szMrMXmTraBpN8ELkXEq5L66m0TESEpWt1cnV42AZsA\nSqUS5XK5qXW2rBzPaqV59eut1mzPVWNjY9NeY6a41/Zwr+3hXieaNByATwJfkPQ54EPArZL+FLgo\nqSsiRtMpo0tp+xFgSWH+4lQbSePaenHOsKS5wALgcm0jEbET2AnQ29sbfX19De1krYe3vpTVtqwc\n54ljjfw4pufsg33Tml8ul2l2v2eae20P99oe7nWiSU8rRcS2iFgcET1ULjS/HBFfAg4AG9JmG4D9\naXwAGEh3IC2lcuH5SDoFdUXSmnQ94aGaOdW11qfv0fYjETMzq286b5UfA/ZK2gicAx4AiIjjkvYC\nJ4BxYHNEXEtzHgV2AfOAg+kB8AzwnKQh4G0qIWRmZrNkSuEQEWWgnMaXgbXX2W47sL1O/Siwok79\nfeD+qfRiZmbt409Im5lZxuFgZmYZh4OZmWUcDmZmlnE4mJlZxuFgZmYZh4OZmWUcDmZmlnE4mJlZ\nxuFgZmYZh4OZmWUcDmZmlnE4mJlZxuFgZmYZh4OZmWUcDmZmlnE4mJlZxuFgZmaZScNB0ockHZH0\nA0nHJf1hqi+SdEjS6fR1YWHONklDkk5JurdQXyXpWHrtKUlK9VskvZDqhyX1tH5XzcysUY0cOVwF\nPh0RdwP3AP2S1gBbgcGIWAYMpudIWg4MAHcB/cDTkuaktXYAjwDL0qM/1TcC70TEHcCTwOMt2Dcz\nM2vSpOEQFWPp6QfSI4B1wO5U3w3cl8brgD0RcTUizgBDwGpJXcCtEfFKRATwbM2c6lr7gLXVowoz\nM5t5DV1zkDRH0hvAJeBQRBwGShExmja5AJTSuBs4X5g+nGrdaVxbnzAnIsaBd4Hbprw3ZmbWEnMb\n2SgirgH3SPoo8KKkFTWvh6RoR4NFkjYBmwBKpRLlcrmpdbasHM9qpXn1663WbM9VY2Nj015jprjX\n9nCv7eFeJ2ooHKoi4ieSvkflWsFFSV0RMZpOGV1Km40ASwrTFqfaSBrX1otzhiXNBRYAl+t8/53A\nToDe3t7o6+ubSvv/6OGtL2W1LSvHeeLYlH4cTTn7YN+05pfLZZrd75nmXtvDvbaHe52okbuVPpaO\nGJA0D/gM8BZwANiQNtsA7E/jA8BAugNpKZULz0fSKagrktak6wkP1cyprrUeeDldlzAzs1nQyFvl\nLmB3uuPoF4C9EfFtSf8L2CtpI3AOeAAgIo5L2gucAMaBzem0FMCjwC5gHnAwPQCeAZ6TNAS8TeVu\nJzMzmyWThkNE/DXw8Tr1y8Da68zZDmyvUz8KrKhTfx+4v4F+zcxsBvgT0mZmlnE4mJlZxuFgZmYZ\nh4OZmWUcDmZmlnE4mJlZxuFgZmYZh4OZmWUcDmZmlnE4mJlZxuFgZmYZh4OZmWUcDmZmlnE4mJlZ\nxuFgZmYZh4OZmWUcDmZmlnE4mJlZZtJwkLRE0vcknZB0XNKXU32RpEOSTqevCwtztkkaknRK0r2F\n+ipJx9JrT0lSqt8i6YVUPyypp/W7amZmjWrkyGEc2BIRy4E1wGZJy4GtwGBELAMG03PSawPAXUA/\n8LSkOWmtHcAjwLL06E/1jcA7EXEH8CTweAv2zczMmjRpOETEaES8lsb/BzgJdAPrgN1ps93AfWm8\nDtgTEVcj4gwwBKyW1AXcGhGvREQAz9bMqa61D1hbPaowM7OZN6VrDul0z8eBw0ApIkbTSxeAUhp3\nA+cL04ZTrTuNa+sT5kTEOPAucNtUejMzs9aZ2+iGkj4C/DnwlYi4UnxjHxEhKdrQX20Pm4BNAKVS\niXK53NQ6W1aOZ7XSvPr1Vmu256qxsbFprzFT3Gt7uNf2cK8TNRQOkj5AJRi+GRHfSuWLkroiYjSd\nMrqU6iPAksL0xak2ksa19eKcYUlzgQXA5do+ImInsBOgt7c3+vr6Gmk/8/DWl7LalpXjPHGs4axs\n2tkH+6Y1v1wu0+x+zzT32h7utT3c60SN3K0k4BngZET8UeGlA8CGNN4A7C/UB9IdSEupXHg+kk5B\nXZG0Jq35UM2c6lrrgZfTdQkzM5sFjbxV/iTwW8AxSW+k2r8DHgP2StoInAMeAIiI45L2Aieo3Om0\nOSKupXmPAruAecDB9IBK+DwnaQh4m8rdTmZmNksmDYeI+B/A9e4cWnudOduB7XXqR4EVdervA/dP\n1ouZmc0Mf0LazMwyDgczM8s4HMzMLONwMDOzjMPBzMwyDgczM8s4HMzMLONwMDOzjMPBzMwyDgcz\nM8s4HMzMLONwMDOzjMPBzMwyDgczM8s4HMzMLONwMDOzjMPBzMwyDgczM8tMGg6SviHpkqQ3C7VF\nkg5JOp2+Liy8tk3SkKRTku4t1FdJOpZee0qSUv0WSS+k+mFJPa3dRTMzm6pGjhx2Af01ta3AYEQs\nAwbTcyQtBwaAu9KcpyXNSXN2AI8Ay9KjuuZG4J2IuAN4Eni82Z0xM7PWmDQcIuK/A2/XlNcBu9N4\nN3Bfob4nIq5GxBlgCFgtqQu4NSJeiYgAnq2ZU11rH7C2elRhZmazo9lrDqWIGE3jC0ApjbuB84Xt\nhlOtO41r6xPmRMQ48C5wW5N9mZlZC8yd7gIREZKiFc1MRtImYBNAqVSiXC43tc6WleNZrTSvfr3V\nmu25amxsbNprzBT32h7utT3c60TNhsNFSV0RMZpOGV1K9RFgSWG7xak2ksa19eKcYUlzgQXA5Xrf\nNCJ2AjsBent7o6+vr6nmH976UlbbsnKcJ45NOysndfbBvmnNL5fLNLvfM829tod7bQ/3OlGzp5UO\nABvSeAOwv1AfSHcgLaVy4flIOgV1RdKadD3hoZo51bXWAy+n6xJmZjZLJn2rLOl5oA+4XdIw8FXg\nMWCvpI3AOeABgIg4LmkvcAIYBzZHxLW01KNU7nyaBxxMD4BngOckDVG58D3Qkj0zM7OmTRoOEfHF\n67y09jrbbwe216kfBVbUqb8P3D9ZH2ZmNnP8CWkzM8s4HMzMLONwMDOzjMPBzMwyDgczM8s4HMzM\nLONwMDOzjMPBzMwyDgczM8s4HMzMLONwMDOzjMPBzMwyDgczM8s4HMzMLONwMDOzjMPBzMwy7f+j\nyXbT66nzN7tnwtnHPj8r39fs54GPHMzMLNMx4SCpX9IpSUOSts52P2ZmN7OOOK0kaQ7wX4DPAMPA\n9yUdiIgTs9tZ6033FMuWleM8PEunaabqRur154VP4VmrdEQ4AKuBoYj4IYCkPcA64OcuHGzmTOV/\nlDdSkHVir9f7Wc9Erw6m9uiUcOgGzheeDwO/Oku9mNkNpFVHS50Yutezq39+27+HIqLt32TSJqT1\nQH9E/HZ6/lvAr0bE79ZstwnYlJ7eCZxqYRu3Az9u4XrtcqP0Ce61Xdxre9wsvf7ziPjYZBt1ypHD\nCLCk8Hxxqk0QETuBne1oQNLRiOhtx9qtdKP0Ce61Xdxre7jXiTrlbqXvA8skLZX0QWAAODDLPZmZ\n3bQ64sghIsYl/S7wl8Ac4BsRcXyW2zIzu2l1RDgARMRfAH8xiy205XRVG9wofYJ7bRf32h7utaAj\nLkibmVln6ZRrDmZm1kFu+nDo5F/bIekbki5JerNQWyTpkKTT6evC2eyxStISSd+TdELScUlfTvWO\n61fShyQdkfSD1OsfdmqvUPkNApJel/Tt9LxT+zwr6ZikNyQdTbVO7fWjkvZJekvSSUmf6MReJd2Z\nfp7VxxVJX5mJXm/qcCj82o7PAsuBL0paPrtdTbAL6K+pbQUGI2IZMJied4JxYEtELAfWAJvTz7IT\n+70KfDoi7gbuAfolraEzewX4MnCy8LxT+wT49Yi4p3CbZaf2+jXgOxHxy8DdVH6+HddrRJxKP897\ngFXA3wMvMhO9RsRN+wA+Afxl4fk2YNts91XTYw/wZuH5KaArjbuAU7Pd43X63k/ld2V1dL/Ah4HX\nqHwiv+N6pfKZn0Hg08C3O/nfAHAWuL2m1nG9AguAM6Rrrp3ca01/vwH8z5nq9aY+cqD+r+3onqVe\nGlWKiNE0vgCUZrOZeiT1AB8HDtOh/aZTNW8Al4BDEdGpvf4x8AfAPxRqndgnQADflfRq+m0G0Jm9\nLgV+BPxJOl33dUnz6cxeiwaA59O47b3e7OFwQ4vK24aOut1M0keAPwe+EhFXiq91Ur8RcS0qh+qL\ngdWSVtS8Puu9SvpN4FJEvHq9bTqhz4JPpZ/pZ6mcVvy14osd1Otc4FeAHRHxceA9ak7LdFCvAKQP\nB38B+LPa19rV680eDg392o4Oc1FSF0D6emmW+/lHkj5AJRi+GRHfSuWO7RcgIn4CfI/KtZ1O6/WT\nwBcknQX2AJ+W9Kd0Xp8ARMRI+nqJynnx1XRmr8PAcDpaBNhHJSw6sdeqzwKvRcTF9Lztvd7s4XAj\n/tqOA8CGNN5A5dz+rJMk4BngZET8UeGljutX0sckfTSN51G5NvIWHdZrRGyLiMUR0UPl3+bLEfEl\nOqxPAEnzJf1idUzl/PibdGCvEXEBOC/pzlRaS+XPA3RcrwVf5GenlGAmep3tiyyz/QA+B/wN8LfA\nv5/tfmp6ex4YBX5K5d3ORuA2KhcoTwPfBRbNdp+p109RObT9a+CN9PhcJ/YL/Avg9dTrm8B/SPWO\n67XQcx8/uyDdcX0CvwT8ID2OV/9b6sReU1/3AEfTv4H/Bizs4F7nA5eBBYVa23v1J6TNzCxzs59W\nMjOzOhwOZmaWcTiYmVnG4WBmZhmHg5mZZRwOZmaWcTiYmVnG4WBmZpn/D1BhuFXWDaSgAAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x20464019278>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "db_clusters_df['db_1_clusters'].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x2041ed3c978>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF99JREFUeJzt3WGMVed95/Hvr0xCqFsTsLNXLKAdJI9SYVCcesTSTVXN\ndjbrSVIFv7CtiZwar1hTybRNVkgVbF+s+gLJltZ1Y2ltCdmpx242GNFkjeI6uxTnqqpUIDhxi8Gm\nTAOUmQ6QYMd0vDLJeP/74j5Dz9xnmLlz587cmcPvI13Nc/73POc+/zGZ39xzzp0oIjAzMyv6hXYv\nwMzMFh6Hg5mZZRwOZmaWcTiYmVnG4WBmZhmHg5mZZRwOZmaWcTiYmVnG4WBmZpmOdi+gWbfffnt0\ndnbO+jjvv/8+t9xyy+wXtICUsScoZ19l7AnK2VdZenr99dd/EhGfmG6/RRsOnZ2dHD9+fNbHqVar\n9PT0zH5BC0gZe4Jy9lXGnqCcfZWlJ0nnG9nPp5XMzCzjcDAzs4zDwczMMg4HMzPLOBzMzCwzbThI\n+qSkNwqPq5K+KmmlpEOSzqSvKwpzdksalHRa0j2F+t2STqTnnpKkVF8q6aVUPyqpcy6aNTOzxkwb\nDhFxOiLuioi7gLuB/wt8G9gFHI6ILuBw2kbSeqAfuBPoA56WtCQd7hngEaArPfpSfRvwbkTcATwJ\nPN6a9szMrBkzPa3UC/xDRJwHtgADqT4A3JvGW4B9EXEtIs4Cg8AmSauAWyPiSNT+v0lfqJszfqwD\nQO/4uwozM5t/M/0QXD/wzTSuRMRIGl8EKmm8GjhSmDOUaj9P4/r6+JwLABExJuk94DbgJ8UXl7Qd\n2A5QqVSoVqszXH5udHS0JcdZSMrYE5SzrzL2BOXsq4w9TaXhcJD0UeCLwO765yIiJEUrFzaZiNgL\n7AXo7u6OZj+t2LnrlevjnRs/5Im/fr8Vy2vIuce+MOevUZZPctYrY19l7AnK2VcZe5rKTE4rfQ74\nQURcStuX0qki0tfLqT4MrC3MW5Nqw2lcX58wR1IHsBy4MoO1mZlZC80kHL7Ev5xSAjgIbE3jrcDL\nhXp/ugNpHbULz8fSKairkjan6wkP1c0ZP9Z9wGvpuoSZmbVBQ6eVJN0CfBb4nUL5MWC/pG3AeeAB\ngIg4KWk/cAoYA3ZExIdpzqPA88Ay4NX0AHgOeFHSIPAOtWsbZmbWJg2FQ0S8T+0CcbF2hdrdS5Pt\nvwfYM0n9OLBhkvoHwP2NrMXMzOaePyFtZmYZh4OZmWUcDmZmlnE4mJlZxuFgZmYZh4OZmWUcDmZm\nlnE4mJlZxuFgZmYZh4OZmWUcDmZmlnE4mJlZxuFgZmYZh4OZmWUcDmZmlnE4mJlZxuFgZmYZh4OZ\nmWUcDmZmlnE4mJlZpqFwkPRxSQckvS3pLUm/JmmlpEOSzqSvKwr775Y0KOm0pHsK9bslnUjPPSVJ\nqb5U0kupflRSZ6sbNTOzxjX6zuFrwHcj4leATwFvAbuAwxHRBRxO20haD/QDdwJ9wNOSlqTjPAM8\nAnSlR1+qbwPejYg7gCeBx2fZl5mZzcK04SBpOfAbwHMAEfGziPgpsAUYSLsNAPem8RZgX0Rci4iz\nwCCwSdIq4NaIOBIRAbxQN2f8WAeA3vF3FWZmNv8aeeewDvgx8KeSfijpWUm3AJWIGEn7XAQqabwa\nuFCYP5Rqq9O4vj5hTkSMAe8Bt828HTMza4WOBvf5VeD3IuKopK+RTiGNi4iQFHOxwCJJ24HtAJVK\nhWq12tRxdm4cuz6uLJu4PdeaXfNMjI6OzsvrzLcy9lXGnqCcfZWxp6k0Eg5DwFBEHE3bB6iFwyVJ\nqyJiJJ0yupyeHwbWFuavSbXhNK6vF+cMSeoAlgNX6hcSEXuBvQDd3d3R09PTwPJzD+965fp458Yx\nnjjRyLehNc492DPnr1GtVmn2e7OQlbGvMvYE5eyrjD1NZdrTShFxEbgg6ZOp1AucAg4CW1NtK/By\nGh8E+tMdSOuoXXg+lk5BXZW0OV1PeKhuzvix7gNeS9clzMysDRr9lfn3gG9I+ijwI+A/UQuW/ZK2\nAeeBBwAi4qSk/dQCZAzYEREfpuM8CjwPLANeTQ+oXex+UdIg8A61u53MzKxNGgqHiHgD6J7kqd4b\n7L8H2DNJ/TiwYZL6B8D9jazFzMzmnj8hbWZmGYeDmZllHA5mZpZxOJiZWcbhYGZmGYeDmZllHA5m\nZpZxOJiZWcbhYGZmGYeDmZllHA5mZpZxOJiZWcbhYGZmGYeDmZllHA5mZpZxOJiZWcbhYGZmGYeD\nmZllHA5mZpZxOJiZWcbhYGZmmYbCQdI5SSckvSHpeKqtlHRI0pn0dUVh/92SBiWdlnRPoX53Os6g\npKckKdWXSnop1Y9K6mxtm2ZmNhMzeefw7yPirojoTtu7gMMR0QUcTttIWg/0A3cCfcDTkpakOc8A\njwBd6dGX6tuAdyPiDuBJ4PHmWzIzs9mazWmlLcBAGg8A9xbq+yLiWkScBQaBTZJWAbdGxJGICOCF\nujnjxzoA9I6/qzAzs/nXaDgE8JeSXpe0PdUqETGSxheBShqvBi4U5g6l2uo0rq9PmBMRY8B7wG0z\n6MPMzFqoo8H9fj0ihiX9K+CQpLeLT0ZESIrWL2+iFEzbASqVCtVqtanj7Nw4dn1cWTZxe641u+aZ\nGB0dnZfXmW9l7KuMPUE5+ypjT1NpKBwiYjh9vSzp28Am4JKkVRExkk4ZXU67DwNrC9PXpNpwGtfX\ni3OGJHUAy4Erk6xjL7AXoLu7O3p6ehpZfubhXa9cH+/cOMYTJxrNyNk792DPnL9GtVql2e/NQlbG\nvsrYE5SzrzL2NJVpTytJukXSL4+Pgf8IvAkcBLam3bYCL6fxQaA/3YG0jtqF52PpFNRVSZvT9YSH\n6uaMH+s+4LV0XcLMzNqgkV+ZK8C30/XhDuB/RsR3JX0f2C9pG3AeeAAgIk5K2g+cAsaAHRHxYTrW\no8DzwDLg1fQAeA54UdIg8A61u53MzKxNpg2HiPgR8KlJ6leA3hvM2QPsmaR+HNgwSf0D4P4G1mtm\nZvPAn5A2M7OMw8HMzDIOBzMzyzgczMws43AwM7OMw8HMzDIOBzMzyzgczMws43AwM7OMw8HMzDIO\nBzMzyzgczMws43AwM7OMw8HMzDIOBzMzyzgczMws43AwM7OMw8HMzDIOBzMzyzgczMws43AwM7NM\nw+EgaYmkH0r6TtpeKemQpDPp64rCvrslDUo6LemeQv1uSSfSc09JUqovlfRSqh+V1Nm6Fs3MbKZm\n8s7hK8Bbhe1dwOGI6AIOp20krQf6gTuBPuBpSUvSnGeAR4Cu9OhL9W3AuxFxB/Ak8HhT3ZiZWUs0\nFA6S1gBfAJ4tlLcAA2k8ANxbqO+LiGsRcRYYBDZJWgXcGhFHIiKAF+rmjB/rANA7/q7CzMzmX0eD\n+/0J8AfALxdqlYgYSeOLQCWNVwNHCvsNpdrP07i+Pj7nAkBEjEl6D7gN+ElxEZK2A9sBKpUK1Wq1\nweVPtHPj2L80sWzi9lxrds0zMTo6Oi+vM9/K2FcZe4Jy9lXGnqYybThI+i3gckS8Lqlnsn0iIiRF\nqxc3yevsBfYCdHd3R0/PpMuZ1sO7Xrk+3rlxjCdONJqRs3fuwZ45f41qtUqz35uFrIx9lbEnKGdf\nZexpKo38VPwM8EVJnwc+Btwq6c+AS5JWRcRIOmV0Oe0/DKwtzF+TasNpXF8vzhmS1AEsB6402ZOZ\nmc3StNccImJ3RKyJiE5qF5pfi4gvAweBrWm3rcDLaXwQ6E93IK2jduH5WDoFdVXS5nQ94aG6OePH\nui+9xpy/EzEzs8nN5nzKY8B+SduA88ADABFxUtJ+4BQwBuyIiA/TnEeB54FlwKvpAfAc8KKkQeAd\naiFkZmZtMqNwiIgqUE3jK0DvDfbbA+yZpH4c2DBJ/QPg/pmsxczM5o4/IW1mZhmHg5mZZRwOZmaW\ncTiYmVnG4WBmZhmHg5mZZRwOZmaWcTiYmVnG4WBmZhmHg5mZZRwOZmaWcTiYmVnG4WBmZhmHg5mZ\nZRwOZmaWcTiYmVnG4WBmZhmHg5mZZRwOZmaWcTiYmVlm2nCQ9DFJxyT9raSTkv4o1VdKOiTpTPq6\nojBnt6RBSacl3VOo3y3pRHruKUlK9aWSXkr1o5I6W9+qmZk1qpF3DteA34yITwF3AX2SNgO7gMMR\n0QUcTttIWg/0A3cCfcDTkpakYz0DPAJ0pUdfqm8D3o2IO4Angcdb0JuZmTVp2nCImtG0+ZH0CGAL\nMJDqA8C9abwF2BcR1yLiLDAIbJK0Crg1Io5ERAAv1M0ZP9YBoHf8XYWZmc2/hq45SFoi6Q3gMnAo\nIo4ClYgYSbtcBCppvBq4UJg+lGqr07i+PmFORIwB7wG3zbgbMzNriY5GdoqID4G7JH0c+LakDXXP\nh6SYiwUWSdoObAeoVCpUq9WmjrNz49j1cWXZxO251uyaZ2J0dHReXme+lbGvMvYE5eyrjD1NpaFw\nGBcRP5X0PWrXCi5JWhURI+mU0eW02zCwtjBtTaoNp3F9vThnSFIHsBy4Msnr7wX2AnR3d0dPT89M\nln/dw7teuT7euXGMJ07M6NswK+ce7Jnz16hWqzT7vVnIythXGXuCcvZVxp6m0sjdSp9I7xiQtAz4\nLPA2cBDYmnbbCrycxgeB/nQH0jpqF56PpVNQVyVtTtcTHqqbM36s+4DX0nUJMzNrg0Z+ZV4FDKQ7\njn4B2B8R35H0N8B+SduA88ADABFxUtJ+4BQwBuxIp6UAHgWeB5YBr6YHwHPAi5IGgXeo3e1kZmZt\nMm04RMTfAZ+epH4F6L3BnD3Anknqx4ENk9Q/AO5vYL1mZjYP/AlpMzPLOBzMzCzjcDAzs4zDwczM\nMg4HMzPLOBzMzCzjcDAzs4zDwczMMg4HMzPLOBzMzCzjcDAzs4zDwczMMg4HMzPLOBzMzCzjcDAz\ns4zDwczMMg4HMzPLOBzMzCzjcDAzs4zDwczMMg4HMzPLTBsOktZK+p6kU5JOSvpKqq+UdEjSmfR1\nRWHObkmDkk5LuqdQv1vSifTcU5KU6kslvZTqRyV1tr5VMzNrVCPvHMaAnRGxHtgM7JC0HtgFHI6I\nLuBw2iY91w/cCfQBT0tako71DPAI0JUefam+DXg3Iu4AngQeb0FvZmbWpGnDISJGIuIHafzPwFvA\namALMJB2GwDuTeMtwL6IuBYRZ4FBYJOkVcCtEXEkIgJ4oW7O+LEOAL3j7yrMzGz+zeiaQzrd82ng\nKFCJiJH01EWgksargQuFaUOptjqN6+sT5kTEGPAecNtM1mZmZq3T0eiOkn4J+HPgqxFxtfiLfUSE\npJiD9dWvYTuwHaBSqVCtVps6zs6NY9fHlWUTt+das2ueidHR0Xl5nflWxr7K2BOUs68y9jSVhsJB\n0keoBcM3IuJbqXxJ0qqIGEmnjC6n+jCwtjB9TaoNp3F9vThnSFIHsBy4Ur+OiNgL7AXo7u6Onp6e\nRpafeXjXK9fHOzeO8cSJhjNy1s492DPnr1GtVmn2e7OQlbGvMvYE5eyrjD1NpZG7lQQ8B7wVEX9c\neOogsDWNtwIvF+r96Q6kddQuPB9Lp6CuStqcjvlQ3ZzxY90HvJauS5iZWRs08ivzZ4DfBk5IeiPV\n/ivwGLBf0jbgPPAAQESclLQfOEXtTqcdEfFhmvco8DywDHg1PaAWPi9KGgTeoXa3k5mZtcm04RAR\nfw3c6M6h3hvM2QPsmaR+HNgwSf0D4P7p1mJmZvPDn5A2M7OMw8HMzDIOBzMzyzgczMws43AwM7OM\nw8HMzDIOBzMzyzgczMws43AwM7OMw8HMzDIOBzMzyzgczMws43AwM7OMw8HMzDIOBzMzyzgczMws\n43AwM7OMw8HMzDIOBzMzyzgczMwsM204SPq6pMuS3izUVko6JOlM+rqi8NxuSYOSTku6p1C/W9KJ\n9NxTkpTqSyW9lOpHJXW2tkUzM5upRt45PA/01dV2AYcjogs4nLaRtB7oB+5Mc56WtCTNeQZ4BOhK\nj/FjbgPejYg7gCeBx5ttxszMWmPacIiIvwLeqStvAQbSeAC4t1DfFxHXIuIsMAhskrQKuDUijkRE\nAC/UzRk/1gGgd/xdhZmZtUez1xwqETGSxheBShqvBi4U9htKtdVpXF+fMCcixoD3gNuaXJeZmbVA\nx2wPEBEhKVqxmOlI2g5sB6hUKlSr1aaOs3Pj2PVxZdnE7bnW7JpnYnR0dF5eZ76Vsa8y9gTl7KuM\nPU2l2XC4JGlVRIykU0aXU30YWFvYb02qDadxfb04Z0hSB7AcuDLZi0bEXmAvQHd3d/T09DS1+Id3\nvXJ9vHPjGE+cmHVGNuzcgz1z/hrVapVmvzcLWRn7KmNPUM6+ytjTVJo9rXQQ2JrGW4GXC/X+dAfS\nOmoXno+lU1BXJW1O1xMeqpszfqz7gNfSdQkzM2uTaX9llvRNoAe4XdIQ8N+Ax4D9krYB54EHACLi\npKT9wClgDNgRER+mQz1K7c6nZcCr6QHwHPCipEFqF777W9KZmZk1bdpwiIgv3eCp3hvsvwfYM0n9\nOLBhkvoHwP3TrcPMzOaPPyFtZmYZh4OZmWUcDmZmlnE4mJlZxuFgZmYZh4OZmWUcDmZmlnE4mJlZ\nxuFgZmYZh4OZmWXm78+Rmt0kOgt/9Xc6OzeOTfgrwbN17rEvtOxYdnPzOwczM8s4HMzMLONwMDOz\njMPBzMwyDgczM8s4HMzMLONwMDOzjMPBzMwy/hCcldZMPoxmZhM5HG4S7fxB6U/tmi0+CyYcJPUB\nXwOWAM9GxGNtXpK1SKuCqdV/asLMbmxBhIOkJcD/AD4LDAHfl3QwIk61d2WtNx+/wfuH6M1roZxK\nK+O/wYXU03y8G18oF6Q3AYMR8aOI+BmwD9jS5jWZmd20Fko4rAYuFLaHUs3MzNpAEdHuNSDpPqAv\nIv5z2v5t4N9GxO/W7bcd2J42PwmcbsHL3w78pAXHWUjK2BOUs68y9gTl7KssPf2biPjEdDstiGsO\nwDCwtrC9JtUmiIi9wN5WvrCk4xHR3cpjtlsZe4Jy9lXGnqCcfZWxp6kslNNK3we6JK2T9FGgHzjY\n5jWZmd20FsQ7h4gYk/S7wP+mdivr1yPiZJuXZWZ201oQ4QAQEX8B/EUbXrqlp6kWiDL2BOXsq4w9\nQTn7KmNPN7QgLkibmdnCslCuOZiZ2QJy04aDpD5JpyUNStrV7vU0S9JaSd+TdErSSUlfSfWVkg5J\nOpO+rmj3WmdK0hJJP5T0nbRdhp4+LumApLclvSXp1xZ7X5L+S/q396akb0r62GLsSdLXJV2W9Gah\ndsM+JO1OPz9OS7qnPaueOzdlOBT+XMfngPXAlyStb++qmjYG7IyI9cBmYEfqZRdwOCK6gMNpe7H5\nCvBWYbsMPX0N+G5E/ArwKWr9Ldq+JK0Gfh/ojogN1G4o6Wdx9vQ80FdXm7SP9L+xfuDONOfp9HOl\nNG7KcKBEf64jIkYi4gdp/M/UftisptbPQNptALi3PStsjqQ1wBeAZwvlxd7TcuA3gOcAIuJnEfFT\nFnlf1G5sWSapA/hF4J9YhD1FxF8B79SVb9THFmBfRFyLiLPAILWfK6Vxs4ZDKf9ch6RO4NPAUaAS\nESPpqYtApU3LatafAH8A/L9CbbH3tA74MfCn6XTZs5JuYRH3FRHDwH8H/hEYAd6LiP/DIu6pzo36\nKOXPkKKbNRxKR9IvAX8OfDUirhafi9otaYvmtjRJvwVcjojXb7TPYusp6QB+FXgmIj4NvE/d6ZbF\n1lc6B7+FWvD9a+AWSV8u7rPYerqRsvTRqJs1HBr6cx2LhaSPUAuGb0TEt1L5kqRV6flVwOV2ra8J\nnwG+KOkctVN+vynpz1jcPUHtt8uhiDiatg9QC4vF3Nd/AM5GxI8j4ufAt4B/x+LuqehGfZTqZ8hk\nbtZwKM2f65Akauew34qIPy48dRDYmsZbgZfne23NiojdEbEmIjqp/bd5LSK+zCLuCSAiLgIXJH0y\nlXqBUyzuvv4R2CzpF9O/xV5q170Wc09FN+rjINAvaamkdUAXcKwN65s7EXFTPoDPA38P/APwh+1e\nzyz6+HVqb3X/DngjPT4P3Ebt7oozwF8CK9u91ib76wG+k8aLvifgLuB4+u/1v4AVi70v4I+At4E3\ngReBpYuxJ+Cb1K6b/Jzau7xtU/UB/GH6+XEa+Fy719/qhz8hbWZmmZv1tJKZmU3B4WBmZhmHg5mZ\nZRwOZmaWcTiYmVnG4WBmZhmHg5mZZRwOZmaW+f+3xym+ibRt3QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2049daec748>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "db_clusters_df['db_2_clusters'].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x2048c232240>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFnlJREFUeJzt3W+QneV53/HvL8jBCg5/7e6ogqmYQeMMoNoOGkLqNLOO\nkqLYHosXwCiDg2hV9AKSOh3NuKJ+0ckLzcC0hNjTQkdjHAR2DBpiF40JbojwTiYzRVgkrmXAFMVA\nkCqQDRgit2CLXn1x7sWHfVbs0Wq15xzz/cyc2fu5nj973atd/fY8z3POpqqQJKnfzw27AUnS6DEc\nJEkdhoMkqcNwkCR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSepYMuwG5uu9731vrVix4riP86Mf/YhT\nTjnl+BsaEvsfvnGfg/0P32LO4dFHH/1BVb1vru3GNhxWrFjBnj17jvs4U1NTTE5OHn9DQ2L/wzfu\nc7D/4VvMOSR5dpDtPK0kSeowHCRJHYaDJKnDcJAkdRgOkqSOgcIhyelJ7k3y3SRPJPnVJGcmeTDJ\nU+3jGX3b35BkX5Ink1zaV78oyd627nNJ0uonJ7mn1XcnWbHQE5UkDW7QZw6fBb5eVb8EfAB4AtgC\n7KqqlcCutkyS84H1wAXAWuDWJCe149wGXAusbI+1rb4ReLmqzgNuAW46znlJko7DnOGQ5DTg14Hb\nAarqx1X1Q2AdsL1tth24rI3XAXdX1etV9TSwD7g4yTLg1Kp6uHp/m/TOGftMH+teYM30swpJ0uIb\n5EVw5wLfB/4kyQeAR4FPARNVdbBt8zww0cbLgYf79t/faj9p45n16X2eA6iqI0leAc4CftDfSJJN\nwCaAiYkJpqamBmj/7R0+fHhBjjMs9j984z4H+x++UZzDIOGwBPhl4PeraneSz9JOIU2rqkpSJ6LB\nGZ9nG7ANYPXq1TXfVxSu2HL/m+PNq97g5r/+0UK0N5BnbvzYgh5v3F8dOu79w/jPwf6HbxTnMMg1\nh/3A/qra3ZbvpRcWL7RTRbSPh9r6A8A5ffuf3WoH2nhm/S37JFkCnAa8eKyTkSQtjDnDoaqeB55L\n8v5WWgM8DuwENrTaBuC+Nt4JrG93IJ1L78LzI+0U1KtJLmnXE66esc/0sS4HHmrXJSRJQzDoG+/9\nPvClJD8PfA/4l/SCZUeSjcCzwJUAVfVYkh30AuQIcH1VvdGOcx1wB7AUeKA9oHex+64k+4CX6N3t\nJEkakoHCoaq+BayeZdWao2y/Fdg6S30PcOEs9deAKwbpRZJ04vkKaUlSh+EgSeowHCRJHYaDJKnD\ncJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwH\nSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLUYThIkjoMB0lSx0DhkOSZJHuTfCvJnlY7M8mD\nSZ5qH8/o2/6GJPuSPJnk0r76Re04+5J8Lkla/eQk97T67iQrFnaakqRjcSzPHD5SVR+sqtVteQuw\nq6pWArvaMknOB9YDFwBrgVuTnNT2uQ24FljZHmtbfSPwclWdB9wC3DT/KUmSjtfxnFZaB2xv4+3A\nZX31u6vq9ap6GtgHXJxkGXBqVT1cVQXcOWOf6WPdC6yZflYhSVp8g4ZDAX+Z5NEkm1ptoqoOtvHz\nwEQbLwee69t3f6stb+OZ9bfsU1VHgFeAs45hHpKkBbRkwO1+raoOJPlHwINJvtu/sqoqSS18e2/V\ngmkTwMTEBFNTU/M6zuZVR94cTyx96/KJNt+ej+bw4cMLfszFNO79w/jPwf6HbxTnMFA4VNWB9vFQ\nkq8CFwMvJFlWVQfbKaNDbfMDwDl9u5/dagfaeGa9f5/9SZYApwEvztLHNmAbwOrVq2tycnKQ9juu\n2XL/m+PNq45w895BM/L4PXPV5IIeb2pqivl+HUbBuPcP4z8H+x++UZzDnKeVkpyS5Benx8C/AL4D\n7AQ2tM02APe18U5gfbsD6Vx6F54faaegXk1ySbuecPWMfaaPdTnwULsuIUkagkF+ZZ4AvtquDy8B\n/rSqvp7km8COJBuBZ4ErAarqsSQ7gMeBI8D1VfVGO9Z1wB3AUuCB9gC4HbgryT7gJXp3O0mShmTO\ncKiq7wEfmKX+IrDmKPtsBbbOUt8DXDhL/TXgigH6lSQtAl8hLUnqMBwkSR2GgySpw3CQJHUYDpKk\nDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSeow\nHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR0Dh0OSk5L8bZKvteUzkzyY\n5Kn28Yy+bW9Isi/Jk0ku7atflGRvW/e5JGn1k5Pc0+q7k6xYuClKko7VsTxz+BTwRN/yFmBXVa0E\ndrVlkpwPrAcuANYCtyY5qe1zG3AtsLI91rb6RuDlqjoPuAW4aV6zkSQtiIHCIcnZwMeAz/eV1wHb\n23g7cFlf/e6qer2qngb2ARcnWQacWlUPV1UBd87YZ/pY9wJrpp9VSJIW35IBt/tj4NPAL/bVJqrq\nYBs/D0y08XLg4b7t9rfaT9p4Zn16n+cAqupIkleAs4Af9DeRZBOwCWBiYoKpqakB23+rzauO/HQS\nS9+6fKLNt+ejOXz48IIfczGNe/8w/nOw/+EbxTnMGQ5JPg4cqqpHk0zOtk1VVZJa6OZm+TzbgG0A\nq1evrsnJWduZ0zVb7n9zvHnVEW7eO2hGHr9nrppc0ONNTU0x36/DKBj3/mH852D/wzeKcxjkf8UP\nA59I8lHg3cCpSb4IvJBkWVUdbKeMDrXtDwDn9O1/dqsdaOOZ9f599idZApwGvDjPOUmSjtOc1xyq\n6oaqOruqVtC70PxQVX0S2AlsaJttAO5r453A+nYH0rn0Ljw/0k5BvZrkknY94eoZ+0wf6/L2OU74\nMxFJ0uyO53zKjcCOJBuBZ4ErAarqsSQ7gMeBI8D1VfVG2+c64A5gKfBAewDcDtyVZB/wEr0QkiQN\nyTGFQ1VNAVNt/CKw5ijbbQW2zlLfA1w4S/014Ipj6UWSdOL4CmlJUofhIEnqMBwkSR2GgySpw3CQ\nJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLUYThIkjoMB0lS\nh+EgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUsec4ZDk3UkeSfI/kzyW5A9b/cwk\nDyZ5qn08o2+fG5LsS/Jkkkv76hcl2dvWfS5JWv3kJPe0+u4kKxZ+qpKkQQ3yzOF14Deq6gPAB4G1\nSS4BtgC7qmolsKstk+R8YD1wAbAWuDXJSe1YtwHXAivbY22rbwRerqrzgFuAmxZgbpKkeZozHKrn\ncFt8V3sUsA7Y3urbgcvaeB1wd1W9XlVPA/uAi5MsA06tqoerqoA7Z+wzfax7gTXTzyokSYtvoGsO\nSU5K8i3gEPBgVe0GJqrqYNvkeWCijZcDz/Xtvr/VlrfxzPpb9qmqI8ArwFnHPBtJ0oJYMshGVfUG\n8MEkpwNfTXLhjPWVpE5Eg/2SbAI2AUxMTDA1NTWv42xedeTN8cTSty6faPPt+WgOHz684MdcTOPe\nP4z/HOx/+EZxDgOFw7Sq+mGSb9C7VvBCkmVVdbCdMjrUNjsAnNO329mtdqCNZ9b799mfZAlwGvDi\nLJ9/G7ANYPXq1TU5OXks7b/pmi33vznevOoIN+89pi/DcXnmqskFPd7U1BTz/TqMgnHvH8Z/DvY/\nfKM4h0HuVnpfe8ZAkqXAbwHfBXYCG9pmG4D72ngnsL7dgXQuvQvPj7RTUK8muaRdT7h6xj7Tx7oc\neKhdl5AkDcEgvzIvA7a3O45+DthRVV9L8j+AHUk2As8CVwJU1WNJdgCPA0eA69tpKYDrgDuApcAD\n7QFwO3BXkn3AS/TudpIkDcmc4VBV3wY+NEv9RWDNUfbZCmydpb4HuHCW+mvAFQP0K0laBL5CWpLU\nYThIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1GE4SJI6DAdJUofhIEnqMBwkSR2G\ngySpw3CQJHUYDpKkDsNBktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLUYThI\nkjrmDIck5yT5RpLHkzyW5FOtfmaSB5M81T6e0bfPDUn2JXkyyaV99YuS7G3rPpckrX5ykntafXeS\nFQs/VUnSoAZ55nAE2FxV5wOXANcnOR/YAuyqqpXArrZMW7ceuABYC9ya5KR2rNuAa4GV7bG21TcC\nL1fVecAtwE0LMDdJ0jzNGQ5VdbCq/qaN/wF4AlgOrAO2t822A5e18Trg7qp6vaqeBvYBFydZBpxa\nVQ9XVQF3zthn+lj3Amumn1VIkhbfMV1zaKd7PgTsBiaq6mBb9Tww0cbLgef6dtvfasvbeGb9LftU\n1RHgFeCsY+lNkrRwlgy6YZL3AH8G/EFVvdr/i31VVZI6Af3N7GETsAlgYmKCqampeR1n86ojb44n\nlr51+USbb89Hc/jw4QU/5mIa9/5h/Odg/8M3inMYKBySvIteMHypqr7Syi8kWVZVB9spo0OtfgA4\np2/3s1vtQBvPrPfvsz/JEuA04MWZfVTVNmAbwOrVq2tycnKQ9juu2XL/m+PNq45w896BM/K4PXPV\n5IIeb2pqivl+HUbBuPcP4z8H+x++UZzDIHcrBbgdeKKq/qhv1U5gQxtvAO7rq69vdyCdS+/C8yPt\nFNSrSS5px7x6xj7Tx7oceKhdl5AkDcEgvzJ/GPhdYG+Sb7XavwduBHYk2Qg8C1wJUFWPJdkBPE7v\nTqfrq+qNtt91wB3AUuCB9oBe+NyVZB/wEr27nSRJQzJnOFTVXwNHu3NozVH22QpsnaW+B7hwlvpr\nwBVz9SJJWhy+QlqS1GE4SJI6DAdJUofhIEnqMBwkSR2GgySpw3CQJHUYDpKkDsNBktRhOEiSOgwH\nSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAk\ndRgOkqSOJcNuQPpZs2LL/UP73M/c+LGhfW79bPGZgySpY85wSPKFJIeSfKevdmaSB5M81T6e0bfu\nhiT7kjyZ5NK++kVJ9rZ1n0uSVj85yT2tvjvJioWdoiTpWA3yzOEOYO2M2hZgV1WtBHa1ZZKcD6wH\nLmj73JrkpLbPbcC1wMr2mD7mRuDlqjoPuAW4ab6TkSQtjDnDoar+CnhpRnkdsL2NtwOX9dXvrqrX\nq+ppYB9wcZJlwKlV9XBVFXDnjH2mj3UvsGb6WYUkaTjS+796jo16p3q+VlUXtuUfVtXpbRx6v/mf\nnuQ/Aw9X1RfbutuBB4BngBur6jdb/Z8D/66qPt5OV62tqv1t3d8Bv1JVP5ilj03AJoCJiYmL7r77\n7nlNeu+BV94cTyyFF/7vvA4zL6uWn7agxzt8+DDvec97FvSYi2nc+4fuHPq/vxbbfL6/xv3fYNz7\nh8Wdw0c+8pFHq2r1XNsd991KVVVJ5k6YBVBV24BtAKtXr67Jycl5HeeavrtJNq86ws17F++mrWeu\nmlzQ401NTTHfr8MoGPf+oTuHa4Z5t9I8vr/G/d9g3PuH0ZzDfO9WeqGdKqJ9PNTqB4Bz+rY7u9UO\ntPHM+lv2SbIEOA14cZ59SZIWwHzDYSewoY03APf11de3O5DOpXfh+ZGqOgi8muSSdhrq6hn7TB/r\ncuChGuRclyTphJnzfEqSLwOTwHuT7Af+A3AjsCPJRuBZ4EqAqnosyQ7gceAIcH1VvdEOdR29O5+W\n0rsO8UCr3w7clWQfvQvf6xdkZpKkeZszHKrqd46yas1Rtt8KbJ2lvge4cJb6a8AVc/UhSVo8vkJa\nktRhOEiSOgwHSVKH4SBJ6jAcJEkdhoMkqcNwkCR1GA6SpA7/TKhOuLf7s5mbVx05YW9U55/MlObP\nZw6SpA7DQZLUYThIkjoMB0lSh+EgSeowHCRJHYaDJKnDcJAkdRgOkqQOw0GS1OHbZ+hn1tu9bcdC\nOpFvASINi88cJEkdhoMkqcNwkCR1GA6SpA7DQZLUMTJ3KyVZC3wWOAn4fFXdOOSWToiFvoNm0Dtl\n/MM3ko7FSIRDkpOA/wL8FrAf+GaSnVX1+HA7k6TZLeQvesd6O/Ri/LI3EuEAXAzsq6rvASS5G1gH\nGA4LZLHu+ddwzeffedxfpzHu/Y+qUbnmsBx4rm95f6tJkoYgVTXsHkhyObC2qv51W/5d4Feq6vdm\nbLcJ2NQW3w88uQCf/r3ADxbgOMNi/8M37nOw/+FbzDn8k6p631wbjcpppQPAOX3LZ7faW1TVNmDb\nQn7iJHuqavVCHnMx2f/wjfsc7H/4RnEOo3Ja6ZvAyiTnJvl5YD2wc8g9SdI71kg8c6iqI0l+D/jv\n9G5l/UJVPTbktiTpHWskwgGgqv4c+PMhfOoFPU01BPY/fOM+B/sfvpGbw0hckJYkjZZRueYgSRoh\n79hwSLI2yZNJ9iXZMux+5pLknCTfSPJ4kseSfKrVz0zyYJKn2sczht3r20lyUpK/TfK1tjxu/Z+e\n5N4k303yRJJfHac5JPm37fvnO0m+nOTdo95/ki8kOZTkO321o/ac5Ib2c/1kkkuH0/VPHaX//9i+\nh76d5KtJTu9bNxL9vyPDoe/tOn4bOB/4nSTnD7erOR0BNlfV+cAlwPWt5y3ArqpaCexqy6PsU8AT\nfcvj1v9nga9X1S8BH6A3l7GYQ5LlwL8BVlfVhfRu/ljP6Pd/B7B2Rm3WntvPxHrggrbPre3nfZju\noNv/g8CFVfVPgf8F3ACj1f87Mhzoe7uOqvoxMP12HSOrqg5W1d+08T/Q+09pOb2+t7fNtgOXDafD\nuSU5G/gY8Pm+8jj1fxrw68DtAFX146r6IWM0B3o3oSxNsgT4BeB/M+L9V9VfAS/NKB+t53XA3VX1\nelU9Deyj9/M+NLP1X1V/UVVH2uLD9F7bBSPU/zs1HMb67TqSrAA+BOwGJqrqYFv1PDAxpLYG8cfA\np4H/11cbp/7PBb4P/Ek7Nfb5JKcwJnOoqgPAfwL+HjgIvFJVf8GY9D/D0Xoex5/tfwU80MYj0/87\nNRzGVpL3AH8G/EFVvdq/rnq3no3k7WdJPg4cqqpHj7bNKPffLAF+Gbitqj4E/IgZp2BGeQ7tvPw6\neiH3j4FTknyyf5tR7v9oxrHnaUk+Q++U8ZeG3ctM79RwGOjtOkZNknfRC4YvVdVXWvmFJMva+mXA\noWH1N4cPA59I8gy903i/keSLjE//0Pstbn9V7W7L99ILi3GZw28CT1fV96vqJ8BXgH/G+PTf72g9\nj83PdpJrgI8DV9VPX1MwMv2/U8Nh7N6uI0nonet+oqr+qG/VTmBDG28A7lvs3gZRVTdU1dlVtYLe\n1/uhqvokY9I/QFU9DzyX5P2ttIbe28qPyxz+HrgkyS+076c19K5djUv//Y7W805gfZKTk5wLrAQe\nGUJ/byu9P272aeATVfV/+laNTv9V9Y58AB+ld5fA3wGfGXY/A/T7a/SeOn8b+FZ7fBQ4i97dGk8B\nfwmcOexeB5jLJPC1Nh6r/oEPAnvav8N/A84YpzkAfwh8F/gOcBdw8qj3D3yZ3jWSn9B79rbx7XoG\nPtN+rp8EfntE+99H79rC9M/yfx21/n2FtCSp4516WkmS9DYMB0lSh+EgSeowHCRJHYaDJKnDcJAk\ndRgOkqQOw0GS1PH/AWIRsXBbeABUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x204bd6ca128>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "db_clusters_df['db_3_clusters'].hist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combine DBSCAN clusters with article data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df2 = pd.concat([snowball_df, db_clusters_df], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime</th>\n",
       "      <th>source</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>text_snowball</th>\n",
       "      <th>db_1_clusters</th>\n",
       "      <th>db_2_clusters</th>\n",
       "      <th>db_3_clusters</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15038</th>\n",
       "      <td>2012-06-08 17:13:47</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>Spain's savings banks' culture of greed, crony...</td>\n",
       "      <td>https://www.theguardian.com/world/2012/jun/08/...</td>\n",
       "      <td>as european taxpay prepar rescu spain ail bank...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15439</th>\n",
       "      <td>2011-08-03 08:28:23</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>Switzerland moves to contain franc appreciation</td>\n",
       "      <td>https://www.theguardian.com/business/2011/aug/...</td>\n",
       "      <td>swiss author move counteract nation bank call ...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22474</th>\n",
       "      <td>2016-03-16 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>Britain's FTSE boosted by oil stocks and house...</td>\n",
       "      <td>http://uk.reuters.com/article/britain-stocks-i...</td>\n",
       "      <td>britain oil gas stock housebuild ralli back pr...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81348</th>\n",
       "      <td>2017-06-20 01:49:00</td>\n",
       "      <td>FT</td>\n",
       "      <td>Noble gets 120-day deadline extension from len...</td>\n",
       "      <td>https://www.ft.com/content/806ad7d4-5fc1-3d0e-...</td>\n",
       "      <td>nobl group troubl singapor list commod trader ...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68373</th>\n",
       "      <td>2014-10-30 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>US STOCKS-Visa gives big boost to Dow, but GDP...</td>\n",
       "      <td>http://uk.reuters.com/article/markets-stocks-u...</td>\n",
       "      <td>us stock most edg strong read third quarter ec...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5585</th>\n",
       "      <td>2013-02-20 23:14:48</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>Europe: the Right Italy</td>\n",
       "      <td>https://www.theguardian.com/commentisfree/2013...</td>\n",
       "      <td>the european econom crisi auster polici came w...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16519</th>\n",
       "      <td>2008-02-18 10:00:00</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>The price isn't right</td>\n",
       "      <td>https://www.theguardian.com/commentisfree/2008...</td>\n",
       "      <td>terror global warm loom mani mind greatest thr...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33155</th>\n",
       "      <td>2017-01-26 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>European shares hit 1-year high on M&amp;A deal, A...</td>\n",
       "      <td>http://uk.reuters.com/article/europe-stocks-id...</td>\n",
       "      <td>european share climb one johnson johnson buy s...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50905</th>\n",
       "      <td>2009-05-08</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>Nikkei hits 6-mth high in seesaw trade, Toyota</td>\n",
       "      <td>http://uk.reuters.com/article/markets-japan-st...</td>\n",
       "      <td>japan nikkei edg seesaw trade briefli touch si...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9743</th>\n",
       "      <td>2016-07-14 18:25:27</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>Industrial strategy welcomed as part of govern...</td>\n",
       "      <td>https://www.theguardian.com/politics/2016/jul/...</td>\n",
       "      <td>busi leader overwhelm welcom theresa may decis...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  datetime    source  \\\n",
       "15038  2012-06-08 17:13:47  Guardian   \n",
       "15439  2011-08-03 08:28:23  Guardian   \n",
       "22474  2016-03-16 00:00:00   Reuters   \n",
       "81348  2017-06-20 01:49:00        FT   \n",
       "68373  2014-10-30 00:00:00   Reuters   \n",
       "5585   2013-02-20 23:14:48  Guardian   \n",
       "16519  2008-02-18 10:00:00  Guardian   \n",
       "33155  2017-01-26 00:00:00   Reuters   \n",
       "50905           2009-05-08   Reuters   \n",
       "9743   2016-07-14 18:25:27  Guardian   \n",
       "\n",
       "                                                   title  \\\n",
       "15038  Spain's savings banks' culture of greed, crony...   \n",
       "15439    Switzerland moves to contain franc appreciation   \n",
       "22474  Britain's FTSE boosted by oil stocks and house...   \n",
       "81348  Noble gets 120-day deadline extension from len...   \n",
       "68373  US STOCKS-Visa gives big boost to Dow, but GDP...   \n",
       "5585                             Europe: the Right Italy   \n",
       "16519                              The price isn't right   \n",
       "33155  European shares hit 1-year high on M&A deal, A...   \n",
       "50905     Nikkei hits 6-mth high in seesaw trade, Toyota   \n",
       "9743   Industrial strategy welcomed as part of govern...   \n",
       "\n",
       "                                                     url  \\\n",
       "15038  https://www.theguardian.com/world/2012/jun/08/...   \n",
       "15439  https://www.theguardian.com/business/2011/aug/...   \n",
       "22474  http://uk.reuters.com/article/britain-stocks-i...   \n",
       "81348  https://www.ft.com/content/806ad7d4-5fc1-3d0e-...   \n",
       "68373  http://uk.reuters.com/article/markets-stocks-u...   \n",
       "5585   https://www.theguardian.com/commentisfree/2013...   \n",
       "16519  https://www.theguardian.com/commentisfree/2008...   \n",
       "33155  http://uk.reuters.com/article/europe-stocks-id...   \n",
       "50905  http://uk.reuters.com/article/markets-japan-st...   \n",
       "9743   https://www.theguardian.com/politics/2016/jul/...   \n",
       "\n",
       "                                           text_snowball  db_1_clusters  \\\n",
       "15038  as european taxpay prepar rescu spain ail bank...             -1   \n",
       "15439  swiss author move counteract nation bank call ...             -1   \n",
       "22474  britain oil gas stock housebuild ralli back pr...             -1   \n",
       "81348  nobl group troubl singapor list commod trader ...             -1   \n",
       "68373  us stock most edg strong read third quarter ec...             -1   \n",
       "5585   the european econom crisi auster polici came w...             -1   \n",
       "16519  terror global warm loom mani mind greatest thr...             -1   \n",
       "33155  european share climb one johnson johnson buy s...             -1   \n",
       "50905  japan nikkei edg seesaw trade briefli touch si...             -1   \n",
       "9743   busi leader overwhelm welcom theresa may decis...             -1   \n",
       "\n",
       "       db_2_clusters  db_3_clusters  \n",
       "15038             -1             -1  \n",
       "15439             -1             40  \n",
       "22474             -1             -1  \n",
       "81348             -1             -1  \n",
       "68373             -1             72  \n",
       "5585              -1             30  \n",
       "16519             -1             -1  \n",
       "33155             -1             -1  \n",
       "50905             -1             46  \n",
       "9743              -1             -1  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There's a problem with DBSCAN clusters - the majority are -1 which means they have been labelled as noise. However, it should be noted that - having looked at the clusters - the clusters it has created are very specific (Brexit, Oil prices, Bitcoin etc).\n",
    "\n",
    "DBSCAN takes a long time to compute, so I think I will move on to K-Means."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df2.to_csv('db_clusters.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K-Means at different values of K"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Try a number of K values and check the inertia scores:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "k_values = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, \n",
    "            160, 170, 180, 190, 200, 210, 220, 230, 240, 250, 260, 270, 280, 290, 300]\n",
    "k_inertia = []\n",
    "k_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "k=10, inertia=64421.068638861805\n",
      "k=20, inertia=59695.813133809774\n",
      "k=30, inertia=56882.7491227177\n",
      "k=40, inertia=54770.116928211355\n",
      "k=50, inertia=53143.25191014049\n",
      "k=60, inertia=51515.55860693105\n",
      "k=70, inertia=50349.19539560028\n",
      "k=80, inertia=49168.36768293017\n",
      "k=90, inertia=48340.6163620607\n",
      "k=100, inertia=47542.51199933807\n",
      "k=110, inertia=46636.65028303084\n",
      "k=120, inertia=45972.27045626982\n",
      "k=130, inertia=45415.32951495396\n",
      "k=140, inertia=44834.8483949335\n",
      "k=150, inertia=44279.16152909724\n",
      "k=160, inertia=43679.053118056974\n",
      "k=170, inertia=43175.29117539236\n",
      "k=180, inertia=42744.69665442959\n",
      "k=190, inertia=42378.83807326705\n",
      "k=200, inertia=41930.99289550696\n",
      "k=210, inertia=41536.91324188674\n",
      "k=220, inertia=41139.14698466502\n",
      "k=230, inertia=40834.748349083595\n",
      "k=240, inertia=40575.604307195325\n",
      "k=250, inertia=40187.26234167285\n",
      "k=260, inertia=39957.616646119415\n",
      "k=270, inertia=39695.43858884415\n",
      "k=280, inertia=39386.12816800209\n",
      "k=290, inertia=39179.12043618162\n",
      "k=300, inertia=38920.714925562694\n"
     ]
    }
   ],
   "source": [
    "for k in k_values:\n",
    "    km = KMeans(n_clusters=k, init='k-means++', n_jobs=-1)\n",
    "    km_clusters = km.fit_predict(lsa_vecs)\n",
    "    k_dict[k] = [km_clusters]\n",
    "    k_inertia.append(km.inertia_)\n",
    "    print('k={}, inertia={}'.format(k, km.inertia_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot inertia at different values of k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x204256a0f28>]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VeW97/HPLwMhAwkQkgAJUyCEAgpICgqoIFXU0+PQ\nOtD2VNrSWhWH1va0envvOe29tz3HtpbW05Zbx+KsdSi2TkcRtKiACTJDIMwJIQkQCCBTkt/9Y69g\nSAQCJNnZyff9eu1XVp69VvJ7Xkv5Zq3nWc82d0dERKS+qHAXICIibY/CQUREGlE4iIhIIwoHERFp\nROEgIiKNKBxERKQRhYOIiDSicBARkUYUDiIi0khMuAs4Uz169PD+/fuHuwwRkYhSUFCw093TTrVf\nxIZD//79yc/PD3cZIiIRxcy2NGW/Jt1WMrOuZvaCma01szVmdoGZ/dTMSsxsafC6st7+95pZkZkV\nmtmUeu2jzWxF8N4DZmZBe5yZPRe0LzKz/qfXXRERaU5NHXP4HfCGuw8BRgBrgvaZ7j4yeL0GYGZD\nganAMOBy4I9mFh3sPwv4DpATvC4P2qcDle4+CJgJ3Hd23RIRkbNxynAwsxTgIuARAHc/4u57TnLI\n1cCz7n7Y3TcBRcAYM+sFJLv7Qg8tBfs4cE29Y2YH2y8Ak+uuKkREpPU15cphAFABPGZmH5vZw2aW\nGLx3h5ktN7NHzaxb0JYJbKt3fHHQlhlsN2w/7hh3rwb2Aqln0iERETl7TQmHGOA8YJa7jwIOAPcQ\nukWUDYwESoH7W6rIOmZ2s5nlm1l+RUVFS/86EZEOqynhUAwUu/ui4PsXgPPcvczda9y9FngIGBO8\nXwL0qXd8VtBWEmw3bD/uGDOLAVKAXQ0LcfcH3T3P3fPS0k45E0tERM7QKcPB3XcA28wsN2iaDKwO\nxhDqXAusDLZfAaYGM5AGEBp4XuzupUCVmZ0fjCfcBMypd8y0YPs64B3XR9SJiIRNU59zuAN4ysw6\nARuBbwIPmNlIwIHNwHcB3H2VmT0PrAaqgRnuXhP8nNuAPwPxwOvBC0KD3U+YWRGwm9BspxaxZGsl\nb60u40dTctGYt4jIZ2tSOLj7UiCvQfPXT7L/z4Gff0Z7PjD8M9oPAdc3pZaztapkL7Pmb+D60Vlk\npyW1xq8UEYk4HW5tpYm56QDML9SAtojIiXS4cOjTPYHstETmr1M4iIicSIcLB4BJueks3LiLg0dq\nTr2ziEgH1CHDYWJuGkeqa/lw485wlyIi0iZ1yHAYM6A78bHRGncQETmBDhkOcTHRjB+UyrzCcvQ4\nhYhIYx0yHAAuzk1n2+6DbNx5INyliIi0OR02HCYODi2/oVtLIiKNddhw6NM9gUHpScwvLA93KSIi\nbU6HDQcIXT0s2ribT45Uh7sUEZE2pWOHQ246R2pq+XBDowVgRUQ6tA4dDp8f0I2ETtHM060lEZHj\ndOhwiIuJZtzAHswvrNCUVhGRejp0OEDoaeniyoNsqNCUVhGROgqH3Loprbq1JCJSp8OHQ1a3BHLS\nk/S8g4hIPR0+HCB09bB4024OHNaUVhERUDgAoSW8j9TU8oGmtIqIAAoHAPL6dyexU7TGHUREAgoH\noFNMFOMGaUqriEgdhUNgUm46JXsOUlS+P9yliIiEncIh8OmUVs1aEhFROAR6d41ncEaSltIQEUHh\ncJxJuel8tHk3+zWlVUQ6OIVDPRfnpnG0xvmgaGe4SxERCSuFQz15/UJTWudp3EFEOjiFQz2dYqKY\nkNODdwvLNaVVRDo0hUMDE3PT2b73EOs1pVVEOjCFQwN1U1rnrdWsJRHpuBQODfRKiWdIzy563kFE\nOrQmhYOZdTWzF8xsrZmtMbMLzKy7mb1lZuuDr93q7X+vmRWZWaGZTanXPtrMVgTvPWBmFrTHmdlz\nQfsiM+vf3B09HRfnppG/ZTf7Dh0NZxkiImHT1CuH3wFvuPsQYASwBrgHmOvuOcDc4HvMbCgwFRgG\nXA780cyig58zC/gOkBO8Lg/apwOV7j4ImAncd5b9OisTB6dztMZ5v0irtIpIx3TKcDCzFOAi4BEA\ndz/i7nuAq4HZwW6zgWuC7auBZ939sLtvAoqAMWbWC0h294Uemgr0eINj6n7WC8DkuquKcMjr342k\nuBjeXadxBxHpmJpy5TAAqAAeM7OPzexhM0sEMty9NNhnB5ARbGcC2+odXxy0ZQbbDduPO8bdq4G9\nQOrpd6d5xEZHMUGrtIpIB9aUcIgBzgNmufso4ADBLaQ6wZVAi/8ramY3m1m+meVXVLTsgPHE3DRK\n9x6isGxfi/4eEZG2qCnhUAwUu/ui4PsXCIVFWXCriOBr3T2YEqBPveOzgraSYLth+3HHmFkMkAI0\nuuHv7g+6e56756WlpTWh9DM3MTcdgLlrdGtJRDqeU4aDu+8AtplZbtA0GVgNvAJMC9qmAXOC7VeA\nqcEMpAGEBp4XB7egqszs/GA84aYGx9T9rOuAdzzM93N6pnTmguxUHv9wM4era8JZiohIq2vqbKU7\ngKfMbDkwEvgF8J/ApWa2HvhC8D3uvgp4nlCAvAHMcPe6f11vAx4mNEi9AXg9aH8ESDWzIuBuGty2\nCpcZkwZRVnWYFwqKT72ziEg7YpE64JqXl+f5+fkt+jvcnWv/+AE79x9m3g8nEhutZwZFJLKZWYG7\n551qP/1rdxJmxu2TBlFceZBXlm4PdzkiIq1G4XAKkz+XzpCeXfjj/CJqayPzKktE5HQpHE7BzJgx\naRAbKg7wxqod4S5HRKRVKBya4MpzepHdI5H/eqdID8WJSIegcGiC6Cjj1okDWVNaxbxCPfcgIu2f\nwqGJrhmVSWbXeH6vqwcR6QAUDk0UGx3FLRdns2TrHj7cqNVaRaR9Uzichuvz+pDWJY4/zCsKdyki\nIi1K4XAaOsdGc/OF2bxftIslWyvDXY6ISItROJymr47tS9eEWP7wjq4eRKT9UjicpsS4GL41fgBz\n15azentVuMsREWkRCoczMO2C/iTFxfCH+bp6EJH2SeFwBlISYrnpgn68tqKUovL94S5HRKTZKRzO\n0PQJA4iLiWLW/A3hLkVEpNkpHM5QalIcXxnTl78uLWHb7k/CXY6ISLNSOJyFmy/KJsrgT+/p6kFE\n2heFw1nolRLPdaP78PxHxZRVHQp3OSIizUbhcJZuvXggNe489N7GcJciItJsFA5nqW9qAleN6M1T\ni7ayfc/BcJcjItIsFA7N4O5LBwPwv/66Uiu2iki7oHBoBn26J/CDywYzd205f19eGu5yRETOmsKh\nmXxz/ABGZKXw01dWUXngSLjLERE5KwqHZhIdZfzHl85l78Gj/Py1NeEuR0TkrCgcmtHQ3sl89+Js\nXigoZsH6neEuR0TkjCkcmtkdl+SQ3SORe19ezidHqsNdjojIGVE4NLPOsdH84kvnsG33QWa+tS7c\n5YiInBGFQws4PzuVr4zpyyMLNrG8eE+4yxEROW0KhxZyzxVD6JEUx49fXMHRmtpwlyMicloUDi0k\nJT6W/3PNcNaUVvHQP7S0hohEFoVDC5oyrCdXDO/Jb99ez6adB8JdjohIkzUpHMxss5mtMLOlZpYf\ntP3UzEqCtqVmdmW9/e81syIzKzSzKfXaRwc/p8jMHjAzC9rjzOy5oH2RmfVv3m6Gz8+uGkZcTBT3\nvLic2lotrSEikeF0rhwmuftId8+r1zYzaBvp7q8BmNlQYCowDLgc+KOZRQf7zwK+A+QEr8uD9ulA\npbsPAmYC951xj9qY9OTO/OTKz7Fo026ez98W7nJERJqkJW4rXQ086+6H3X0TUASMMbNeQLK7L/TQ\n6nSPA9fUO2Z2sP0CMLnuqqI9uPHzfTg/uzs/f20N5frcBxGJAE0NBwfeNrMCM7u5XvsdZrbczB41\ns25BWyZQ/0/k4qAtM9hu2H7cMe5eDewFUhsWYWY3m1m+meVXVFQ0sfTwMwstrXG4upZ/f2VVuMsR\nETmlpobDBHcfCVwBzDCziwjdIsoGRgKlwP0tU+Kn3P1Bd89z97y0tLSW/nXNakCPRL73hRxeX7mD\n11Zo5VYRaduaFA7uXhJ8LQdeBsa4e5m717h7LfAQMCbYvQToU+/wrKCtJNhu2H7cMWYWA6QAu86k\nQ23Zdy7MZkRWCv/6l2Ws3VEV7nJERE7olOFgZolm1qVuG7gMWBmMIdS5FlgZbL8CTA1mIA0gNPC8\n2N1LgSozOz8YT7gJmFPvmGnB9nXAO94OPzUnNjqKB2/KI6lzDNP/nM/O/YfDXZKIyGdqypVDBrDA\nzJYBi4FX3f0N4JfBtNTlwCTg+wDuvgp4HlgNvAHMcPea4GfdBjxMaJB6A/B60P4IkGpmRcDdwD3N\n0bm2KCO5Mw/dlMeuA4e55YkCDlfXnPogEZFWZpH6B3peXp7n5+eHu4wz9vfl27n96Y+5bnQWv7ru\nXNrR5CwRacPMrKDBIwmfKaY1ipHGvnhub9aX7ed3c9czOCOJmy8aGO6SRESOUTiE0V2Tcygq389/\nvL6WgWlJTP5cRrhLEhEBtLZSWEVFGb++fgTDe6dw5zMfU7hjX7hLEhEBFA5hF98pmoduyiMxLobp\nsz9il2YwiUgboHBoA3qmhGYwVew7zK1PLuFItT7/QUTCS+HQRozo05VfXT+CxZt38z//uoJInUUm\nIu2DBqTbkKtG9KaofD8PzF3P4IwufPvC7HCXJCIdlMKhjfne5ByKyvfxi9fWkJ2WyCVDNINJRFqf\nbiu1MVFRxv3Xj2Ro72TufGYp68o0g0lEWp/CoQ2qm8GU0Cma6bM/YveBI+EuSUQ6GIVDG9UrJZ4H\nb8qjrOowtzxZoBlMItKqFA5t2Mg+XfnVdeeyeNNu/m3OSs1gEpFWowHpNu7qkZmsL9vP7+cVkZPR\nhekTBoS7JBHpAHTlEAHuvnQwU4Zl8PNXVzOvsDzc5YhIB6BwiABRUcbMG0cypGcydz79MUXlmsEk\nIi1L4RAhEjrF8NC0POJio5k+O59KzWASkRakcIggmV3j+dPXR1O65xC3PqUZTCLSchQOEWZ0v27c\nd905LNy4m39/ZZVmMIlIi9BspQh07ags1pXtZ9b8DeRmJPGN8ZrBJCLNS1cOEepfL8vl0qEZ/O+/\nr+a9dRXhLkdE2hmFQ4SKijJ+e+NIBmd0YcbTS1ivNZhEpBkpHCJYYlwMD0/LIy4mmuv+34cs2rgr\n3CWJSDuhcIhwWd0SePm2cfRI6sTXH1nMXz8uCXdJItIOKBzagT7dE3jp1vGM7teN7z23lN+9vV6z\nmETkrCgc2omUhFhmf2sMXz4vi5lvr+MHf1mm5yBE5IxpKms70ikmil9ffy79UhP4zVvr2L7nIH/6\nlzxSEmLDXZqIRBhdObQzZsadk3P47Y0jWbJlD9fOep+tuz4Jd1kiEmEUDu3UNaMyeWL6GHYfOMI1\nf3yfgi2V4S5JRCKIwqEdG5udyku3jiO5cwxfeWghry4vDXdJIhIhmhQOZrbZzFaY2VIzyw/aupvZ\nW2a2Pvjard7+95pZkZkVmtmUeu2jg59TZGYPmJkF7XFm9lzQvsjM+jdvNzuu7LQkXrptPOdmpjDj\n6SXMmr9BM5lE5JRO58phkruPdPe84Pt7gLnungPMDb7HzIYCU4FhwOXAH80sOjhmFvAdICd4XR60\nTwcq3X0QMBO478y7JA11T+zEk98eyz+P6M19b6zlf7y8kuoazWQSkRM7m9tKVwOzg+3ZwDX12p91\n98PuvgkoAsaYWS8g2d0XeuhP18cbHFP3s14AJtddVUjz6Bwbze9uHMltEwfyzOKtfPvxfPYfrg53\nWSLSRjU1HBx428wKzOzmoC3D3etuYu8AMoLtTGBbvWOLg7bMYLth+3HHuHs1sBdIPY1+SBNERRk/\nunwIv7j2HP6xfic3/ulDyqoOhbssEWmDmhoOE9x9JHAFMMPMLqr/ZnAl0OI3ss3sZjPLN7P8igqt\nRHqmvjq2Lw9Py2PTzgNc+4f3KdyhRftE5HhNCgd3Lwm+lgMvA2OAsuBWEcHX8mD3EqBPvcOzgraS\nYLth+3HHmFkMkAI0WkXO3R909zx3z0tLS2tK6XICk3LTef67F1Bd61w36wM+KNoZ7pJEpA05ZTiY\nWaKZdanbBi4DVgKvANOC3aYBc4LtV4CpwQykAYQGnhcHt6CqzOz8YDzhpgbH1P2s64B3XFNqWtzw\nzBRenjGeXl07M+2xxbxYUHzqg0SkQ2jK8hkZwMvB+HAM8LS7v2FmHwHPm9l0YAtwA4C7rzKz54HV\nQDUww91rgp91G/BnIB54PXgBPAI8YWZFwG5Cs52kFWR2jecvt4zjlicK+MFfllFceZA7Jw9C8wFE\nOjaL1D/Q8/LyPD8/P9xltBtHqmu558XlvPRxCdePzuIXXzqH2Gg9IynS3phZQb1HEk5IC+8JEFq0\n7/4bRpDVPYEH5q6ndO8h/vC180iJ16J9Ih2R/jSUY8yMuy8dzC+/fC4LN+7i0t+8y9+WbdcT1SId\nkMJBGrnh83148dZxpCfHccczH3PTo4vZtPNAuMsSkVakcJDPNKJPV+bMmMDPrhrG0q17mDLzPX7z\n1joOHa059cEiEvEUDnJC0VHGtHH9mfvDi7ninJ48MHc9U377HvMLy099sIhENIWDnFJ6l878buoo\nnv72WKKjjG889hG3PVVA6d6D4S5NRFqIwkGabNygHrx+14X88LLBzF1Tzhfuf5eH/7FRK7yKtEMK\nBzktcTHR3H5JDm/ffTFjs1P5v6+u4Yv/tYCFGxutdiIiEUzhIGekT/cEHpmWx5++Ppp9h6qZ+uBC\n7njmY91qEmknFA5yxsyMKcN68vbdF3PX5Bz+e9UOLvn1u/xhXhGHqzWrSSSSKRzkrMV3iub7lw7m\n7bsv5qLBPfjVm4VcNvM95q4pC3dpInKGFA7SbPp0T+BPX8/jieljiIkyps/O55uP6QE6kUikcJBm\nd2FOGq/fdRE/ufJzfLS5kikz3+O+N9ZyQB9LKhIxFA7SIjrFRPGdi7J55wcX88URvZg1fwOX3D+f\nN1aWnvpgEQk7hYO0qPTkzvzmhpG8eOsFpCbGccuTS7jjmY+pPHAk3KWJyEkoHKRVjO7XnTm3j+fu\nSwfzxspSLp35Lm+s3BHuskTkBBQO0mpio6O4c3IOr9w+gfQunbnlyQLuelZXESJtkcJBWt3neiUz\n5/bxfP8Lg3l1eSmXznyP/16lqwiRtkThIGERGx3FXV/IYc7t40nrEsfNTxTw/eeWsucTXUWItAUK\nBwmrYb1TmDNjPHdNzuFvy7Zz6cz3eHu1Hp4TCTeFg4Rdp5govn/pYP46YzypiZ349uP53PXsx2zZ\npYfnRMJF4SBtxvDMFF65fQJ3XjKI11fu4JL73+Xu55eysWJ/uEsT6XAsUj88Pi8vz/Pz88NdhrSQ\n8qpD/Om9jTy1aAtHqmv55xG9uX3SIHIyuoS7NJGIZmYF7p53yv0UDtKW7dx/mIf+sZEnPtzCwaM1\nXDm8F7dfMojP9UoOd2kiEUnhIO3K7gNHeGTBRmZ/sIX9h6u5bGgGd07OYXhmSrhLE4koCgdpl/Z+\ncpRH39/EY+9voupQNZcMSefmi7IZO6A7Zhbu8kTaPIWDtGtVh47y+AebeWTBJio/OUpOehJfG9uX\nL43OIrlzbLjLE2mzFA7SIRw6WsPflm3nyYVbWFa8l/jYaK4Z1Zuvje2nW04in0HhIB3OiuK9PLlw\nC3OWlXDoaC0j+3Tl6+f345/O7UXn2OhwlyfSJigcpMPa+8lRXlxSzJOLtrCx4gBdE2K5fnQWXxvb\nj/49EsNdnkhYNTUcmvwQnJlFm9nHZvb34PufmlmJmS0NXlfW2/deMysys0Izm1KvfbSZrQjee8CC\nEUQzizOz54L2RWbW/3Q6K1JfSkIs35owgLl3X8zT3x7LuIGpPPb+ZibdP59vPraYd9dVUFsbmX8U\nibSWmNPY9y5gDVB/gvlMd/91/Z3MbCgwFRgG9AbeNrPB7l4DzAK+AywCXgMuB14HpgOV7j7IzKYC\n9wE3nlmXRELMjHGDejBuUA/Kqg7x9KKtPLVoK9MeXUx2WiLfGNefL52XRVLc6fxvINIxNOnKwcyy\ngH8CHm7C7lcDz7r7YXffBBQBY8ysF5Ds7gs9dC/rceCaesfMDrZfACbXXVWINIeM5M58/9LBvH/P\nJH5740i6xMXwb3NWccEv5vKzv61i806t4yRSX1P/ZPot8COg4doFd5jZTUA+8AN3rwQygYX19ikO\n2o4G2w3bCb5uA3D3ajPbC6QCO5veFZFTi4uJ5ppRmVwzKpOPt1by5w828+TCLfz5g81Myk1n2rj+\nXDioB1FR+ttEOrZTXjmY2ReBcncvaPDWLCAbGAmUAvc3f3mNarnZzPLNLL+ioqKlf520c6P6duN3\nU0fx/o8v4c5LclhevJdpjy7mCzPf5bH3N7H34NFwlygSNk25rTQeuMrMNgPPApeY2ZPuXubuNe5e\nCzwEjAn2LwH61Ds+K2grCbYbth93jJnFACnAroaFuPuD7p7n7nlpaWlN7KLIyaU3vOXUOZaf/W01\n5/9iLj9+YTkriveGu0SRVnfKcHD3e909y937Expofsfd/yUYQ6hzLbAy2H4FmBrMQBoA5ACL3b0U\nqDKz84PxhJuAOfWOmRZsXxf8Dk0nkVZVd8tpzozx/O32CVw1ojdzlpXwz79fwNW/X8Dz+ds4eKQm\n3GWKtIrTes7BzCYCP3T3L5rZE4RuKTmwGfhuEACY2U+AbwHVwPfc/fWgPQ/4MxBPaJbSHe7uZtYZ\neAIYBewGprr7xpPVouccpDXsPXiUl5YU8+TCLWyoOEBy5xiuG92Hr53fl4FpSeEuT+S06SE4kWbk\n7izcuJsnF23hzZU7qK51xg9K5evn9+PSoT2J1gC2RIimhoMmeIs0gZlxwcBULhiYSvm+Qzz/0Tae\nXrSVW55cQla3eL45fgA35GXRRYv+STuhKweRM1RT67y1egePLNjER5sr6RIXw42f78M3xvcnq1tC\nuMsT+Uy6rSTSipZu28MjCzbx2opS3J0rhvdi+oUDOK9vt3CXJnIchYNIGGzfc5DZH2zm6cVb2Xeo\nmlF9u/LtCdlMGZZBTHSTlzITaTEKB5EwOnC4mhcKinn0/U1s2fUJmV3j+cqYPlx7XhaZXePDXZ50\nYAoHkTagptaZu6aMx97fzIcbd2EGF2Sn8uXzsrjinJ4kdNKcEGldCgeRNmbb7k94aUkJLy4pZuvu\nT0jsFM0V5/Tiy+dlMXZAd63nJK1C4SDSRrk7H22u5MWCYl5dUcr+w9VkdYvnS6My+fLoLPql6gOJ\npOUoHEQiwMEjNfz36h28UFDMgqKduMPoft24YnhPpgzrSZ/umhIrzUvhIBJhSvce5OWPS/jbslLW\nlFYBMLRXMlOG9eTy4T0ZnJGEPuZEzpbCQSSCbd31CW+u2sGbq3ZQsLUSd+ifmsCUYT2ZMrwnI7O6\naoxCzojCQaSdKN93iLdWl/HmqjI+KNpJda2TkRzHZUN7MmlIGmMHpJKojzqVJlI4iLRDew8eZd7a\nct5ctYP5hRUcPFpDbLQxqm83LhzUgwsHp3FOZooWApQTUjiItHOHjtZQsKWSf6zfyT/WV7Bqe2ic\nIiU+lnEDU5mQ04MLB6XRN1WD2vIphYNIB7Nr/2He37CLBesrWLB+J9v3HgKgb/cELh6cxhXDezJm\nQHct49HBKRxEOjB3Z0PFgVBQFO3k/aJdHDxaQ2piJ6YM78mVw3txfraCoiNSOIjIMZ8cqWZ+YQWv\nrSjlnbXlfHKkhu6JnZgyLIMrhvfigoGpxCooOgSFg4h8poNHanh3XTmvrdjB3DVlHDhSQ9eEWKYM\n7ckV5/Rk3MAedIpRULRXCgcROaVDR2t4d10Fr68o5e015ew/XE1ip2gm5PRgUm46k4akk5HcOdxl\nSjPSx4SKyCl1jo0OPVg3rCeHjtawYP1O3iksD6bLlgEwrHfysaAY2aerpsl2ELpyEJFG3J3Csn3M\nW1vBvLXlFGytpKbW6ZYQy8WD05g0JJ2LB6fRNaFTuEuV06TbSiLSbPZ+cpT31lcwr7Ccdwsr2HXg\nCFEG5/XtxqQh6VwyJJ0hPbto7acIoHAQkRZRW+ssK97DvMIK3llbxsqS0MN3vVM6HwuKcQN7EN8p\nOsyVymdROIhIqyirOsT8wnLeWVvOgvU7OXCkhriYKC4YmMolQ9KZlJuupcfbEIWDiLS6w9U1LN60\nm3fWhga1N+/6BICsbvGM7teN8/p2Y3S/bgzp2UUP4IWJwkFEwm5jxX7mF1aQv2U3BVsqKas6DEB8\nbDQj+qQcC4tRfbvRPVGD261B4SAibYq7s33vIQq2VLJkSyVLtlayensV1bWhf4OyeyQysm9XhvVO\nYVjvZIb2Tia5c2yYq25/9JyDiLQpZkZm13gyu8Zz1YjeQOhp7eXFe1iydc+xFWZfWlJy7Ji+3RMY\n1js5eIVCI10P5bUKhYOIhE18p2jGZqcyNjv1WFv5vkOs2l7F6u1VrNq+l1Xbq3h95Y5j7/dIimN4\nZjLnZ6cyKTddH5/aQnRbSUTavKpDR1mzvYrVpVWs2l7F8uI9rCvbD0CvlM5MzE1jYm464wf1IEmf\nindSzT7mYGbRQD5Q4u5fNLPuwHNAf2AzcIO7Vwb73gtMB2qAO939zaB9NPBnIB54DbjL3d3M4oDH\ngdHALuBGd998snoUDiId2469h3h3XTnz1oaWJd9/uJrYaOPz/bszMTeNSbnpDErXVUVDLREOdwN5\nQHIQDr8Edrv7f5rZPUA3d/+xmQ0FngHGAL2Bt4HB7l5jZouBO4FFhMLhAXd/3cxuA85191vMbCpw\nrbvfeLJ6FA4iUudoTS35myuZv66c+WsrKCzbB0Bm13jGDUwlt2cXcnt2YXBGF9K7xHXowGjWcDCz\nLGA28HPg7iAcCoGJ7l5qZr2A+e6eG1w14O7/ERz7JvBTQlcX89x9SND+leD479bt4+4fmlkMsANI\n85MUp3AQkRPZvucg8wsrmF9YTsGWSnYdOHLsvZT4WAZnJJGT0YXcjC7kZCSRm9GF1KS4MFbcepp7\nttJvgR/wJgciAAAHLUlEQVQBXeq1Zbh7abC9A8gItjOBhfX2Kw7ajgbbDdvrjtkG4O7VZrYXSAV2\nNrE+EZFjeneN56tj+/LVsX2B0Eeorivbz7qyfcdery4v5emDW48dk5rYiaHBFNrhwcyo/qmJRHXQ\nVWhPGQ5m9kWg3N0LzGziZ+0TjBu0+Mi2md0M3AzQt2/flv51ItJOpCbFcUFSHBcM/HRWlLtTvu9w\nEBb7KdwRGvB+dMEmjtaE/jlL7BTN0HrTaIdnpjAoPalDfGpeU64cxgNXmdmVQGcg2cyeBMrMrFe9\n20rlwf4lQJ96x2cFbSXBdsP2+scUB7eVUggNTB/H3R8EHoTQbaWmdVFEpDEzIyO5MxnJnbkwJ+1Y\n+5HqWtaX72NVSWgq7crtVTz30TYOHq0BoFNMFJ/rlczovt3I6x96wrs9fiDSaU1lDa4cfhiMOfwK\n2FVvQLq7u//IzIYBT/PpgPRcIOcEA9L/5e6vmdkM4Jx6A9JfcvcbTlaLxhxEpLXU1Dqbdh449tzF\n0q17WFa8h8PVtcCna0fl9evGef26MaRncpv9UKTWeEL6P4HnzWw6sAW4AcDdV5nZ88BqoBqY4e41\nwTG38elU1teDF8AjwBNmVgTsBqaeRV0iIs0qOsoYlJ7EoPQkrh4ZGio9Ul3L6tIq8jfvZsnWSj7c\nsIs5S7cDodtRo/rWBUUXBqYl0S81gc6xkbOMuR6CExFpBu5OceVBCrZUUrClkvwtlRTuqCJYOgqz\n0BVGdo8kstMSGZj26dfWnF6rtZVERFqRmdGnewJ9uidwzajQ1cWBw9Vs2nmADRX72VhxgI07D7Ch\nfD+LN+0+NoYBkBQXw8D0JIb2Sj62ltSQnslh/cAkhYOISAtJjItheGYKwzNTjmuvrXV2VB0KAiMU\nHIU79vHq8u08szg0vTbKYGBaUjBb6tMZU631ud0KBxGRVhYVZfTuGk/vrvFMyOlxrN3dKdlzkFXb\nq4LFB/eyeNPuY2MZEHrq+0eX5x4b+2gpCgcRkTbCzMjqlkBWtwSmDOt5rH33gSPHrVKb1qXln+ZW\nOIiItHHdEzsxIafHcVcZLa39P+YnIiKnTeEgIiKNKBxERKQRhYOIiDSicBARkUYUDiIi0ojCQURE\nGlE4iIhIIxG7KquZVRBaKry+HrSvjxZtb/2B9ten9tYfaH99am/9gbPrUz93TzvVThEbDp/FzPKb\nshRtpGhv/YH216f21h9of31qb/2B1umTbiuJiEgjCgcREWmkvYXDg+EuoJm1t/5A++tTe+sPtL8+\ntbf+QCv0qV2NOYiISPNob1cOIiLSDNpFOJjZ5WZWaGZFZnZPuOs5U2a22cxWmNlSM8sP2rqb2Vtm\ntj742i3cdZ6ImT1qZuVmtrJe2wnrN7N7g3NWaGZTwlP1yZ2gTz81s5LgPC01syvrvdem+2Rmfcxs\nnpmtNrNVZnZX0B6R5+kk/Ynkc9TZzBab2bKgTz8L2lv3HLl7RL+AaGADkA10ApYBQ8Nd1xn2ZTPQ\no0HbL4F7gu17gPvCXedJ6r8IOA9Year6gaHBuYoDBgTnMDrcfWhin34K/PAz9m3zfQJ6AecF212A\ndUHdEXmeTtKfSD5HBiQF27HAIuD81j5H7eHKYQxQ5O4b3f0I8CxwdZhrak5XA7OD7dnANWGs5aTc\n/T1gd4PmE9V/NfCsux92901AEaFz2aacoE8n0ub75O6l7r4k2N4HrAEyidDzdJL+nEib7g+Ah+wP\nvo0NXk4rn6P2EA6ZwLZ63xdz8v842jIH3jazAjO7OWjLcPfSYHsHkBGe0s7YieqP9PN2h5ktD247\n1V3eR1SfzKw/MIrQX6YRf54a9Aci+ByZWbSZLQXKgbfcvdXPUXsIh/ZkgruPBK4AZpjZRfXf9NA1\nZMROL4v0+uuZReg25kigFLg/vOWcPjNLAl4EvufuVfXfi8Tz9Bn9iehz5O41wb8FWcAYMxve4P0W\nP0ftIRxKgD71vs8K2iKOu5cEX8uBlwldGpaZWS+A4Gt5+Co8IyeqP2LPm7uXBf/z1gIP8eklfET0\nycxiCf1D+pS7vxQ0R+x5+qz+RPo5quPue4B5wOW08jlqD+HwEZBjZgPMrBMwFXglzDWdNjNLNLMu\nddvAZcBKQn2ZFuw2DZgTngrP2InqfwWYamZxZjYAyAEWh6G+01b3P2jgWkLnCSKgT2ZmwCPAGnf/\nTb23IvI8nag/EX6O0sysa7AdD1wKrKW1z1G4R+abaXT/SkKzFDYAPwl3PWfYh2xCMw6WAavq+gGk\nAnOB9cDbQPdw13qSPjxD6BL+KKH7ntNPVj/wk+CcFQJXhLv+0+jTE8AKYHnwP2avSOkTMIHQ7Yjl\nwNLgdWWknqeT9CeSz9G5wMdB7SuBfwvaW/Uc6QlpERFppD3cVhIRkWamcBARkUYUDiIi0ojCQURE\nGlE4iIhIIwoHERFpROEgIiKNKBxERKSR/w+vTHOtZSQe+gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2050a17e630>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(k_values, k_inertia)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The inertia score is quite a smooth line. I think around 100-150 clusters is when the gradient begins to flatten."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "k_vals_inertia_list = list(zip(k_values, k_inertia))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savetxt(\"k_inertia.csv\", k_vals_inertia_list, delimiter=\",\", fmt='%s')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save the cluster labels for each value of k into a DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "km_df_list = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for k, clusters in k_dict.items():\n",
    "    k = pd.DataFrame({k:clusters[0]})\n",
    "    km_df_list.append(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "km_df = pd.concat(km_df_list, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Combine with article + DBSCAN clusters DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df3 = pd.concat([df2, km_df], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime</th>\n",
       "      <th>source</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>text_snowball</th>\n",
       "      <th>db_1_clusters</th>\n",
       "      <th>db_2_clusters</th>\n",
       "      <th>db_3_clusters</th>\n",
       "      <th>10</th>\n",
       "      <th>20</th>\n",
       "      <th>...</th>\n",
       "      <th>210</th>\n",
       "      <th>220</th>\n",
       "      <th>230</th>\n",
       "      <th>240</th>\n",
       "      <th>250</th>\n",
       "      <th>260</th>\n",
       "      <th>270</th>\n",
       "      <th>280</th>\n",
       "      <th>290</th>\n",
       "      <th>300</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23536</th>\n",
       "      <td>2008-09-12 00:00:00</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>Buoyant miners help European stocks snap losin...</td>\n",
       "      <td>http://uk.reuters.com/article/markets-europe-s...</td>\n",
       "      <td>european stock end snap three session lose str...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>26</td>\n",
       "      <td>196</td>\n",
       "      <td>227</td>\n",
       "      <td>121</td>\n",
       "      <td>168</td>\n",
       "      <td>7</td>\n",
       "      <td>16</td>\n",
       "      <td>64</td>\n",
       "      <td>61</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58865</th>\n",
       "      <td>2017-06-15</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>SE Asia Stocks-End lower on hawkish Fed; Singa...</td>\n",
       "      <td>http://uk.reuters.com/article/southeast-asia-s...</td>\n",
       "      <td>southeast asian end hit hawkish feder reserv w...</td>\n",
       "      <td>21</td>\n",
       "      <td>56</td>\n",
       "      <td>69</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>23</td>\n",
       "      <td>30</td>\n",
       "      <td>40</td>\n",
       "      <td>10</td>\n",
       "      <td>16</td>\n",
       "      <td>39</td>\n",
       "      <td>11</td>\n",
       "      <td>62</td>\n",
       "      <td>64</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1814</th>\n",
       "      <td>2008-08-20 08:46:30</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>Bank split three ways on rates</td>\n",
       "      <td>https://www.theguardian.com/business/2008/aug/...</td>\n",
       "      <td>policymak bank england split three way second ...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>131</td>\n",
       "      <td>28</td>\n",
       "      <td>195</td>\n",
       "      <td>155</td>\n",
       "      <td>171</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>251</td>\n",
       "      <td>214</td>\n",
       "      <td>130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41612</th>\n",
       "      <td>2017-04-13</td>\n",
       "      <td>Reuters</td>\n",
       "      <td>GLOBAL MARKETS-Trump comments trigger U.S. dol...</td>\n",
       "      <td>http://uk.reuters.com/article/global-markets-i...</td>\n",
       "      <td>the us dollar treasuri yield slid track bigges...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>76</td>\n",
       "      <td>96</td>\n",
       "      <td>162</td>\n",
       "      <td>45</td>\n",
       "      <td>194</td>\n",
       "      <td>76</td>\n",
       "      <td>59</td>\n",
       "      <td>228</td>\n",
       "      <td>259</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4782</th>\n",
       "      <td>2010-06-14 18:40:53</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>Demographic shift 'will cut UK growth'</td>\n",
       "      <td>https://www.theguardian.com/business/2010/jun/...</td>\n",
       "      <td>the long term consequ effect ban immigr age so...</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>173</td>\n",
       "      <td>193</td>\n",
       "      <td>199</td>\n",
       "      <td>13</td>\n",
       "      <td>233</td>\n",
       "      <td>82</td>\n",
       "      <td>115</td>\n",
       "      <td>262</td>\n",
       "      <td>150</td>\n",
       "      <td>203</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 38 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  datetime    source  \\\n",
       "23536  2008-09-12 00:00:00   Reuters   \n",
       "58865           2017-06-15   Reuters   \n",
       "1814   2008-08-20 08:46:30  Guardian   \n",
       "41612           2017-04-13   Reuters   \n",
       "4782   2010-06-14 18:40:53  Guardian   \n",
       "\n",
       "                                                   title  \\\n",
       "23536  Buoyant miners help European stocks snap losin...   \n",
       "58865  SE Asia Stocks-End lower on hawkish Fed; Singa...   \n",
       "1814                      Bank split three ways on rates   \n",
       "41612  GLOBAL MARKETS-Trump comments trigger U.S. dol...   \n",
       "4782              Demographic shift 'will cut UK growth'   \n",
       "\n",
       "                                                     url  \\\n",
       "23536  http://uk.reuters.com/article/markets-europe-s...   \n",
       "58865  http://uk.reuters.com/article/southeast-asia-s...   \n",
       "1814   https://www.theguardian.com/business/2008/aug/...   \n",
       "41612  http://uk.reuters.com/article/global-markets-i...   \n",
       "4782   https://www.theguardian.com/business/2010/jun/...   \n",
       "\n",
       "                                           text_snowball  db_1_clusters  \\\n",
       "23536  european stock end snap three session lose str...             -1   \n",
       "58865  southeast asian end hit hawkish feder reserv w...             21   \n",
       "1814   policymak bank england split three way second ...             -1   \n",
       "41612  the us dollar treasuri yield slid track bigges...             -1   \n",
       "4782   the long term consequ effect ban immigr age so...             -1   \n",
       "\n",
       "       db_2_clusters  db_3_clusters  10  20 ...   210  220  230  240  250  \\\n",
       "23536             -1             -1   7  10 ...    26  196  227  121  168   \n",
       "58865             56             69   8   4 ...    23   30   40   10   16   \n",
       "1814              -1              1   1  11 ...   131   28  195  155  171   \n",
       "41612             -1             -1   0   2 ...    76   96  162   45  194   \n",
       "4782              -1             -1   1  17 ...   173  193  199   13  233   \n",
       "\n",
       "       260  270  280  290  300  \n",
       "23536    7   16   64   61   97  \n",
       "58865   39   11   62   64   53  \n",
       "1814    35    0  251  214  130  \n",
       "41612   76   59  228  259   82  \n",
       "4782    82  115  262  150  203  \n",
       "\n",
       "[5 rows x 38 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df3.to_csv('clusters_with_kmeans.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K-Means, k=150"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Having explored the data, k=150 appeared to generate good clusters of topics. I will re-create so that this time I can capture & explore the cluster centre data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "km = KMeans(n_clusters=150, init='k-means++')\n",
    "km.fit(lsa_vecs)\n",
    "clusters = km.predict(lsa_vecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150, 400)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "km.cluster_centers_.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "150 clusters based on the 400 latent dimensions created through LSA."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explore the original dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150, 157260)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "original_space_centroids = lsa.inverse_transform(km.cluster_centers_)\n",
    "original_space_centroids.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150, 157260)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "order_centroids = original_space_centroids.argsort()[:, ::-1]\n",
    "order_centroids.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save('ordered_centroids', order_centroids)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Examine top 10 TF-IDF features  in each cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cluster 0\n",
      "\n",
      "spreadbett\n",
      "financi spreadbett\n",
      "seen\n",
      "expect britain ftse\n",
      "expect britain\n",
      "spreadbett expect\n",
      "financi spreadbett expect\n",
      "britain ftse germani dax\n",
      "britain ftse germani\n",
      "spreadbett expect britain\n",
      "\n",
      "\n",
      "Cluster 1\n",
      "\n",
      "economi\n",
      "debt\n",
      "growth\n",
      "govern\n",
      "econom\n",
      "rate\n",
      "bank\n",
      "polici\n",
      "crisi\n",
      "recess\n",
      "\n",
      "\n",
      "Cluster 2\n",
      "\n",
      "nikkei\n",
      "yen\n",
      "corp\n",
      "japan\n",
      "japanes\n",
      "euro\n",
      "trade\n",
      "nikkei futur\n",
      "billion yen\n",
      "gold\n",
      "\n",
      "\n",
      "Cluster 3\n",
      "\n",
      "nasdaq\n",
      "high low\n",
      "low\n",
      "dow\n",
      "issu\n",
      "high\n",
      "rate\n",
      "fed\n",
      "advanc\n",
      "et\n",
      "\n",
      "\n",
      "Cluster 4\n",
      "\n",
      "russia\n",
      "russian\n",
      "ukrain\n",
      "sanction\n",
      "putin\n",
      "moscow\n",
      "roubl\n",
      "compani\n",
      "oil\n",
      "tension\n",
      "\n",
      "\n",
      "Cluster 5\n",
      "\n",
      "mc\n",
      "santand\n",
      "san mc\n",
      "santand san mc\n",
      "santand san\n",
      "banco\n",
      "european\n",
      "san\n",
      "banco santand\n",
      "banco santand san\n",
      "\n",
      "\n",
      "Cluster 6\n",
      "\n",
      "nikkei\n",
      "yen\n",
      "corp\n",
      "motor\n",
      "japan\n",
      "jpx\n",
      "jpx nikkei\n",
      "jpx nikkei index\n",
      "nikkei index\n",
      "toyota\n",
      "\n",
      "\n",
      "Cluster 7\n",
      "\n",
      "greec\n",
      "greek\n",
      "athen\n",
      "tsipra\n",
      "debt\n",
      "bailout\n",
      "eurozon\n",
      "countri\n",
      "minist\n",
      "auster\n",
      "\n",
      "\n",
      "Cluster 8\n",
      "\n",
      "european\n",
      "stoxx\n",
      "share\n",
      "bank\n",
      "stock\n",
      "pan european\n",
      "pan\n",
      "sector\n",
      "europ\n",
      "european share\n",
      "\n",
      "\n",
      "Cluster 9\n",
      "\n",
      "ft\n",
      "trump\n",
      "video\n",
      "nyt\n",
      "presid\n",
      "firstft\n",
      "mr\n",
      "donald\n",
      "brexit\n",
      "donald trump\n",
      "\n",
      "\n",
      "Cluster 10\n",
      "\n",
      "earn releas\n",
      "releas\n",
      "earn\n",
      "year\n",
      "plc\n",
      "group\n",
      "half year\n",
      "statement releas\n",
      "statement\n",
      "year earn\n",
      "\n",
      "\n",
      "Cluster 11\n",
      "\n",
      "china\n",
      "hong\n",
      "kong\n",
      "hong kong\n",
      "shanghai\n",
      "index\n",
      "csi\n",
      "stock\n",
      "shanghai composit index\n",
      "shanghai composit\n",
      "\n",
      "\n",
      "Cluster 12\n",
      "\n",
      "month\n",
      "increas\n",
      "quarter\n",
      "spend\n",
      "consum\n",
      "economist\n",
      "home\n",
      "growth\n",
      "rate\n",
      "price\n",
      "\n",
      "\n",
      "Cluster 13\n",
      "\n",
      "nasdaq\n",
      "new york\n",
      "york\n",
      "new\n",
      "nyse\n",
      "high low\n",
      "ratio\n",
      "high\n",
      "exchang\n",
      "dow\n",
      "\n",
      "\n",
      "Cluster 14\n",
      "\n",
      "nikkei\n",
      "aiko\n",
      "aiko hayashi\n",
      "hayashi\n",
      "topx\n",
      "topix topx\n",
      "broader topix topx\n",
      "benchmark nikkei\n",
      "topix\n",
      "broader topix\n",
      "\n",
      "\n",
      "Cluster 15\n",
      "\n",
      "australia\n",
      "new zealand\n",
      "zealand\n",
      "australian\n",
      "nz\n",
      "new\n",
      "asx\n",
      "nzx\n",
      "bank\n",
      "australian share\n",
      "\n",
      "\n",
      "Cluster 16\n",
      "\n",
      "mortgag\n",
      "rate\n",
      "loan\n",
      "bank\n",
      "lend\n",
      "borrow\n",
      "credit\n",
      "lender\n",
      "hous\n",
      "home\n",
      "\n",
      "\n",
      "Cluster 17\n",
      "\n",
      "mini futur\n",
      "mini\n",
      "futur\n",
      "premarket\n",
      "premarket trade\n",
      "nasdaq mini futur\n",
      "jone industri mini\n",
      "dow jone industri mini\n",
      "industri mini futur\n",
      "jone industri mini futur\n",
      "\n",
      "\n",
      "Cluster 18\n",
      "\n",
      "italian\n",
      "itali\n",
      "bank\n",
      "paschi\n",
      "mont\n",
      "european\n",
      "dei paschi\n",
      "dei\n",
      "mont dei\n",
      "mont dei paschi\n",
      "\n",
      "\n",
      "Cluster 19\n",
      "\n",
      "european\n",
      "european ftseurofirst fteu\n",
      "pan european ftseurofirst fteu\n",
      "european ftseurofirst\n",
      "pan european ftseurofirst\n",
      "pan european\n",
      "pan\n",
      "ftseurofirst\n",
      "share\n",
      "ftseurofirst fteu share\n",
      "\n",
      "\n",
      "Cluster 20\n",
      "\n",
      "nikkei\n",
      "yen\n",
      "corp\n",
      "japan\n",
      "japanes\n",
      "investor\n",
      "motor\n",
      "topix\n",
      "gain\n",
      "trade\n",
      "\n",
      "\n",
      "Cluster 21\n",
      "\n",
      "job\n",
      "labor\n",
      "unemploy\n",
      "employ\n",
      "rate\n",
      "month\n",
      "number\n",
      "economist\n",
      "worker\n",
      "wage\n",
      "\n",
      "\n",
      "Cluster 22\n",
      "\n",
      "rio\n",
      "european\n",
      "rio tinto\n",
      "tinto\n",
      "european share\n",
      "bhp\n",
      "miner\n",
      "share\n",
      "tinto rio\n",
      "rio tinto rio\n",
      "\n",
      "\n",
      "Cluster 23\n",
      "\n",
      "retail\n",
      "store\n",
      "sale\n",
      "shop\n",
      "christma\n",
      "tesco\n",
      "shopper\n",
      "onlin\n",
      "like\n",
      "supermarket\n",
      "\n",
      "\n",
      "Cluster 24\n",
      "\n",
      "hous\n",
      "home\n",
      "price\n",
      "properti\n",
      "hous price\n",
      "mortgag\n",
      "buyer\n",
      "rate\n",
      "month\n",
      "buy\n",
      "\n",
      "\n",
      "Cluster 25\n",
      "\n",
      "compani\n",
      "share\n",
      "execut\n",
      "offer\n",
      "list\n",
      "sharehold\n",
      "busi\n",
      "deal\n",
      "group\n",
      "billion\n",
      "\n",
      "\n",
      "Cluster 26\n",
      "\n",
      "deutsch\n",
      "euro\n",
      "boers\n",
      "deutsch boers\n",
      "million\n",
      "billion\n",
      "group\n",
      "compani\n",
      "german\n",
      "lse\n",
      "\n",
      "\n",
      "Cluster 27\n",
      "\n",
      "eu\n",
      "brexit\n",
      "britain\n",
      "uk\n",
      "vote\n",
      "leav\n",
      "union\n",
      "referendum\n",
      "european\n",
      "european union\n",
      "\n",
      "\n",
      "Cluster 28\n",
      "\n",
      "dow\n",
      "dow jone industri averag\n",
      "jone industri averag\n",
      "industri averag\n",
      "averag\n",
      "nasdaq\n",
      "dow jone\n",
      "jone\n",
      "jone industri\n",
      "dow jone industri\n",
      "\n",
      "\n",
      "Cluster 29\n",
      "\n",
      "dollar\n",
      "yield\n",
      "barrel\n",
      "price\n",
      "bp\n",
      "currenc\n",
      "week\n",
      "trump\n",
      "bond\n",
      "gain\n",
      "\n",
      "\n",
      "Cluster 30\n",
      "\n",
      "nikkei\n",
      "yen\n",
      "mitsubishi\n",
      "morgan\n",
      "morgan stanley\n",
      "stanley\n",
      "ufj\n",
      "mitsubishi ufj\n",
      "fujito\n",
      "japan\n",
      "\n",
      "\n",
      "Cluster 31\n",
      "\n",
      "yen\n",
      "nikkei\n",
      "corp\n",
      "nt\n",
      "japan\n",
      "share\n",
      "maker\n",
      "profit\n",
      "stock\n",
      "steel\n",
      "\n",
      "\n",
      "Cluster 32\n",
      "\n",
      "osborn\n",
      "chancellor\n",
      "budget\n",
      "cut\n",
      "spend\n",
      "tax\n",
      "govern\n",
      "georg\n",
      "georg osborn\n",
      "deficit\n",
      "\n",
      "\n",
      "Cluster 33\n",
      "\n",
      "industri gain nasdaq composit\n",
      "industri gain nasdaq\n",
      "jone industri gain nasdaq\n",
      "dow jone industri gain\n",
      "jone industri gain\n",
      "industri gain\n",
      "gain nasdaq composit\n",
      "gain nasdaq\n",
      "gain nasdaq composit ad\n",
      "nasdaq composit ad\n",
      "\n",
      "\n",
      "Cluster 34\n",
      "\n",
      "wage\n",
      "worker\n",
      "pay\n",
      "employ\n",
      "work\n",
      "hour\n",
      "job\n",
      "labour\n",
      "minimum\n",
      "minimum wage\n",
      "\n",
      "\n",
      "Cluster 35\n",
      "\n",
      "deficit\n",
      "budget\n",
      "public\n",
      "tax\n",
      "spend\n",
      "govern\n",
      "borrow\n",
      "chancellor\n",
      "public financ\n",
      "obr\n",
      "\n",
      "\n",
      "Cluster 36\n",
      "\n",
      "news\n",
      "com\n",
      "uk\n",
      "britain\n",
      "british\n",
      "inform news\n",
      "inform news com\n",
      "com bridgest view\n",
      "bridgest view stori\n",
      "bridgest view\n",
      "\n",
      "\n",
      "Cluster 37\n",
      "\n",
      "hong\n",
      "kong\n",
      "hong kong\n",
      "china\n",
      "hang seng\n",
      "seng\n",
      "hang\n",
      "china enterpris\n",
      "enterpris index\n",
      "china enterpris index\n",
      "\n",
      "\n",
      "Cluster 38\n",
      "\n",
      "incom\n",
      "women\n",
      "work\n",
      "famili\n",
      "household\n",
      "poverti\n",
      "children\n",
      "benefit\n",
      "live\n",
      "inequ\n",
      "\n",
      "\n",
      "Cluster 39\n",
      "\n",
      "european\n",
      "equiti\n",
      "share\n",
      "earn\n",
      "ftseurofirst\n",
      "european share\n",
      "expect\n",
      "bank\n",
      "europ\n",
      "investor\n",
      "\n",
      "\n",
      "Cluster 40\n",
      "\n",
      "asian\n",
      "singapor\n",
      "southeast\n",
      "southeast asian\n",
      "asian stock\n",
      "southeast asian stock\n",
      "jakarta\n",
      "bangkok\n",
      "indonesia\n",
      "stock\n",
      "\n",
      "\n",
      "Cluster 41\n",
      "\n",
      "yield\n",
      "bond\n",
      "treasuri\n",
      "rate\n",
      "bond yield\n",
      "price\n",
      "basi\n",
      "treasuri yield\n",
      "note\n",
      "inflat\n",
      "\n",
      "\n",
      "Cluster 42\n",
      "\n",
      "bond\n",
      "debt\n",
      "yield\n",
      "investor\n",
      "bank\n",
      "rate\n",
      "corpor\n",
      "fund\n",
      "compani\n",
      "default\n",
      "\n",
      "\n",
      "Cluster 43\n",
      "\n",
      "yen\n",
      "nikkei\n",
      "corp\n",
      "japan\n",
      "motor\n",
      "share\n",
      "gain\n",
      "export\n",
      "toyota\n",
      "tokyo\n",
      "\n",
      "\n",
      "Cluster 44\n",
      "\n",
      "nikkei\n",
      "japan\n",
      "topix\n",
      "broader topix\n",
      "japan nikkei\n",
      "broader\n",
      "benchmark nikkei\n",
      "gain\n",
      "edit joseph radford\n",
      "edit joseph\n",
      "\n",
      "\n",
      "Cluster 45\n",
      "\n",
      "singapor\n",
      "asian\n",
      "manila\n",
      "jakarta\n",
      "singapor bangkok\n",
      "lumpur ho chi\n",
      "lumpur ho chi minh\n",
      "kuala lumpur ho chi\n",
      "lumpur ho\n",
      "kuala lumpur ho\n",
      "\n",
      "\n",
      "Cluster 46\n",
      "\n",
      "shanghai\n",
      "shenzhen\n",
      "china\n",
      "csi\n",
      "shanghai shenzhen\n",
      "shanghai composit\n",
      "largest list compani\n",
      "shanghai composit index\n",
      "compani shanghai shenzhen\n",
      "list compani shanghai shenzhen\n",
      "\n",
      "\n",
      "Cluster 47\n",
      "\n",
      "china\n",
      "chines\n",
      "beij\n",
      "economi\n",
      "growth\n",
      "countri\n",
      "world\n",
      "yuan\n",
      "export\n",
      "econom\n",
      "\n",
      "\n",
      "Cluster 48\n",
      "\n",
      "dividend\n",
      "ftse compani ex\n",
      "ftse compani ex dividend\n",
      "compani ex dividend\n",
      "compani ex\n",
      "ex dividend\n",
      "ric dividend\n",
      "ftse compani\n",
      "ric\n",
      "compani ric\n",
      "\n",
      "\n",
      "Cluster 49\n",
      "\n",
      "mike dolan\n",
      "dolan\n",
      "eikon\n",
      "mike\n",
      "european\n",
      "stock\n",
      "market\n",
      "alert client\n",
      "bell includ stock buzz\n",
      "london close bell\n",
      "\n",
      "\n",
      "Cluster 50\n",
      "\n",
      "bp\n",
      "pa\n",
      "european\n",
      "shell\n",
      "oil\n",
      "share\n",
      "royal\n",
      "dutch\n",
      "royal dutch\n",
      "royal dutch shell\n",
      "\n",
      "\n",
      "Cluster 51\n",
      "\n",
      "trade\n",
      "deal\n",
      "agreement\n",
      "countri\n",
      "free\n",
      "free trade\n",
      "negoti\n",
      "trump\n",
      "eu\n",
      "trade agreement\n",
      "\n",
      "\n",
      "Cluster 52\n",
      "\n",
      "nasdaq\n",
      "declin\n",
      "high low\n",
      "ratio\n",
      "low\n",
      "high\n",
      "nasdaq composit\n",
      "composit\n",
      "advanc\n",
      "issu\n",
      "\n",
      "\n",
      "Cluster 53\n",
      "\n",
      "futur\n",
      "fcec\n",
      "fdxc\n",
      "stxec\n",
      "cac fcec\n",
      "dax fdxc\n",
      "european stock futur\n",
      "stxec germani dax\n",
      "stxec germani\n",
      "germani dax fdxc\n",
      "\n",
      "\n",
      "Cluster 54\n",
      "\n",
      "nikkei\n",
      "yen\n",
      "nt\n",
      "jpi\n",
      "chg\n",
      "usd\n",
      "corp\n",
      "billion yen\n",
      "japan\n",
      "nikkei busi\n",
      "\n",
      "\n",
      "Cluster 55\n",
      "\n",
      "bank\n",
      "financi\n",
      "regul\n",
      "govern\n",
      "crisi\n",
      "lend\n",
      "lloyd\n",
      "money\n",
      "capit\n",
      "busi\n",
      "\n",
      "\n",
      "Cluster 56\n",
      "\n",
      "ftse\n",
      "cap\n",
      "mid cap\n",
      "chip\n",
      "blue chip\n",
      "blue\n",
      "uk\n",
      "stock\n",
      "britain\n",
      "mid\n",
      "\n",
      "\n",
      "Cluster 57\n",
      "\n",
      "european\n",
      "share\n",
      "european share\n",
      "blais robinson\n",
      "blais\n",
      "robinson\n",
      "ftseurofirst\n",
      "earli\n",
      "european stock\n",
      "earli trade\n",
      "\n",
      "\n",
      "Cluster 58\n",
      "\n",
      "european\n",
      "euro\n",
      "stoxx\n",
      "euro stoxx\n",
      "ftseurofirst\n",
      "euro zone blue\n",
      "zone blue\n",
      "euro zone\n",
      "euro zone blue chip\n",
      "zone blue chip\n",
      "\n",
      "\n",
      "Cluster 59\n",
      "\n",
      "yellen\n",
      "fed\n",
      "janet yellen\n",
      "janet\n",
      "chair\n",
      "chair janet\n",
      "chair janet yellen\n",
      "rate\n",
      "reserv chair\n",
      "feder reserv chair\n",
      "\n",
      "\n",
      "Cluster 60\n",
      "\n",
      "pension\n",
      "save\n",
      "retir\n",
      "scheme\n",
      "pension fund\n",
      "age\n",
      "fund\n",
      "annuiti\n",
      "benefit\n",
      "govern\n",
      "\n",
      "\n",
      "Cluster 61\n",
      "\n",
      "euro\n",
      "euro zone\n",
      "zone\n",
      "european\n",
      "stoxx\n",
      "euro stoxx\n",
      "equiti\n",
      "ftseurofirst\n",
      "share\n",
      "euro zone blue\n",
      "\n",
      "\n",
      "Cluster 62\n",
      "\n",
      "bank\n",
      "central\n",
      "central bank\n",
      "rate\n",
      "polici\n",
      "inflat\n",
      "currenc\n",
      "monetari\n",
      "economi\n",
      "boj\n",
      "\n",
      "\n",
      "Cluster 63\n",
      "\n",
      "ecb\n",
      "draghi\n",
      "euro\n",
      "bank\n",
      "eurozon\n",
      "central\n",
      "central bank\n",
      "bond\n",
      "european\n",
      "mario\n",
      "\n",
      "\n",
      "Cluster 64\n",
      "\n",
      "pound\n",
      "million\n",
      "million pound\n",
      "penc\n",
      "compani\n",
      "billion pound\n",
      "billion\n",
      "revenu\n",
      "pound million\n",
      "million pound million\n",
      "\n",
      "\n",
      "Cluster 65\n",
      "\n",
      "mini\n",
      "contract\n",
      "mini contract\n",
      "contract chang hand\n",
      "contract chang\n",
      "mini contract chang\n",
      "mini contract chang hand\n",
      "chang hand\n",
      "premarket\n",
      "contract dow mini\n",
      "\n",
      "\n",
      "Cluster 66\n",
      "\n",
      "nikkei\n",
      "broader topix\n",
      "topix\n",
      "broader\n",
      "gibb\n",
      "edwina gibb\n",
      "edwina\n",
      "ayai tomisawa\n",
      "tomisawa\n",
      "ayai\n",
      "\n",
      "\n",
      "Cluster 67\n",
      "\n",
      "yen\n",
      "nikkei\n",
      "japan\n",
      "tokyo\n",
      "share\n",
      "japanes\n",
      "investor\n",
      "stock\n",
      "billion\n",
      "gain\n",
      "\n",
      "\n",
      "Cluster 68\n",
      "\n",
      "nikkei\n",
      "topx\n",
      "topix topx\n",
      "topix\n",
      "broader topix topx\n",
      "broader topix\n",
      "broader\n",
      "corp\n",
      "japan\n",
      "japan nikkei\n",
      "\n",
      "\n",
      "Cluster 69\n",
      "\n",
      "emerg\n",
      "currenc\n",
      "south\n",
      "africa\n",
      "em\n",
      "countri\n",
      "mr\n",
      "dollar\n",
      "south africa\n",
      "african\n",
      "\n",
      "\n",
      "Cluster 70\n",
      "\n",
      "stock\n",
      "investor\n",
      "volatil\n",
      "trump\n",
      "equiti\n",
      "elect\n",
      "earn\n",
      "rate\n",
      "sinc\n",
      "invest\n",
      "\n",
      "\n",
      "Cluster 71\n",
      "\n",
      "kong\n",
      "hong\n",
      "hong kong\n",
      "australia\n",
      "asx\n",
      "kong hang seng\n",
      "kong hang\n",
      "hong kong hang seng\n",
      "hong kong hang\n",
      "hang seng\n",
      "\n",
      "\n",
      "Cluster 72\n",
      "\n",
      "nikkei\n",
      "elain lie\n",
      "elain\n",
      "topix topx elain\n",
      "topix topx elain lie\n",
      "topx elain\n",
      "topx elain lie\n",
      "broader topix topx elain\n",
      "lie\n",
      "broader topix topx\n",
      "\n",
      "\n",
      "Cluster 73\n",
      "\n",
      "nasdaq\n",
      "high low\n",
      "high\n",
      "issu\n",
      "dow\n",
      "low\n",
      "nyse nasdaq issu\n",
      "nyse nasdaq\n",
      "et\n",
      "advanc\n",
      "\n",
      "\n",
      "Cluster 74\n",
      "\n",
      "rate\n",
      "bank\n",
      "mpc\n",
      "inflat\n",
      "committe\n",
      "carney\n",
      "bank england\n",
      "england\n",
      "polici\n",
      "economi\n",
      "\n",
      "\n",
      "Cluster 75\n",
      "\n",
      "growth\n",
      "quarter\n",
      "economi\n",
      "uk\n",
      "gdp\n",
      "figur\n",
      "recess\n",
      "export\n",
      "economist\n",
      "spend\n",
      "\n",
      "\n",
      "Cluster 76\n",
      "\n",
      "pa\n",
      "european share\n",
      "european\n",
      "societ general\n",
      "societ\n",
      "sogn\n",
      "general sogn\n",
      "sogn pa\n",
      "general sogn pa\n",
      "societ general sogn\n",
      "\n",
      "\n",
      "Cluster 77\n",
      "\n",
      "unemploy\n",
      "job\n",
      "work\n",
      "employ\n",
      "labour\n",
      "number\n",
      "figur\n",
      "young\n",
      "million\n",
      "rate\n",
      "\n",
      "\n",
      "Cluster 78\n",
      "\n",
      "brown\n",
      "gordon\n",
      "chancellor\n",
      "gordon brown\n",
      "labour\n",
      "darl\n",
      "govern\n",
      "tax\n",
      "minist\n",
      "mr brown\n",
      "\n",
      "\n",
      "Cluster 79\n",
      "\n",
      "valetkevitch edit\n",
      "carolin valetkevitch edit\n",
      "valetkevitch\n",
      "carolin valetkevitch\n",
      "carolin\n",
      "base latest avail\n",
      "latest avail dow jone\n",
      "avail dow jone\n",
      "latest avail dow\n",
      "avail dow\n",
      "\n",
      "\n",
      "Cluster 80\n",
      "\n",
      "climat\n",
      "carbon\n",
      "emiss\n",
      "energi\n",
      "green\n",
      "fuel\n",
      "world\n",
      "fossil\n",
      "fossil fuel\n",
      "environment\n",
      "\n",
      "\n",
      "Cluster 81\n",
      "\n",
      "labour\n",
      "parti\n",
      "tori\n",
      "cameron\n",
      "govern\n",
      "conserv\n",
      "elect\n",
      "polit\n",
      "miliband\n",
      "osborn\n",
      "\n",
      "\n",
      "Cluster 82\n",
      "\n",
      "scotland\n",
      "scottish\n",
      "independ\n",
      "vote\n",
      "uk\n",
      "referendum\n",
      "salmond\n",
      "yes\n",
      "poll\n",
      "campaign\n",
      "\n",
      "\n",
      "Cluster 83\n",
      "\n",
      "french\n",
      "le pen\n",
      "pen\n",
      "le\n",
      "macron\n",
      "elect\n",
      "franc\n",
      "presidenti\n",
      "euro\n",
      "marin le pen\n",
      "\n",
      "\n",
      "Cluster 84\n",
      "\n",
      "pa\n",
      "bank\n",
      "rio\n",
      "bp\n",
      "bg\n",
      "ftse\n",
      "european\n",
      "share\n",
      "vx\n",
      "european share\n",
      "\n",
      "\n",
      "Cluster 85\n",
      "\n",
      "cnbc\n",
      "told cnbc\n",
      "cnbc intern\n",
      "cnbc intern twitter facebook\n",
      "intern twitter facebook\n",
      "cnbc intern twitter\n",
      "intern twitter\n",
      "follow cnbc\n",
      "follow cnbc intern twitter\n",
      "follow cnbc intern\n",
      "\n",
      "\n",
      "Cluster 86\n",
      "\n",
      "kong\n",
      "hong\n",
      "hong kong\n",
      "china\n",
      "hang seng\n",
      "seng\n",
      "index\n",
      "hang\n",
      "shanghai\n",
      "chines\n",
      "\n",
      "\n",
      "Cluster 87\n",
      "\n",
      "mini\n",
      "contract\n",
      "mini contract\n",
      "et\n",
      "contract chang hand\n",
      "contract chang\n",
      "dow mini\n",
      "contract trade\n",
      "mini contract trade\n",
      "volum contract\n",
      "\n",
      "\n",
      "Cluster 88\n",
      "\n",
      "singapor\n",
      "jakarta\n",
      "bangkok\n",
      "singapor kuala\n",
      "singapor kuala lumpur\n",
      "kuala lumpur bangkok\n",
      "lumpur bangkok\n",
      "bangkok jakarta\n",
      "jakarta manila\n",
      "singapor kuala lumpur bangkok\n",
      "\n",
      "\n",
      "Cluster 89\n",
      "\n",
      "nlde\n",
      "pa\n",
      "ftse\n",
      "bank\n",
      "europ\n",
      "european\n",
      "euro\n",
      "stoxx\n",
      "share\n",
      "euro zone\n",
      "\n",
      "\n",
      "Cluster 90\n",
      "\n",
      "eurozon\n",
      "countri\n",
      "spain\n",
      "eu\n",
      "european\n",
      "europ\n",
      "bank\n",
      "greec\n",
      "crisi\n",
      "german\n",
      "\n",
      "\n",
      "Cluster 91\n",
      "\n",
      "univers\n",
      "educ\n",
      "school\n",
      "graduat\n",
      "student\n",
      "skill\n",
      "young\n",
      "econom\n",
      "work\n",
      "job\n",
      "\n",
      "\n",
      "Cluster 92\n",
      "\n",
      "citi\n",
      "govern\n",
      "busi\n",
      "uk\n",
      "local\n",
      "london\n",
      "job\n",
      "north\n",
      "public\n",
      "need\n",
      "\n",
      "\n",
      "Cluster 93\n",
      "\n",
      "trump\n",
      "mr\n",
      "presid\n",
      "mr trump\n",
      "elect\n",
      "donald\n",
      "donald trump\n",
      "tax\n",
      "clinton\n",
      "administr\n",
      "\n",
      "\n",
      "Cluster 94\n",
      "\n",
      "ftse\n",
      "stock\n",
      "britain\n",
      "trader\n",
      "chip ftse\n",
      "blue chip ftse\n",
      "chip\n",
      "blue chip\n",
      "blue\n",
      "record\n",
      "\n",
      "\n",
      "Cluster 95\n",
      "\n",
      "european\n",
      "com\n",
      "europ\n",
      "euro\n",
      "today european research\n",
      "today european research round\n",
      "european research round\n",
      "research round\n",
      "european research\n",
      "asset com gap\n",
      "\n",
      "\n",
      "Cluster 96\n",
      "\n",
      "nikkei\n",
      "tomisawa\n",
      "ayai tomisawa\n",
      "ayai\n",
      "tomisawa edit\n",
      "ayai tomisawa edit\n",
      "jpx nikkei\n",
      "jpx\n",
      "jpx nikkei index\n",
      "nikkei index\n",
      "\n",
      "\n",
      "Cluster 97\n",
      "\n",
      "yen\n",
      "financi group\n",
      "nikkei\n",
      "financi\n",
      "mitsubishi\n",
      "japan\n",
      "group\n",
      "share\n",
      "corp\n",
      "ufj\n",
      "\n",
      "\n",
      "Cluster 98\n",
      "\n",
      "dollar\n",
      "yield\n",
      "european\n",
      "live market blog european\n",
      "blog european uk\n",
      "blog european\n",
      "blog european uk stock\n",
      "market blog european uk\n",
      "market blog european\n",
      "uk stock realtim verb\n",
      "\n",
      "\n",
      "Cluster 99\n",
      "\n",
      "compani coverag\n",
      "sourc text\n",
      "text eikon compani\n",
      "sourc text eikon compani\n",
      "text eikon compani coverag\n",
      "sourc text eikon\n",
      "text eikon\n",
      "eikon compani coverag\n",
      "eikon compani\n",
      "text\n",
      "\n",
      "\n",
      "Cluster 100\n",
      "\n",
      "price\n",
      "commod\n",
      "tonn\n",
      "coal\n",
      "copper\n",
      "metal\n",
      "ore\n",
      "iron ore\n",
      "iron\n",
      "glencor\n",
      "\n",
      "\n",
      "Cluster 101\n",
      "\n",
      "nikkei\n",
      "nikkei shed\n",
      "benchmark nikkei shed\n",
      "topx lost\n",
      "topix topx lost\n",
      "broader topix topx lost\n",
      "elain lie\n",
      "broader topix topx\n",
      "topix topx\n",
      "topx\n",
      "\n",
      "\n",
      "Cluster 102\n",
      "\n",
      "plc\n",
      "britain\n",
      "uk\n",
      "news\n",
      "british\n",
      "com\n",
      "brexit\n",
      "financi\n",
      "european\n",
      "com news com\n",
      "\n",
      "\n",
      "Cluster 103\n",
      "\n",
      "week\n",
      "dollar\n",
      "long\n",
      "week long\n",
      "long short net\n",
      "short net\n",
      "week long short net\n",
      "week long short\n",
      "dollar long\n",
      "long posit\n",
      "\n",
      "\n",
      "Cluster 104\n",
      "\n",
      "yr\n",
      "euro\n",
      "yld\n",
      "usd\n",
      "chg\n",
      "gold\n",
      "compani\n",
      "usd jpi yr\n",
      "spot gold crude\n",
      "gold crude\n",
      "\n",
      "\n",
      "Cluster 105\n",
      "\n",
      "oil\n",
      "opec\n",
      "barrel\n",
      "price\n",
      "crude\n",
      "oil price\n",
      "product\n",
      "saudi\n",
      "output\n",
      "energi\n",
      "\n",
      "\n",
      "Cluster 106\n",
      "\n",
      "ftse\n",
      "britain ftse ftse\n",
      "germani dax gdaxi\n",
      "dax gdaxi\n",
      "gdaxi\n",
      "fchi\n",
      "cac fchi\n",
      "franc cac fchi\n",
      "ftse ftse\n",
      "ftse ftse germani\n",
      "\n",
      "\n",
      "Cluster 107\n",
      "\n",
      "sale\n",
      "retail\n",
      "consum\n",
      "spend\n",
      "month\n",
      "retail sale\n",
      "survey\n",
      "confid\n",
      "figur\n",
      "growth\n",
      "\n",
      "\n",
      "Cluster 108\n",
      "\n",
      "dollar\n",
      "currenc\n",
      "yen\n",
      "euro\n",
      "trump\n",
      "rate\n",
      "greenback\n",
      "trade\n",
      "basket\n",
      "week\n",
      "\n",
      "\n",
      "Cluster 109\n",
      "\n",
      "imf\n",
      "lagard\n",
      "countri\n",
      "growth\n",
      "fund\n",
      "economi\n",
      "intern\n",
      "world\n",
      "global\n",
      "financi\n",
      "\n",
      "\n",
      "Cluster 110\n",
      "\n",
      "pa\n",
      "bank\n",
      "european\n",
      "share\n",
      "ftse\n",
      "stock\n",
      "gain\n",
      "credit\n",
      "european share\n",
      "franc\n",
      "\n",
      "\n",
      "Cluster 111\n",
      "\n",
      "composit ad chuck\n",
      "composit ad chuck mikolajczak\n",
      "nasdaq composit ad chuck\n",
      "ad chuck\n",
      "ad chuck mikolajczak\n",
      "ad chuck mikolajczak edit\n",
      "chuck mikolajczak edit\n",
      "mikolajczak edit\n",
      "chuck mikolajczak\n",
      "mikolajczak\n",
      "\n",
      "\n",
      "Cluster 112\n",
      "\n",
      "develop\n",
      "countri\n",
      "world\n",
      "world bank\n",
      "poverti\n",
      "bank\n",
      "develop countri\n",
      "aid\n",
      "africa\n",
      "poor\n",
      "\n",
      "\n",
      "Cluster 113\n",
      "\n",
      "futur\n",
      "european stock futur\n",
      "futur euro stoxx\n",
      "european stock futur point\n",
      "futur euro\n",
      "stock futur point\n",
      "futur point\n",
      "dax franc\n",
      "dax franc cac\n",
      "germani dax franc\n",
      "\n",
      "\n",
      "Cluster 114\n",
      "\n",
      "econom\n",
      "social\n",
      "polit\n",
      "inequ\n",
      "world\n",
      "wealth\n",
      "capit\n",
      "work\n",
      "need\n",
      "societi\n",
      "\n",
      "\n",
      "Cluster 115\n",
      "\n",
      "jone industri lost nasdaq\n",
      "industri lost nasdaq\n",
      "industri lost nasdaq composit\n",
      "dow jone industri lost\n",
      "jone industri lost\n",
      "industri lost\n",
      "lost nasdaq composit\n",
      "lost nasdaq\n",
      "lost nasdaq composit drop\n",
      "nasdaq composit drop\n",
      "\n",
      "\n",
      "Cluster 116\n",
      "\n",
      "fund\n",
      "manag\n",
      "hedg\n",
      "hedg fund\n",
      "asset\n",
      "invest\n",
      "etf\n",
      "billion\n",
      "investor\n",
      "fund manag\n",
      "\n",
      "\n",
      "Cluster 117\n",
      "\n",
      "renminbi\n",
      "rmb\n",
      "offshor\n",
      "china\n",
      "currenc\n",
      "onshor\n",
      "dollar\n",
      "cnh\n",
      "trade\n",
      "offshor renminbi\n",
      "\n",
      "\n",
      "Cluster 118\n",
      "\n",
      "nasdaq composit yashaswini swamynathan\n",
      "composit yashaswini swamynathan\n",
      "nasdaq composit yashaswini\n",
      "composit yashaswini\n",
      "composit yashaswini swamynathan bengaluru\n",
      "stock open\n",
      "yashaswini swamynathan bengaluru\n",
      "swamynathan bengaluru\n",
      "industri nasdaq composit yashaswini\n",
      "yashaswini\n",
      "\n",
      "\n",
      "Cluster 119\n",
      "\n",
      "trade\n",
      "exchang\n",
      "regul\n",
      "mr\n",
      "trader\n",
      "clear\n",
      "london\n",
      "rule\n",
      "sarao\n",
      "execut\n",
      "\n",
      "\n",
      "Cluster 120\n",
      "\n",
      "china\n",
      "yuan\n",
      "bln\n",
      "ipo\n",
      "profit\n",
      "bln yuan\n",
      "ipo time oversubscrib\n",
      "ipo time\n",
      "mln\n",
      "time oversubscrib\n",
      "\n",
      "\n",
      "Cluster 121\n",
      "\n",
      "obama\n",
      "republican\n",
      "presid\n",
      "democrat\n",
      "hous\n",
      "senat\n",
      "tax\n",
      "american\n",
      "congress\n",
      "budget\n",
      "\n",
      "\n",
      "Cluster 122\n",
      "\n",
      "china\n",
      "csi\n",
      "shanghai\n",
      "index\n",
      "shanghai composit index\n",
      "composit index\n",
      "shanghai composit\n",
      "chip\n",
      "blue chip\n",
      "blue\n",
      "\n",
      "\n",
      "Cluster 123\n",
      "\n",
      "gijsel\n",
      "european\n",
      "forti\n",
      "philipp gijsel\n",
      "pariba forti\n",
      "bnp pariba forti\n",
      "philipp\n",
      "research bnp pariba forti\n",
      "philipp gijsel head research\n",
      "philipp gijsel head\n",
      "\n",
      "\n",
      "Cluster 124\n",
      "\n",
      "saudi\n",
      "egypt\n",
      "qatar\n",
      "dubai\n",
      "arabia\n",
      "saudi arabia\n",
      "kuwait\n",
      "iran\n",
      "oil\n",
      "uae\n",
      "\n",
      "\n",
      "Cluster 125\n",
      "\n",
      "familiar\n",
      "matter\n",
      "familiar matter\n",
      "sourc\n",
      "report cite\n",
      "report\n",
      "cite\n",
      "sourc familiar\n",
      "relat stori report media\n",
      "corpor financ relat stori\n",
      "\n",
      "\n",
      "Cluster 126\n",
      "\n",
      "yield\n",
      "crude\n",
      "dollar\n",
      "stock\n",
      "price\n",
      "new york\n",
      "york\n",
      "treasuri\n",
      "new\n",
      "oil\n",
      "\n",
      "\n",
      "Cluster 127\n",
      "\n",
      "ftse\n",
      "european\n",
      "share\n",
      "frankfurt dax\n",
      "london ftse ftse\n",
      "london ftse\n",
      "pari cac\n",
      "frankfurt dax gdaxi\n",
      "pari cac fchi\n",
      "frankfurt\n",
      "\n",
      "\n",
      "Cluster 128\n",
      "\n",
      "inflat\n",
      "price\n",
      "rate\n",
      "food\n",
      "cost\n",
      "month\n",
      "cpi\n",
      "consum\n",
      "bank\n",
      "increas\n",
      "\n",
      "\n",
      "Cluster 129\n",
      "\n",
      "open window\n",
      "window\n",
      "open\n",
      "window share facebook open\n",
      "share facebook open\n",
      "open window emailshar\n",
      "window emailshar\n",
      "window share facebook\n",
      "share facebook open window\n",
      "open window share\n",
      "\n",
      "\n",
      "Cluster 130\n",
      "\n",
      "nlde\n",
      "nl\n",
      "pa\n",
      "euro\n",
      "nn\n",
      "rr\n",
      "yt\n",
      "yt rr\n",
      "usd\n",
      "chg\n",
      "\n",
      "\n",
      "Cluster 131\n",
      "\n",
      "work\n",
      "like\n",
      "make\n",
      "money\n",
      "thing\n",
      "think\n",
      "way\n",
      "say\n",
      "world\n",
      "want\n",
      "\n",
      "\n",
      "Cluster 132\n",
      "\n",
      "gold\n",
      "ounc\n",
      "gold price\n",
      "metal\n",
      "price\n",
      "precious metal\n",
      "dollar\n",
      "rate\n",
      "precious\n",
      "fed\n",
      "\n",
      "\n",
      "Cluster 133\n",
      "\n",
      "yen\n",
      "financi group\n",
      "nikkei\n",
      "financi\n",
      "group\n",
      "mitsubishi\n",
      "ufj\n",
      "mitsubishi ufj\n",
      "japan\n",
      "ufj financi\n",
      "\n",
      "\n",
      "Cluster 134\n",
      "\n",
      "nikkei\n",
      "topix topx\n",
      "topx\n",
      "nikkei end\n",
      "broader topix topx\n",
      "topix\n",
      "broader topix\n",
      "end morn\n",
      "benchmark nikkei\n",
      "broader\n",
      "\n",
      "\n",
      "Cluster 135\n",
      "\n",
      "manufactur\n",
      "survey\n",
      "sector\n",
      "month\n",
      "output\n",
      "growth\n",
      "uk\n",
      "pmi\n",
      "servic\n",
      "order\n",
      "\n",
      "\n",
      "Cluster 136\n",
      "\n",
      "ftse\n",
      "price\n",
      "stock\n",
      "oil\n",
      "analyst\n",
      "britain\n",
      "miner\n",
      "share\n",
      "compani\n",
      "commod\n",
      "\n",
      "\n",
      "Cluster 137\n",
      "\n",
      "earn\n",
      "quarter\n",
      "compani\n",
      "share\n",
      "expect\n",
      "stock\n",
      "revenu\n",
      "dow\n",
      "nasdaq\n",
      "sale\n",
      "\n",
      "\n",
      "Cluster 138\n",
      "\n",
      "nasdaq\n",
      "high low\n",
      "ratio\n",
      "advanc\n",
      "nasdaq composit\n",
      "low\n",
      "high\n",
      "composit\n",
      "dow\n",
      "issu\n",
      "\n",
      "\n",
      "Cluster 139\n",
      "\n",
      "sterl\n",
      "pound\n",
      "brexit\n",
      "uk\n",
      "currenc\n",
      "dollar\n",
      "vote\n",
      "eu\n",
      "bank\n",
      "gilt\n",
      "\n",
      "\n",
      "Cluster 140\n",
      "\n",
      "yield\n",
      "dollar\n",
      "crude\n",
      "barrel\n",
      "price\n",
      "oil\n",
      "treasuri\n",
      "euro\n",
      "brent\n",
      "stock\n",
      "\n",
      "\n",
      "Cluster 141\n",
      "\n",
      "nikkei\n",
      "jpx\n",
      "jpx nikkei\n",
      "jpx nikkei index\n",
      "nikkei index\n",
      "japanes\n",
      "yen\n",
      "japan\n",
      "topix\n",
      "index\n",
      "\n",
      "\n",
      "Cluster 142\n",
      "\n",
      "cramer\n",
      "mad money\n",
      "mad\n",
      "jim cramer\n",
      "money\n",
      "jim\n",
      "stock\n",
      "mad money host\n",
      "money host\n",
      "host\n",
      "\n",
      "\n",
      "Cluster 143\n",
      "\n",
      "emerg\n",
      "chg\n",
      "currenc\n",
      "turkey\n",
      "south africa\n",
      "graphic\n",
      "czech\n",
      "poland\n",
      "south\n",
      "hungari\n",
      "\n",
      "\n",
      "Cluster 144\n",
      "\n",
      "yuan\n",
      "shanghai\n",
      "shenzhen\n",
      "china\n",
      "volum\n",
      "list compani shanghai\n",
      "compani shanghai\n",
      "largest list compani shanghai\n",
      "csi largest list compani\n",
      "csi largest list\n",
      "\n",
      "\n",
      "Cluster 145\n",
      "\n",
      "dollar\n",
      "yield\n",
      "euro\n",
      "week\n",
      "oil\n",
      "bond\n",
      "rate\n",
      "currenc\n",
      "month\n",
      "asia\n",
      "\n",
      "\n",
      "Cluster 146\n",
      "\n",
      "european share\n",
      "european\n",
      "share\n",
      "ftseurofirst fteu european\n",
      "fteu european\n",
      "ftseurofirst fteu european share\n",
      "fteu european share\n",
      "fteu\n",
      "ftseurofirst fteu\n",
      "ftseurofirst\n",
      "\n",
      "\n",
      "Cluster 147\n",
      "\n",
      "fed\n",
      "rate\n",
      "hike\n",
      "inflat\n",
      "yellen\n",
      "polici\n",
      "feder\n",
      "rate hike\n",
      "rais rate\n",
      "rais\n",
      "\n",
      "\n",
      "Cluster 148\n",
      "\n",
      "bank\n",
      "european\n",
      "vx\n",
      "european share\n",
      "share\n",
      "ub\n",
      "rbs\n",
      "fteu\n",
      "ftseurofirst fteu\n",
      "ftseurofirst\n",
      "\n",
      "\n",
      "Cluster 149\n",
      "\n",
      "tax\n",
      "incom\n",
      "pay\n",
      "govern\n",
      "cut\n",
      "corpor\n",
      "rate\n",
      "tax rate\n",
      "corpor tax\n",
      "incom tax\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for cluster in range(150):\n",
    "    features = order_centroids[cluster,0:10]\n",
    "    print('Cluster {}\\n'.format(cluster))\n",
    "    for feature in features:\n",
    "        print(feature_names[feature])\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Move data to DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "km_150 = pd.DataFrame({'original_km_150':k_dict[150][0], 'new_km_150':clusters})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 84155 entries, 0 to 84154\n",
      "Data columns (total 2 columns):\n",
      "new_km_150         84155 non-null int32\n",
      "original_km_150    84155 non-null int32\n",
      "dtypes: int32(2)\n",
      "memory usage: 657.5 KB\n"
     ]
    }
   ],
   "source": [
    "km_150.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>new_km_150</th>\n",
       "      <th>original_km_150</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>32985</th>\n",
       "      <td>146</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42236</th>\n",
       "      <td>58</td>\n",
       "      <td>73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73447</th>\n",
       "      <td>147</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38327</th>\n",
       "      <td>110</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5859</th>\n",
       "      <td>90</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       new_km_150  original_km_150\n",
       "32985         146               33\n",
       "42236          58               73\n",
       "73447         147              140\n",
       "38327         110               65\n",
       "5859           90                2"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "km_150.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combine with original article data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_km_150 = pd.concat([snowball_df, km_150], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime</th>\n",
       "      <th>source</th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>text_snowball</th>\n",
       "      <th>new_km_150</th>\n",
       "      <th>original_km_150</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2008-05-23 23:01:00</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>$135 and rising ... has cheap oil gone for ever?</td>\n",
       "      <td>https://www.theguardian.com/business/2008/may/...</td>\n",
       "      <td>fresh maul byelect gordon brown met two leader...</td>\n",
       "      <td>105</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2007-08-15 15:41:12</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>$2 pound sails into the sunset</td>\n",
       "      <td>https://www.theguardian.com/business/2007/aug/...</td>\n",
       "      <td>take kid florida next week yet collect dollar ...</td>\n",
       "      <td>139</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2010-12-20 17:58:56</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>$2tn debt crisis threatens to bring down 100 U...</td>\n",
       "      <td>https://www.theguardian.com/business/2010/dec/...</td>\n",
       "      <td>more american citi could go bust next debt cri...</td>\n",
       "      <td>92</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2005-01-28 15:07:10</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>'A dream deal'</td>\n",
       "      <td>https://www.theguardian.com/business/2005/jan/...</td>\n",
       "      <td>look around hous bound good made procter gambl...</td>\n",
       "      <td>23</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2016-12-06 18:28:50</td>\n",
       "      <td>Guardian</td>\n",
       "      <td>'A landscape of exhaustion and moral decay' – ...</td>\n",
       "      <td>https://www.theguardian.com/business/2016/dec/...</td>\n",
       "      <td>when mark carney insist speech liverpool john ...</td>\n",
       "      <td>131</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              datetime    source  \\\n",
       "0  2008-05-23 23:01:00  Guardian   \n",
       "1  2007-08-15 15:41:12  Guardian   \n",
       "2  2010-12-20 17:58:56  Guardian   \n",
       "3  2005-01-28 15:07:10  Guardian   \n",
       "4  2016-12-06 18:28:50  Guardian   \n",
       "\n",
       "                                               title  \\\n",
       "0   $135 and rising ... has cheap oil gone for ever?   \n",
       "1                     $2 pound sails into the sunset   \n",
       "2  $2tn debt crisis threatens to bring down 100 U...   \n",
       "3                                     'A dream deal'   \n",
       "4  'A landscape of exhaustion and moral decay' – ...   \n",
       "\n",
       "                                                 url  \\\n",
       "0  https://www.theguardian.com/business/2008/may/...   \n",
       "1  https://www.theguardian.com/business/2007/aug/...   \n",
       "2  https://www.theguardian.com/business/2010/dec/...   \n",
       "3  https://www.theguardian.com/business/2005/jan/...   \n",
       "4  https://www.theguardian.com/business/2016/dec/...   \n",
       "\n",
       "                                       text_snowball  new_km_150  \\\n",
       "0  fresh maul byelect gordon brown met two leader...         105   \n",
       "1  take kid florida next week yet collect dollar ...         139   \n",
       "2  more american citi could go bust next debt cri...          92   \n",
       "3  look around hous bound good made procter gambl...          23   \n",
       "4  when mark carney insist speech liverpool john ...         131   \n",
       "\n",
       "   original_km_150  \n",
       "0               23  \n",
       "1                0  \n",
       "2               62  \n",
       "3              101  \n",
       "4              129  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_km_150.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_km_150.to_csv('km.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_km_150.to_pickle('km.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "km_150.to_csv('km_150_clusters.csv', encoding='utf-8')\n",
    "km_150.to_pickle('km_150_clusters.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}