DataBase/2_Centre_Left/goodreads/quotes_centre_left.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='Centre-Left-Scraping-and-cleaning'></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from tqdm import tqdm_notebook\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "#from langdetect import detect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.options.display.max_rows=1000\n",
    "pd.options.display.max_columns=1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.set_printoptions(threshold=np.inf)\n",
    "pd.options.display.max_colwidth = 1000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- List of the people I considered for this reasearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "lista = ['Bill Moyers', \n",
    "         'Anthony Giddens', \n",
    "         'Paul Krugman',\n",
    "         'Benjamin Barber', \n",
    "         'Joseph E. Stiglitz',\n",
    "         'Paul Mason',\n",
    "         'Owen Jones', \n",
    "         'Jesse Jackson',\n",
    "         'George Soros', \n",
    "         'Donna Brazile', \n",
    "         'James Carville',\n",
    "         'Robert Reich',\n",
    "         'Tony Judt', \n",
    "         'Jane Jacobs', \n",
    "         'Lawrence Lessig',\n",
    "         'Simon Mckay',\n",
    "         'Glenn Greenwald',\n",
    "         'Ezra Klein',\n",
    "         'Nicholas Kristof', \n",
    "         'Juan Williams',\n",
    "         'Al Gore',\n",
    "         'E. J. Dionne Jr.',\n",
    "         'Alan Bennett',\n",
    "         'Polly Toynbee'\n",
    "         \n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Getting Data from Goodreads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#List of name in the format for Goodreads website\n",
    "new_lista = []\n",
    "temp_lista = []\n",
    "\n",
    "for name in lista:\n",
    "    temp_lista.append(name.replace('.',''))\n",
    "\n",
    "\n",
    "for n in temp_lista:\n",
    "    new_lista.append(n.replace(' ','+'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Bill+Moyers',\n",
       " 'Anthony+Giddens',\n",
       " 'Paul+Krugman',\n",
       " 'Benjamin+Barber',\n",
       " 'Joseph+E+Stiglitz',\n",
       " 'Paul+Mason',\n",
       " 'Owen+Jones',\n",
       " 'Jesse+Jackson',\n",
       " 'George+Soros',\n",
       " 'Donna+Brazile',\n",
       " 'James+Carville',\n",
       " 'Robert+Reich',\n",
       " 'Tony+Judt',\n",
       " 'Jane+Jacobs',\n",
       " 'Lawrence+Lessig',\n",
       " 'Simon+Mckay',\n",
       " 'Glenn+Greenwald',\n",
       " 'Ezra+Klein',\n",
       " 'Nicholas+Kristof',\n",
       " 'Juan+Williams',\n",
       " 'Al+Gore',\n",
       " 'E+J+Dionne+Jr',\n",
       " 'Alan+Bennett',\n",
       " 'Polly+Toynbee']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_lista"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating a list for the Brainquote format and for a Goodreads Double check\n",
    "brain_lista = []\n",
    "for n in lista:\n",
    "    brain_lista.append(n.lower().replace(' ','_'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b340ae678f1840b499b57cdca377b3b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c705f36a9ea94970b593e6ac1017de61",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0cf880a0b6b0477bafbb17e2f1473c27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fefbf7a949fe49b2b3dd4fb29d856a21",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a121a8c0d87d47488b80670eae3ccc7c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "104b95f90a60410886d8b2d53e9e5079",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c81db7a9d08d4c738fcc192091fe1df1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8346fdabc3a24d1eaf6d12d733adfeeb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7bef0d4fd5ac4e2786be6220bd8250f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24fb895682d44a31b291c0431212541c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=12), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2562b6ed62b44eeba89d79336720f96",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=17), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0436e41f755442a3a76d6bf48bfa5a39",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=6), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f297b230983a43ad8562cb7f3ba91c9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d7d7cf9e74a46b6b2976b3743e3b7a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=12), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f8e3a8714ce649e98182f34d5aac966a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e3b1dafc8574ef8a48937fcee4e7c0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=9), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91104de797e845cf8ba6e6e83592c6c5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=9), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "#Scraping all the existing quotes for the person I need from Goodreads\n",
    "quotes = []\n",
    "author = []\n",
    "\n",
    "for name in new_lista:\n",
    "    second_temp_quote = []\n",
    "    second_temp_author = []\n",
    "    r = requests.get(\"https://goodquotesapi.herokuapp.com/author/{}\".format(name))\n",
    "    data = r.json()\n",
    "    if len(data) < 1:\n",
    "        pass\n",
    "    else:\n",
    "        for page in tqdm_notebook(range(1,data['total_pages']+1)):\n",
    "            new_r = requests.get(\"https://goodquotesapi.herokuapp.com/author/{}?page={}\".format(name,page))\n",
    "            new_data = new_r.json()\n",
    "            first_temp_quote = []\n",
    "            first_temp_author = []\n",
    "\n",
    "            for i in range(20):\n",
    "                temp_quote = []\n",
    "                temp_author = []\n",
    "\n",
    "                try: \n",
    "                    temp_quote.append(new_data['quotes'][i]['quote'])\n",
    "                    temp_author.append(new_data['quotes'][0]['author'])\n",
    "\n",
    "                except:\n",
    "                    temp_quote.append(np.nan)\n",
    "                    temp_author.append(np.nan)\n",
    "            first_temp_author.append(temp_author)\n",
    "            first_temp_quote.append(temp_quote)\n",
    "            #second_temp_quote.append(first_temp_quote)\n",
    "            #second_temp_author.append(first_temp_author)\n",
    "        quotes.append(first_temp_quote)\n",
    "        author.append(first_temp_author)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating the goodreads Dataframe\n",
    "goodreads = pd.DataFrame({\n",
    "    'Author': [item for l in author for sublist in l  for item in sublist], \n",
    "    'Quotes': [item for l in quotes for sublist in l  for item in sublist],\n",
    "    'source': 'goodreads'\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17, 3)"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "goodreads.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Leaving every cell in a string\n",
    "goodreads.Author = goodreads.Author.apply(lambda x : x[0])\n",
    "goodreads.Quotes = goodreads.Quotes.apply(lambda x : x[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Deleting all the rows where the Author is not in the list\n",
    "goodreads = goodreads[goodreads['Author'].isin(lista)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Getting data from Wikiquote // I took the data in a different Jupyter Notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Wiki scraping / cleaning](../WikiQuote/Centre_Left.ipynb#Wiki-Scraping-and-cleaning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading the previous dataframe scraped from Wikiquotes\n",
    "wikiquotes = pd.read_csv('../WikiQuote/centre_left.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Paul Krugman               98\n",
       "George Soros               80\n",
       "Lawrence Lessig            56\n",
       "Tony Judt                  33\n",
       "Bill Moyers                27\n",
       "George Will                26\n",
       "Jane Jacobs                25\n",
       "Joseph Stiglitz            23\n",
       "James Carville             22\n",
       "Benjamin Barber            15\n",
       "Anthony Giddens            13\n",
       "Paul Mason (journalist)    11\n",
       "Jesse Jackson               9\n",
       "Robert Reich                4\n",
       "Donna Brazile               3\n",
       "Owen Jones (writer)         1\n",
       "Name: Author, dtype: int64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Checking the author\n",
    "wikiquotes.Author.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "#removing few Authors\n",
    "wiki_mask = (wikiquotes['Author'] != 'Owen Jones (writer)') & (wikiquotes['Author'] != 'Paul Mason (journalist)') & (wikiquotes['Author'] != 'George Will')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "wikiquotes=wikiquotes[wiki_mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Paul Krugman       98\n",
       "George Soros       80\n",
       "Lawrence Lessig    56\n",
       "Tony Judt          33\n",
       "Bill Moyers        27\n",
       "Jane Jacobs        25\n",
       "Joseph Stiglitz    23\n",
       "James Carville     22\n",
       "Benjamin Barber    15\n",
       "Anthony Giddens    13\n",
       "Jesse Jackson       9\n",
       "Robert Reich        4\n",
       "Donna Brazile       3\n",
       "Name: Author, dtype: int64"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wikiquotes.Author.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "wikiquotes = wikiquotes.drop('Political Orientation', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "wikiquotes['source'] = 'wikiquotes'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " - Getting data from Brainquote  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Scraping the quotes from Brainquote website\n",
    "q = []\n",
    "a = []\n",
    "\n",
    "for name in brain_lista:\n",
    "    temp_quote = []\n",
    "    temp_author = []\n",
    "    r = requests.get(\"https://www.brainyquote.com/authors/{}\".format(name))\n",
    "    soup = BeautifulSoup(r.text,'html.parser')\n",
    "    for each in soup.find_all('a', attrs={'title':\"view quote\"}):\n",
    "        temp_quote.append(each.text)\n",
    "        #if len(temp_quote) > 0:\n",
    "        temp_author.append(name)\n",
    "    q.append(temp_quote)\n",
    "    a.append(temp_author)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating the Brainquote dataframe\n",
    "brain = pd.DataFrame({'Author': [item for l in a for item in l],\n",
    "                      'Quotes': [item for l in q for item in l],\n",
    "                      'source': 'brainquote'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Removing '_' from the name and capitalize Name and Surname\n",
    "brain.Author = brain.Author.apply(lambda x : x.replace('_',' ').title())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:2: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "#Putting everything together\n",
    "Centre_left_quotes = pd.concat([goodreads, brain, wikiquotes, d_goodreads])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "Centre_left_quotes = Centre_left_quotes.drop_duplicates(subset='Quotes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "Centre_left_quotes['language'] = Centre_left_quotes.Quotes.apply(lambda x : detect(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Deleting all the not english quotes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "en    2048\n",
       "pt      26\n",
       "es      19\n",
       "it      19\n",
       "cs      14\n",
       "bg       9\n",
       "ar       8\n",
       "de       6\n",
       "fi       4\n",
       "nl       3\n",
       "fr       3\n",
       "no       1\n",
       "da       1\n",
       "tl       1\n",
       "pl       1\n",
       "Name: language, dtype: int64"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Centre_left_quotes.language.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "Centre_left_quotes = Centre_left_quotes[Centre_left_quotes.language == 'en']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Tony Judt             334\n",
       "Glenn Greenwald       243\n",
       "Alan Bennett          173\n",
       "Paul Krugman          165\n",
       "George Soros          142\n",
       "Al Gore               136\n",
       "Jane Jacobs           131\n",
       "Joseph E. Stiglitz    127\n",
       "Lawrence Lessig       119\n",
       "Donna Brazile         108\n",
       "Jesse Jackson          69\n",
       "James Carville         67\n",
       "Bill Moyers            64\n",
       "Robert Reich           30\n",
       "Juan Williams          26\n",
       "Nicholas Kristof       26\n",
       "Polly Toynbee          24\n",
       "Joseph Stiglitz        23\n",
       "Benjamin Barber        15\n",
       "Ezra Klein             13\n",
       "Anthony Giddens        13\n",
       "Name: Author, dtype: int64"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Centre_left_quotes.Author.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Correcting the string name of same authors\n",
    "Centre_left_quotes.loc[Centre_left_quotes['Author'].str.contains('Stiglitz'), 'Author'] = 'Joseph E. Stiglitz'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Checking for quote in the quotes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "dup = {}\n",
    "#count = 0\n",
    "for name in Centre_left_quotes.Author.unique():\n",
    "    conta = []\n",
    "    for text in Centre_left_quotes[Centre_left_quotes.Author==name].Quotes :\n",
    "        count = 0\n",
    "        for i in range(len(Centre_left_quotes[Centre_left_quotes.Author==name])):\n",
    "            if text in Centre_left_quotes[Centre_left_quotes.Author==name].iloc[i,1]:\n",
    "                count += 1\n",
    "\n",
    "\n",
    "        conta.append(count)\n",
    "    dup[name] = conta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "Centre_left_quotes['Counter'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bill Moyers\n",
      "Paul Krugman\n",
      "Joseph E. Stiglitz\n",
      "Jesse Jackson\n",
      "George Soros\n",
      "Donna Brazile\n",
      "James Carville\n",
      "Tony Judt\n",
      "Jane Jacobs\n",
      "Lawrence Lessig\n",
      "Glenn Greenwald\n",
      "Al Gore\n",
      "Alan Bennett\n",
      "Robert Reich\n",
      "Ezra Klein\n",
      "Nicholas Kristof\n",
      "Juan Williams\n",
      "Polly Toynbee\n",
      "Anthony Giddens\n",
      "Benjamin Barber\n"
     ]
    }
   ],
   "source": [
    "for name in Centre_left_quotes.Author.unique():\n",
    "    print(name)\n",
    "    Centre_left_quotes.loc[Centre_left_quotes.Author==name, 'Counter'] = dup[name]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    1925\n",
       "2     106\n",
       "3      11\n",
       "4       5\n",
       "5       1\n",
       "Name: Counter, dtype: int64"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Centre_left_quotes.Counter.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "Centre_left_quotes=Centre_left_quotes[Centre_left_quotes.Counter==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "Centre_left_quotes.drop(['source', 'language', 'Counter'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Adding the label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "Centre_left_quotes['Political Orientation'] = 'Progressive'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "rec = pd.read_csv('test_centre_left.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "CL = pd.concat([rec, Centre_left_quotes])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Tony Judt             300\n",
       "Glenn Greenwald       214\n",
       "Paul Krugman          165\n",
       "Alan Bennett          164\n",
       "Joseph E. Stiglitz    140\n",
       "George Soros          136\n",
       "Al Gore               136\n",
       "Jane Jacobs           127\n",
       "Donna Brazile         103\n",
       "Lawrence Lessig        96\n",
       "Jesse Jackson          69\n",
       "James Carville         66\n",
       "Bill Moyers            63\n",
       "Robert Reich           30\n",
       "Nicholas Kristof       26\n",
       "Juan Williams          25\n",
       "Polly Toynbee          24\n",
       "Benjamin Barber        15\n",
       "Ezra Klein             13\n",
       "Anthony Giddens        13\n",
       "Name: Author, dtype: int64"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CL.Author.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Selecting 4 Authors for a total of bit more than 100 quotes for the final test df\n",
    "list_test = ['Ezra Klein', 'Nicholas Kristof', 'Robert Reich', 'Polly Toynbee']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating the Dataframe for the test\n",
    "CL_test = CL[CL['Author'].isin(list_test)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Robert Reich        30\n",
       "Nicholas Kristof    26\n",
       "Polly Toynbee       24\n",
       "Ezra Klein          13\n",
       "Name: Author, dtype: int64"
      ]
     },
     "execution_count": 223,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CL_test.Author.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "#removing the 4 authors from the main database\n",
    "Centre_left_quotes = CL[~CL['Author'].isin(list_test)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Tony Judt             300\n",
       "Glenn Greenwald       214\n",
       "Paul Krugman          165\n",
       "Alan Bennett          164\n",
       "Joseph E. Stiglitz    140\n",
       "George Soros          136\n",
       "Al Gore               136\n",
       "Jane Jacobs           127\n",
       "Donna Brazile         103\n",
       "Lawrence Lessig        96\n",
       "Jesse Jackson          69\n",
       "James Carville         66\n",
       "Bill Moyers            63\n",
       "Juan Williams          25\n",
       "Benjamin Barber        15\n",
       "Anthony Giddens        13\n",
       "Name: Author, dtype: int64"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#checking everyhting is fine\n",
    "Centre_left_quotes.Author.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#csv file for the train\n",
    "Centre_left_quotes.to_csv('train_centre_left.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "#csv file for the test\n",
    "CL_test.to_csv('test_centre_left.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "CL.to_csv('centre_left_quotes.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Robert Reich        30\n",
       "Nicholas Kristof    26\n",
       "Polly Toynbee       24\n",
       "Ezra Klein          13\n",
       "Name: Author, dtype: int64"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CL_test.Author.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**SECOND PART**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading files\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "CL_train = pd.read_csv('train_centre_left.csv')\n",
    "CL_test = pd.read_csv('test_centre_left.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "CL_train.drop(CL_train.index[[1223, 91, 93, 278, 1232, 306, 338, 466, \n",
    "                                                904, 951, 1119, 1052, 1157, 1135, 1498, \n",
    "                                                228, 278,223, 238, 247, 69, 404, 368, \n",
    "                                                341, 329, 453, 476,218, 1202, 1129, 1160,\n",
    "                                                584,1610, 1688, 628, 950, 936, 949, 818,  \n",
    "                                                1058, 1090, 1031, 1024, 72, 653, 853, 844,\n",
    "                                               847, 767]], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Looking for quotes about the authors and not from them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_autoref(df):\n",
    "    autoref = {}\n",
    "    for name in df.Author.unique():\n",
    "        conta = []\n",
    "\n",
    "        for text in df[df.Author==name].Quotes :\n",
    "            count = 0\n",
    "\n",
    "            if name in text or name.split(' ')[-1] in text:\n",
    "                    count += 1\n",
    "\n",
    "            conta.append(count)\n",
    "        autoref[name] = conta \n",
    "    return autoref"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "autoref = find_autoref(CL_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Juan Williams  done\n",
      "Bill Moyers  done\n",
      "Paul Krugman  done\n",
      "Joseph E. Stiglitz  done\n",
      "Jesse Jackson  done\n",
      "George Soros  done\n",
      "Donna Brazile  done\n",
      "James Carville  done\n",
      "Tony Judt  done\n",
      "Jane Jacobs  done\n",
      "Lawrence Lessig  done\n",
      "Glenn Greenwald  done\n",
      "Al Gore  done\n",
      "Alan Bennett  done\n",
      "Anthony Giddens  done\n",
      "Benjamin Barber  done\n"
     ]
    }
   ],
   "source": [
    "CL_train['auto'] = 0\n",
    "for name in CL_train.Author.unique():\n",
    "    print(name, ' done')\n",
    "    CL_train.loc[CL_train.Author==name, 'auto'] = autoref[name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "CL_train = CL_train[CL_train.auto==0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Checking for quotes in the quote with tokenize quotes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "from textacy.preprocess import preprocess_text\n",
    "import re\n",
    "#import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Author</th>\n",
       "      <th>Quotes</th>\n",
       "      <th>Contain</th>\n",
       "      <th>Political Orientation</th>\n",
       "      <th>auto</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Author, Quotes, Contain, Political Orientation, auto]\n",
       "Index: []"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nans = lambda df: df[df.isnull().any(axis=1)]\n",
    "nans(CL_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "CL_train['texicity'] = CL_train.Quotes.apply(lambda x : preprocess_text(x, \n",
    "                                                                      lowercase=True, \n",
    "                                                                      no_punct=True, \n",
    "                                                                      no_accents=True, \n",
    "                                                                      no_contractions=True, \n",
    "                                                                        no_numbers=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Author</th>\n",
       "      <th>Quotes</th>\n",
       "      <th>Contain</th>\n",
       "      <th>Political Orientation</th>\n",
       "      <th>auto</th>\n",
       "      <th>texicity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Juan Williams</td>\n",
       "      <td>In the 2000 presidential election, Al Gore got more votes than George W. Bush, but still lost the election. The Supreme Court's ruling in Florida gave Bush that pivotal state, and doomed Gore to lose the Electoral College. That odd scenario - where the candidate with the most votes loses - has happened three times in U.S. history.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "      <td>in the number presidential election al gore got more votes than george w bush but still lost the election the supreme court s ruling in florida gave bush that pivotal state and doomed gore to lose the electoral college that odd scenario where the candidate with the most votes loses has happened three times in u s history</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Juan Williams</td>\n",
       "      <td>While Planned Parenthood provides abortions at some of their clinics, it also provides healthcare services for poor women, including checkups, mammograms, cervical cancer screenings and contraceptives.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "      <td>while planned parenthood provides abortions at some of their clinics it also provides healthcare services for poor women including checkups mammograms cervical cancer screenings and contraceptives</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Juan Williams</td>\n",
       "      <td>The 2012 presidential campaign's turn away from the classic, straight-up, American election - where the candidate who gets the most votes nationwide wins - is another sad reminder of the extreme political polarization distorting today's politics. No one talks about a 50-state strategy for winning the presidency these days.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "      <td>the number presidential campaign s turn away from the classic straight up american election where the candidate who gets the most votes nationwide wins is another sad reminder of the extreme political polarization distorting today s politics no one talks about a number state strategy for winning the presidency these days</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Juan Williams</td>\n",
       "      <td>I support gun control. But speaking honestly about the combustible mix of race and guns may be more important to stopping the slaughter in minority communities than any new gun-control laws.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "      <td>i support gun control but speaking honestly about the combustible mix of race and guns may be more important to stopping the slaughter in minority communities than any new gun control laws</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Juan Williams</td>\n",
       "      <td>I pointed out that the Atlanta Olympic bomber - as well as Timothy McVeigh and the people who protest against gay rights at military funerals - are Christians but we journalists don't identify them by their religion.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "      <td>i pointed out that the atlanta olympic bomber as well as timothy mcveigh and the people who protest against gay rights at military funerals are christians but we journalists do not identify them by their religion</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Author  \\\n",
       "0  Juan Williams   \n",
       "1  Juan Williams   \n",
       "2  Juan Williams   \n",
       "3  Juan Williams   \n",
       "4  Juan Williams   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                         Quotes  \\\n",
       "0  In the 2000 presidential election, Al Gore got more votes than George W. Bush, but still lost the election. The Supreme Court's ruling in Florida gave Bush that pivotal state, and doomed Gore to lose the Electoral College. That odd scenario - where the candidate with the most votes loses - has happened three times in U.S. history.   \n",
       "1                                                                                                                                     While Planned Parenthood provides abortions at some of their clinics, it also provides healthcare services for poor women, including checkups, mammograms, cervical cancer screenings and contraceptives.   \n",
       "2          The 2012 presidential campaign's turn away from the classic, straight-up, American election - where the candidate who gets the most votes nationwide wins - is another sad reminder of the extreme political polarization distorting today's politics. No one talks about a 50-state strategy for winning the presidency these days.   \n",
       "3                                                                                                                                                I support gun control. But speaking honestly about the combustible mix of race and guns may be more important to stopping the slaughter in minority communities than any new gun-control laws.   \n",
       "4                                                                                                                      I pointed out that the Atlanta Olympic bomber - as well as Timothy McVeigh and the people who protest against gay rights at military funerals - are Christians but we journalists don't identify them by their religion.   \n",
       "\n",
       "  Contain Political Orientation  auto  \\\n",
       "0      no           Progressive     0   \n",
       "1      no           Progressive     0   \n",
       "2      no           Progressive     0   \n",
       "3      no           Progressive     0   \n",
       "4      no           Progressive     0   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                             texicity  \n",
       "0  in the number presidential election al gore got more votes than george w bush but still lost the election the supreme court s ruling in florida gave bush that pivotal state and doomed gore to lose the electoral college that odd scenario where the candidate with the most votes loses has happened three times in u s history  \n",
       "1                                                                                                                                while planned parenthood provides abortions at some of their clinics it also provides healthcare services for poor women including checkups mammograms cervical cancer screenings and contraceptives  \n",
       "2  the number presidential campaign s turn away from the classic straight up american election where the candidate who gets the most votes nationwide wins is another sad reminder of the extreme political polarization distorting today s politics no one talks about a number state strategy for winning the presidency these days  \n",
       "3                                                                                                                                        i support gun control but speaking honestly about the combustible mix of race and guns may be more important to stopping the slaughter in minority communities than any new gun control laws  \n",
       "4                                                                                                                i pointed out that the atlanta olympic bomber as well as timothy mcveigh and the people who protest against gay rights at military funerals are christians but we journalists do not identify them by their religion  "
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CL_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "stops = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tok_stop(df):\n",
    "    df['texicity'] = df['texicity'].apply(lambda x: x.split())\n",
    "    df['texicity'] = df['texicity'].apply(lambda x : [item for item in x if item not in stops])\n",
    "    df['texicity'] = df['texicity'].apply(lambda x: ' '.join(x))\n",
    "    return df\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "CL_train = tok_stop(CL_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Juan Williams\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "Bill Moyers\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "Paul Krugman\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "140\n",
      "141\n",
      "142\n",
      "143\n",
      "144\n",
      "145\n",
      "Joseph E. Stiglitz\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "Jesse Jackson\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "George Soros\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "Donna Brazile\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "James Carville\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "Tony Judt\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "140\n",
      "141\n",
      "142\n",
      "143\n",
      "144\n",
      "145\n",
      "146\n",
      "147\n",
      "148\n",
      "149\n",
      "150\n",
      "151\n",
      "152\n",
      "153\n",
      "154\n",
      "155\n",
      "156\n",
      "157\n",
      "158\n",
      "159\n",
      "160\n",
      "161\n",
      "162\n",
      "163\n",
      "164\n",
      "165\n",
      "166\n",
      "167\n",
      "168\n",
      "169\n",
      "170\n",
      "171\n",
      "172\n",
      "173\n",
      "174\n",
      "175\n",
      "176\n",
      "177\n",
      "178\n",
      "179\n",
      "180\n",
      "181\n",
      "182\n",
      "183\n",
      "184\n",
      "185\n",
      "186\n",
      "187\n",
      "188\n",
      "189\n",
      "190\n",
      "191\n",
      "192\n",
      "193\n",
      "194\n",
      "195\n",
      "196\n",
      "197\n",
      "198\n",
      "199\n",
      "200\n",
      "201\n",
      "202\n",
      "203\n",
      "204\n",
      "205\n",
      "206\n",
      "207\n",
      "208\n",
      "209\n",
      "210\n",
      "211\n",
      "212\n",
      "213\n",
      "214\n",
      "215\n",
      "216\n",
      "217\n",
      "218\n",
      "219\n",
      "220\n",
      "221\n",
      "222\n",
      "223\n",
      "224\n",
      "225\n",
      "226\n",
      "227\n",
      "228\n",
      "229\n",
      "230\n",
      "231\n",
      "232\n",
      "233\n",
      "234\n",
      "235\n",
      "236\n",
      "237\n",
      "238\n",
      "239\n",
      "240\n",
      "241\n",
      "242\n",
      "243\n",
      "244\n",
      "245\n",
      "246\n",
      "247\n",
      "248\n",
      "249\n",
      "250\n",
      "251\n",
      "252\n",
      "253\n",
      "254\n",
      "255\n",
      "256\n",
      "257\n",
      "258\n",
      "259\n",
      "260\n",
      "261\n",
      "262\n",
      "263\n",
      "264\n",
      "265\n",
      "266\n",
      "267\n",
      "268\n",
      "269\n",
      "270\n",
      "271\n",
      "272\n",
      "273\n",
      "274\n",
      "275\n",
      "276\n",
      "277\n",
      "278\n",
      "279\n",
      "280\n",
      "281\n",
      "282\n",
      "283\n",
      "284\n",
      "285\n",
      "286\n",
      "287\n",
      "288\n",
      "Jane Jacobs\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "Lawrence Lessig\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "Glenn Greenwald\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "140\n",
      "141\n",
      "142\n",
      "143\n",
      "144\n",
      "145\n",
      "146\n",
      "147\n",
      "148\n",
      "149\n",
      "150\n",
      "151\n",
      "152\n",
      "153\n",
      "154\n",
      "155\n",
      "156\n",
      "157\n",
      "158\n",
      "159\n",
      "160\n",
      "161\n",
      "162\n",
      "163\n",
      "164\n",
      "165\n",
      "166\n",
      "167\n",
      "168\n",
      "169\n",
      "170\n",
      "171\n",
      "172\n",
      "173\n",
      "174\n",
      "175\n",
      "176\n",
      "177\n",
      "178\n",
      "179\n",
      "180\n",
      "181\n",
      "182\n",
      "183\n",
      "184\n",
      "185\n",
      "186\n",
      "187\n",
      "188\n",
      "189\n",
      "190\n",
      "191\n",
      "192\n",
      "193\n",
      "194\n",
      "195\n",
      "196\n",
      "197\n",
      "198\n",
      "199\n",
      "Al Gore\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "Alan Bennett\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n",
      "34\n",
      "35\n",
      "36\n",
      "37\n",
      "38\n",
      "39\n",
      "40\n",
      "41\n",
      "42\n",
      "43\n",
      "44\n",
      "45\n",
      "46\n",
      "47\n",
      "48\n",
      "49\n",
      "50\n",
      "51\n",
      "52\n",
      "53\n",
      "54\n",
      "55\n",
      "56\n",
      "57\n",
      "58\n",
      "59\n",
      "60\n",
      "61\n",
      "62\n",
      "63\n",
      "64\n",
      "65\n",
      "66\n",
      "67\n",
      "68\n",
      "69\n",
      "70\n",
      "71\n",
      "72\n",
      "73\n",
      "74\n",
      "75\n",
      "76\n",
      "77\n",
      "78\n",
      "79\n",
      "80\n",
      "81\n",
      "82\n",
      "83\n",
      "84\n",
      "85\n",
      "86\n",
      "87\n",
      "88\n",
      "89\n",
      "90\n",
      "91\n",
      "92\n",
      "93\n",
      "94\n",
      "95\n",
      "96\n",
      "97\n",
      "98\n",
      "99\n",
      "100\n",
      "101\n",
      "102\n",
      "103\n",
      "104\n",
      "105\n",
      "106\n",
      "107\n",
      "108\n",
      "109\n",
      "110\n",
      "111\n",
      "112\n",
      "113\n",
      "114\n",
      "115\n",
      "116\n",
      "117\n",
      "118\n",
      "119\n",
      "120\n",
      "121\n",
      "122\n",
      "123\n",
      "124\n",
      "125\n",
      "126\n",
      "127\n",
      "128\n",
      "129\n",
      "130\n",
      "131\n",
      "132\n",
      "133\n",
      "134\n",
      "135\n",
      "136\n",
      "137\n",
      "138\n",
      "139\n",
      "140\n",
      "141\n",
      "142\n",
      "143\n",
      "144\n",
      "145\n",
      "146\n",
      "147\n",
      "148\n",
      "149\n",
      "Anthony Giddens\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "Benjamin Barber\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n"
     ]
    }
   ],
   "source": [
    "fake = {}\n",
    "#count = 0\n",
    "for name in CL_train.Author.unique():\n",
    "    print(name)\n",
    "    conta = []\n",
    "    for j,text in enumerate(CL_train[CL_train.Author==name].texicity):\n",
    "        print(j)\n",
    "        count = 0\n",
    "        for i in range(len(CL_train[CL_train.Author==name])):\n",
    "            if text in CL_train[CL_train.Author==name].iloc[i,5]:\n",
    "                count += 1\n",
    "\n",
    "\n",
    "        conta.append(count)\n",
    "    fake[name] = conta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Juan Williams\n",
      "Bill Moyers\n",
      "Paul Krugman\n",
      "Joseph E. Stiglitz\n",
      "Jesse Jackson\n",
      "George Soros\n",
      "Donna Brazile\n",
      "James Carville\n",
      "Tony Judt\n",
      "Jane Jacobs\n",
      "Lawrence Lessig\n",
      "Glenn Greenwald\n",
      "Al Gore\n",
      "Alan Bennett\n",
      "Anthony Giddens\n",
      "Benjamin Barber\n"
     ]
    }
   ],
   "source": [
    "for name in CL_train.Author.unique():\n",
    "    print(name)\n",
    "    CL_train.loc[CL_train.Author==name, 'rep'] = fake[name]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0     1647\n",
       "2.0       21\n",
       "3.0        5\n",
       "5.0        1\n",
       "10.0       1\n",
       "12.0       1\n",
       "Name: rep, dtype: int64"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CL_train.rep.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "CL_train.to_csv('CL_train_clean.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Robert Reich', 'Nicholas Kristof', 'Polly Toynbee', 'Ezra Klein'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CL_test.Author.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "autoref_test=find_autoref(CL_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Robert Reich  done\n",
      "Nicholas Kristof  done\n",
      "Polly Toynbee  done\n",
      "Ezra Klein  done\n"
     ]
    }
   ],
   "source": [
    "CL_test['auto'] = 0\n",
    "for name in CL_test.Author.unique():\n",
    "    print(name, ' done')\n",
    "    CL_test.loc[CL_test.Author==name, 'auto'] = autoref_test[name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CL_test.auto.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Author</th>\n",
       "      <th>Quotes</th>\n",
       "      <th>Contain</th>\n",
       "      <th>Political Orientation</th>\n",
       "      <th>auto</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>Globalization and free trade do spur economic growth, and they lead to lower prices on many goods.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>True patriotism isn't cheap. It's about taking on a fair share of the burden of keeping America going.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>The faith that anyone could move from rags to riches - with enough guts and gumption, hard work and nose to the grindstone - was once at the core of the American Dream.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>The only way to grow the economy in a way that benefits the bottom 90 percent is to change the structure of the economy. At the least, this requires stronger unions and a higher minimum wage.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>You can't inspire people if you are going to be uninspiring.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>One tax dodge often used by multi-national companies is to squirrel their earnings abroad in foreign subsidiaries located in countries where taxes are lower.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>Bankruptcy laws allow companies to smoothly reorganize, but not college graduates burdened by student loans.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>As public schools deteriorate, the upper-middle class and wealthy send their kids to private ones. As public pools and playgrounds decay, the better-off buy memberships in private tennis and swimming clubs. As public hospitals decline, the well-off pay premium rates for private care.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>As income from work has become more concentrated in America, the super rich have invested in businesses, real estate, art, and other assets. The income from these assets is now concentrating even faster than income from work.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>What are called 'public schools' in many of America's wealthy communities aren't really 'public' at all. In effect, they're private schools, whose tuition is hidden away in the purchase price of upscale homes there, and in the corresponding property taxes.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>Nations are becoming less relevant in a world where everyone and everything is interconnected. The connections that matter most are again becoming more personal.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>Before the rise of the nation-state, between the eighteenth and twentieth centuries, the world was mostly tribal. Tribes were united by language, religion, blood, and belief. They feared other tribes and often warred against them.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>Standing up to bullies is the hallmark of a civilized society.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>Can we please agree that in the real world, corporations exist for one purpose and one purpose only - to make as much money as possible, which means cutting costs as much as possible?</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>Those at the top would do better with a smaller share of a booming economy that elicits a positive politics than they will do with an ever-larger share of an anemic economy that fuels the politics of anger.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>A leader is someone who steps back from the entire system and tries to build a more collaborative, more innovative system that will work over the long term.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>We do not want to live in a theocracy. We should maintain that barrier and government has no business telling someone what they ought to believe or how they should conduct their private lives.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>Our young people - their capacities to think, understand, investigate, and innovate - are America's future.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>The job creators are members of America's vast middle class and the poor, whose purchases cause businesses to expand and invest.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>Technology is changing so fast that knowledge about specifics can quickly become obsolete. That's why so much of what technicians learn is on the job.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>It is hard to bite the hands that feed you, especially when you are competing for food.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>Sugary drinks are blamed for increasing the rates of chronic disease and obesity in America. Yet efforts to reduce their consumption through taxes or other measures have gone nowhere. The beverage industry has spent millions defeating them.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>The generosity of the super-rich is sometimes proffered as evidence they're contributing as much to the nation's well-being as they did decades ago when they paid a much larger share of their earnings in taxes.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>It's not government's business what people do in their private bedrooms.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>We never used to blink at taking a leadership role in the world. And we understood leadership often required something other than drones and bombs. We accepted global leadership not just for humanitarian reasons, but also because it was in our own best interest. We knew we couldn't isolate ourselves from trouble. There was no place to hide.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>The largest party in America, by the way, is neither the Democrats nor the Republicans. It's the party of non-voters.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>The wilderness is healing, a therapy for the soul.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>Our world is enriched when coders and marketers dazzle us with smartphones and tablets, but, by themselves, they are just slabs. It is the music, essays, entertainment and provocations that they access, spawned by the humanities, that animate them - and us.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>The conflict in Darfur could escalate to where we're seeing 100,000 victims per month.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>A few countries like Sri Lanka and Honduras have led the way in slashing maternal mortality.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>Every high school and college graduate in America should, I think, have some familiarity with statistics, economics and a foreign language such as Spanish. Religion may not be as indispensable, but the humanities should be a part of our repertory. They may not enrich our wallets, but they do enrich our lives. They civilize us. They provide context.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>I've always been interested in public health approaches because it seems to me we have this yearning for silver bullets, and that is not in fact how change comes about. Change comes through silver buckshot - a lot of little things that achieve results. That's a classic public health approach.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>Perhaps no country in Latin America is more picturesque than Bolivia, and the most memorable Bolivian city may be Potosi.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>Most of the time in the 21st century, we dominate our surroundings: We tweak the thermostat, and the temperature falls one degree. We push a button, and Taylor Swift sings for us. It's the opposite in the wilderness, which teaches us constantly that we are not lords of the universe but rather building blocks of it.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>Inequality causes problems by creating fissures in societies, leaving those at the bottom feeling marginalized or disenfranchised.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>Humans pull together in an odd way when they're in the wilderness. It's astonishing how few people litter and how much they help one another. Indeed, the smartphone app to navigate the Pacific Crest Trail, Halfmile, is a labor of love by hikers who make it available as a free download.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>One of the most crucial kinds of intervention is in advocacy. We can think about charities in the context of delivering services, and indeed that is part of their job, but advocacy is also getting governments to step up to the plate. They can also give more voice to those who don't have one.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>One of the aims of higher education is to broaden perspectives, and what better way than by a home stay in a really different country, like Bangladesh or Senegal? Time abroad also leaves one more aware of the complex prism of suspicion through which the United States is often viewed.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>Sure, food stamps are occasionally misused, but anyone familiar with business knows that the abuse of food subsidies is far greater in the corporate suite. Every time an executive wines and dines a hot date on the corporate dime, the average taxpayer helps foot the bill.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>Wilderness trails constitute a rare space in America marked by economic diversity. Lawyers and construction workers get bitten by the same mosquitoes and sip from the same streams; there are none of the usual signals about socioeconomic status, for most hikers are in shorts and a T-shirt and enveloped by an aroma that would make a skunk queasy.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>You no more have the right to risk others by failing to vaccinate than you do by sending your child to school with a hunting knife. Vaccination isn't a private choice but a civic obligation.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>Individual storytelling is incredibly powerful. We as journalists know intuitively what scientists of the brain are discovering through brain scans, which is that emotional stories tend to open the portals, and that once there's a connection made, people are more open to rational arguments.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>Random violence is incredibly infectious.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>In Angola, I visited 'HeroRats' that have been trained to sniff out land mines (and, in some countries, diagnose tuberculosis). In a day, they can clear 20 times as much of a minefield as a human, and they work for bananas!</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>Since the end of the 1970s, something has gone profoundly wrong in America. Inequality has soared. Educational progress slowed. Incarceration rates quintupled. Family breakdown accelerated. Median household income stagnated.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>The caricature of Islam as a violent and intolerant religion is horrendously incomplete. Remember that those standing up to Muslim fanatics are mostly Muslims.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>Utah may well be the most cosmopolitan state in America. Vast numbers of young Mormons - increasingly women as well as men - spend a couple of years abroad as missionaries and return jabbering in Thai or Portuguese and bearing a wealth of international experience.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>Beware of generalizations about any faith because they sometimes amount to the religious equivalent of racial profiling. Hinduism contained both Gandhi and the fanatic who assassinated him.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>For all Trump's criticisms of government, his family wealth came from feeding at the government trough. His father, Fred Trump, leveraged government housing programs into a construction business; the empire was founded on public money.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>There are very few things I've done just twice in my life, 40 years apart, and one is to backpack on the Pacific Crest Trail across the California/Oregon border.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>Most moms and dads, they want to be good moms and dads. But it's an incredibly hard job when you are stressed out, when you are poor, when your life is in chaos. And giving them some of the tools to be better parents, to whittle away at that parenting gap, gives those kids a much better starting point in life.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>Nicholas Kristof</td>\n",
       "      <td>We journalists are a bit like vultures, feasting on war, scandal and disaster. Turn on the news, and you see Syrian refugees, Volkswagen corruption, dysfunctional government. Yet that reflects a selection bias in how we report the news: We cover planes that crash, not planes that take off.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>People want the right to die at a time of their own choosing. Too many families have watched helplessly as a relative dies slowly, longing for death.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>Could a government dare to set out with happiness as its goal? Now that there are accepted scientific proofs, it would be easy to audit the progress of national happiness annually, just as we monitor money and GDP.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>Inequality makes everyone unhappy, the poor most of all, and that is well within the remit of the state. More money gives less extra happiness the richer we get, yet we are addicted to earning and spending more every year.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>But how odd that in this heathen nation of empty pews, where churches' bare, ruined choirs are converted into luxury loft living, a Labour government - yes, a Labour government - is deliberately creating a huge expansion of faith schools.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>How do you make any sense of history, art or literature without knowing the stories and iconography of your own culture and all the world's main religions?</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>Thresholds of pain, indignity and incapacity are entirely personal.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>Crime is only the worst example, but it is a paradigm for other Labour policy disasters. No one tells the voters that crime is falling: let them stay scared senseless.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>But the single overwhelming reason why jails are bursting is longer sentences given for more crimes.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>So what really works? Treatments in jail do some good, but it's mostly too late: finding a family and a job or just growing older make most prisoners eventually give up crime.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>Is anyone serious about the politics of happiness? David Cameron dipped a toe in the water, using the word lightly, but denying the hard policies it implies. Labour shies away from it, but should take up the challenge.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>It is now possible to quantify people's levels of happiness pretty accurately by asking them, by observation, and by measuring electrical activity in the brain, in degrees from terrible pain to sublime joy.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>Happiness is a real, objective phenomenon, scientifically verifiable. That means people and whole societies can now be measured over time and compared accurately with one another. Causes and cures for unhappiness can be quantified.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>One in six people suffer depression or a chronic anxiety disorder. These are not the worried well but those in severe mental pain with conditions crippling enough to prevent them living normal lives.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>The strongest predictor of unhappiness is anyone who has had a mental illness in the last 10 years. It is an even stronger predictor of unhappiness than poverty - which also ranks highly.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>Working lives are for the state to influence. Unemployment makes people unhappy. So does instability.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>My Lords temporal, today is the day to rise up against the regiment of Lords spiritual and proclaim the values of enlightenment, compassion and common sense.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>In the polls, over 80% support the right to die and have done for the last 25 years. Even 80% of practising Catholics and Protestants support it, plus 76% of Church Times readers.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>Most people come to fear not death itself, but the many terrible ways of dying.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>The best care on earth cannot prevent us all dying in the end.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>Openness about death has led to greater care about all aspects of dying.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>My mother begged doctors to end her life. She was beyond the physical ability to swallow enough of the weak morphine pills she had around her. When she knew she was dying I promised to make sure she could go at a time of her choosing, but it was impossible. I couldn't help.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>There is all the difference in the world between teaching children about religion and handing them over to be taught by the religious.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>This is indeed a clash of civilisations, not between Islam and Christendom but between reason and superstition.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>Polly Toynbee</td>\n",
       "      <td>But instead of standing up for reason, our government is handing education over to the world of faith.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>If nothing is done to counter present trends, the major fault line in American politics will no longer be between Democrats and Republicans, liberals and conservatives. It will be between the 'establishment'—political insiders, power brokers, the heads of American business, Wall Street, and the mainstream media—and an increasingly mad-as-hell populace determined to 'take back America' from them.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>Government doesn’t \"intrude\" on the \"free market.\" It creates the market.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>Those who argue for \"less government\" area really arguing for a different government—often one that favors them or their patrons.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>Robert Reich</td>\n",
       "      <td>The critical things to watch out for aren't the rare big events, such as the 2008 bailout of the Street itself, but the ongoing multitude of small rule changes that continuously alter the economic game. ... The \"free market\" is a myth that prevents us from examining these rule changes and asking whom they serve. The myth is therefore highly useful to those who do not wish such an examination to be undertaken.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>Ezra Klein</td>\n",
       "      <td>If I'm sitting around exulting over traffic data, I'm an idiot.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>Ezra Klein</td>\n",
       "      <td>The thing to remember about being young is you eventually get old.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>Ezra Klein</td>\n",
       "      <td>One of my big beliefs about Washington is that we highly overstate the power of individuals and highly underrate seeing Washington as a system, in general, but, in particular, we highly underrate the power of Congress.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>Ezra Klein</td>\n",
       "      <td>The idea of changing and fixing the problem of how news is presented on the Internet has been recognized for a long time.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>Ezra Klein</td>\n",
       "      <td>When I talk to people about 'KnowMore', it is as an experiment. The biggest thing I'll say there is that we've learned a lot from 'KnowMore'.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>Ezra Klein</td>\n",
       "      <td>My only advice is to try to get the job that's most like the job you want, rather than the one that's more prestigious. Always try to be the talent.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>Ezra Klein</td>\n",
       "      <td>Every newspaper in the country covers stories that other newspapers cover. Every industry is filled with people who are competing to do the best job providing a particular service.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>Ezra Klein</td>\n",
       "      <td>I think it's weird that the news cedes so much ground to Wikipedia. That isn't true in other informational sectors.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>Ezra Klein</td>\n",
       "      <td>My career wouldn't exist without blogs, electronic text, hyperlinks, and mass online audiences.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>Ezra Klein</td>\n",
       "      <td>It's not a good idea to conceptualize a static relationship with long-standing policies, like health care.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>Ezra Klein</td>\n",
       "      <td>When I first came to Washington, what I admired most was that people were just really, really smart with a tremendous amount of intellectual horsepower and the ability to look at an issue and say something fresh.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>Ezra Klein</td>\n",
       "      <td>When you're trying to come up with a good approach to reporting on the bleeding edge of where the conversation's moving, you're just leaving a lot of people who aren't on the bleeding edge of that conversation out.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>Ezra Klein</td>\n",
       "      <td>People set newspapers on fire; they use them for wrapping fish. The Internet does not have that property. What I don't think we've gotten is that you can make things last longer than in print.</td>\n",
       "      <td>no</td>\n",
       "      <td>Progressive</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Author  \\\n",
       "0       Robert Reich   \n",
       "1       Robert Reich   \n",
       "2       Robert Reich   \n",
       "3       Robert Reich   \n",
       "4       Robert Reich   \n",
       "5       Robert Reich   \n",
       "6       Robert Reich   \n",
       "7       Robert Reich   \n",
       "8       Robert Reich   \n",
       "9       Robert Reich   \n",
       "10      Robert Reich   \n",
       "11      Robert Reich   \n",
       "12      Robert Reich   \n",
       "13      Robert Reich   \n",
       "14      Robert Reich   \n",
       "15      Robert Reich   \n",
       "16      Robert Reich   \n",
       "17      Robert Reich   \n",
       "18      Robert Reich   \n",
       "19      Robert Reich   \n",
       "20      Robert Reich   \n",
       "21      Robert Reich   \n",
       "22      Robert Reich   \n",
       "23      Robert Reich   \n",
       "24      Robert Reich   \n",
       "25      Robert Reich   \n",
       "26  Nicholas Kristof   \n",
       "27  Nicholas Kristof   \n",
       "28  Nicholas Kristof   \n",
       "29  Nicholas Kristof   \n",
       "30  Nicholas Kristof   \n",
       "31  Nicholas Kristof   \n",
       "32  Nicholas Kristof   \n",
       "33  Nicholas Kristof   \n",
       "34  Nicholas Kristof   \n",
       "35  Nicholas Kristof   \n",
       "36  Nicholas Kristof   \n",
       "37  Nicholas Kristof   \n",
       "38  Nicholas Kristof   \n",
       "39  Nicholas Kristof   \n",
       "40  Nicholas Kristof   \n",
       "41  Nicholas Kristof   \n",
       "42  Nicholas Kristof   \n",
       "43  Nicholas Kristof   \n",
       "44  Nicholas Kristof   \n",
       "45  Nicholas Kristof   \n",
       "46  Nicholas Kristof   \n",
       "47  Nicholas Kristof   \n",
       "48  Nicholas Kristof   \n",
       "49  Nicholas Kristof   \n",
       "50  Nicholas Kristof   \n",
       "51  Nicholas Kristof   \n",
       "52     Polly Toynbee   \n",
       "53     Polly Toynbee   \n",
       "54     Polly Toynbee   \n",
       "55     Polly Toynbee   \n",
       "56     Polly Toynbee   \n",
       "57     Polly Toynbee   \n",
       "58     Polly Toynbee   \n",
       "59     Polly Toynbee   \n",
       "60     Polly Toynbee   \n",
       "61     Polly Toynbee   \n",
       "62     Polly Toynbee   \n",
       "63     Polly Toynbee   \n",
       "64     Polly Toynbee   \n",
       "65     Polly Toynbee   \n",
       "66     Polly Toynbee   \n",
       "67     Polly Toynbee   \n",
       "68     Polly Toynbee   \n",
       "69     Polly Toynbee   \n",
       "70     Polly Toynbee   \n",
       "71     Polly Toynbee   \n",
       "72     Polly Toynbee   \n",
       "73     Polly Toynbee   \n",
       "74     Polly Toynbee   \n",
       "75     Polly Toynbee   \n",
       "76      Robert Reich   \n",
       "77      Robert Reich   \n",
       "78      Robert Reich   \n",
       "79      Robert Reich   \n",
       "80        Ezra Klein   \n",
       "81        Ezra Klein   \n",
       "82        Ezra Klein   \n",
       "83        Ezra Klein   \n",
       "84        Ezra Klein   \n",
       "85        Ezra Klein   \n",
       "86        Ezra Klein   \n",
       "87        Ezra Klein   \n",
       "88        Ezra Klein   \n",
       "89        Ezra Klein   \n",
       "90        Ezra Klein   \n",
       "91        Ezra Klein   \n",
       "92        Ezra Klein   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                          Quotes  \\\n",
       "0                                                                                                                                                                                                                                                                                                                             Globalization and free trade do spur economic growth, and they lead to lower prices on many goods.   \n",
       "1                                                                                                                                                                                                                                                                                                                         True patriotism isn't cheap. It's about taking on a fair share of the burden of keeping America going.   \n",
       "2                                                                                                                                                                                                                                                       The faith that anyone could move from rags to riches - with enough guts and gumption, hard work and nose to the grindstone - was once at the core of the American Dream.   \n",
       "3                                                                                                                                                                                                                                The only way to grow the economy in a way that benefits the bottom 90 percent is to change the structure of the economy. At the least, this requires stronger unions and a higher minimum wage.   \n",
       "4                                                                                                                                                                                                                                                                                                                                                                   You can't inspire people if you are going to be uninspiring.   \n",
       "5                                                                                                                                                                                                                                                                  One tax dodge often used by multi-national companies is to squirrel their earnings abroad in foreign subsidiaries located in countries where taxes are lower.   \n",
       "6                                                                                                                                                                                                                                                                                                                   Bankruptcy laws allow companies to smoothly reorganize, but not college graduates burdened by student loans.   \n",
       "7                                                                                                                                   As public schools deteriorate, the upper-middle class and wealthy send their kids to private ones. As public pools and playgrounds decay, the better-off buy memberships in private tennis and swimming clubs. As public hospitals decline, the well-off pay premium rates for private care.   \n",
       "8                                                                                                                                                                                              As income from work has become more concentrated in America, the super rich have invested in businesses, real estate, art, and other assets. The income from these assets is now concentrating even faster than income from work.   \n",
       "9                                                                                                                                                               What are called 'public schools' in many of America's wealthy communities aren't really 'public' at all. In effect, they're private schools, whose tuition is hidden away in the purchase price of upscale homes there, and in the corresponding property taxes.   \n",
       "10                                                                                                                                                                                                                                                             Nations are becoming less relevant in a world where everyone and everything is interconnected. The connections that matter most are again becoming more personal.   \n",
       "11                                                                                                                                                                                        Before the rise of the nation-state, between the eighteenth and twentieth centuries, the world was mostly tribal. Tribes were united by language, religion, blood, and belief. They feared other tribes and often warred against them.   \n",
       "12                                                                                                                                                                                                                                                                                                                                                                Standing up to bullies is the hallmark of a civilized society.   \n",
       "13                                                                                                                                                                                                                                       Can we please agree that in the real world, corporations exist for one purpose and one purpose only - to make as much money as possible, which means cutting costs as much as possible?   \n",
       "14                                                                                                                                                                                                                Those at the top would do better with a smaller share of a booming economy that elicits a positive politics than they will do with an ever-larger share of an anemic economy that fuels the politics of anger.   \n",
       "15                                                                                                                                                                                                                                                                  A leader is someone who steps back from the entire system and tries to build a more collaborative, more innovative system that will work over the long term.   \n",
       "16                                                                                                                                                                                                                              We do not want to live in a theocracy. We should maintain that barrier and government has no business telling someone what they ought to believe or how they should conduct their private lives.   \n",
       "17                                                                                                                                                                                                                                                                                                                   Our young people - their capacities to think, understand, investigate, and innovate - are America's future.   \n",
       "18                                                                                                                                                                                                                                                                                              The job creators are members of America's vast middle class and the poor, whose purchases cause businesses to expand and invest.   \n",
       "19                                                                                                                                                                                                                                                                        Technology is changing so fast that knowledge about specifics can quickly become obsolete. That's why so much of what technicians learn is on the job.   \n",
       "20                                                                                                                                                                                                                                                                                                                                       It is hard to bite the hands that feed you, especially when you are competing for food.   \n",
       "21                                                                                                                                                                              Sugary drinks are blamed for increasing the rates of chronic disease and obesity in America. Yet efforts to reduce their consumption through taxes or other measures have gone nowhere. The beverage industry has spent millions defeating them.   \n",
       "22                                                                                                                                                                                                            The generosity of the super-rich is sometimes proffered as evidence they're contributing as much to the nation's well-being as they did decades ago when they paid a much larger share of their earnings in taxes.   \n",
       "23                                                                                                                                                                                                                                                                                                                                                      It's not government's business what people do in their private bedrooms.   \n",
       "24                                                                        We never used to blink at taking a leadership role in the world. And we understood leadership often required something other than drones and bombs. We accepted global leadership not just for humanitarian reasons, but also because it was in our own best interest. We knew we couldn't isolate ourselves from trouble. There was no place to hide.   \n",
       "25                                                                                                                                                                                                                                                                                                         The largest party in America, by the way, is neither the Democrats nor the Republicans. It's the party of non-voters.   \n",
       "26                                                                                                                                                                                                                                                                                                                                                                            The wilderness is healing, a therapy for the soul.   \n",
       "27                                                                                                                                                             Our world is enriched when coders and marketers dazzle us with smartphones and tablets, but, by themselves, they are just slabs. It is the music, essays, entertainment and provocations that they access, spawned by the humanities, that animate them - and us.   \n",
       "28                                                                                                                                                                                                                                                                                                                                        The conflict in Darfur could escalate to where we're seeing 100,000 victims per month.   \n",
       "29                                                                                                                                                                                                                                                                                                                                  A few countries like Sri Lanka and Honduras have led the way in slashing maternal mortality.   \n",
       "30                                                                Every high school and college graduate in America should, I think, have some familiarity with statistics, economics and a foreign language such as Spanish. Religion may not be as indispensable, but the humanities should be a part of our repertory. They may not enrich our wallets, but they do enrich our lives. They civilize us. They provide context.   \n",
       "31                                                                                                                         I've always been interested in public health approaches because it seems to me we have this yearning for silver bullets, and that is not in fact how change comes about. Change comes through silver buckshot - a lot of little things that achieve results. That's a classic public health approach.   \n",
       "32                                                                                                                                                                                                                                                                                                     Perhaps no country in Latin America is more picturesque than Bolivia, and the most memorable Bolivian city may be Potosi.   \n",
       "33                                                                                                  Most of the time in the 21st century, we dominate our surroundings: We tweak the thermostat, and the temperature falls one degree. We push a button, and Taylor Swift sings for us. It's the opposite in the wilderness, which teaches us constantly that we are not lords of the universe but rather building blocks of it.   \n",
       "34                                                                                                                                                                                                                                                                                            Inequality causes problems by creating fissures in societies, leaving those at the bottom feeling marginalized or disenfranchised.   \n",
       "35                                                                                                                                Humans pull together in an odd way when they're in the wilderness. It's astonishing how few people litter and how much they help one another. Indeed, the smartphone app to navigate the Pacific Crest Trail, Halfmile, is a labor of love by hikers who make it available as a free download.   \n",
       "36                                                                                                                          One of the most crucial kinds of intervention is in advocacy. We can think about charities in the context of delivering services, and indeed that is part of their job, but advocacy is also getting governments to step up to the plate. They can also give more voice to those who don't have one.   \n",
       "37                                                                                                                                  One of the aims of higher education is to broaden perspectives, and what better way than by a home stay in a really different country, like Bangladesh or Senegal? Time abroad also leaves one more aware of the complex prism of suspicion through which the United States is often viewed.   \n",
       "38                                                                                                                                               Sure, food stamps are occasionally misused, but anyone familiar with business knows that the abuse of food subsidies is far greater in the corporate suite. Every time an executive wines and dines a hot date on the corporate dime, the average taxpayer helps foot the bill.   \n",
       "39                                                                    Wilderness trails constitute a rare space in America marked by economic diversity. Lawyers and construction workers get bitten by the same mosquitoes and sip from the same streams; there are none of the usual signals about socioeconomic status, for most hikers are in shorts and a T-shirt and enveloped by an aroma that would make a skunk queasy.   \n",
       "40                                                                                                                                                                                                                                You no more have the right to risk others by failing to vaccinate than you do by sending your child to school with a hunting knife. Vaccination isn't a private choice but a civic obligation.   \n",
       "41                                                                                                                           Individual storytelling is incredibly powerful. We as journalists know intuitively what scientists of the brain are discovering through brain scans, which is that emotional stories tend to open the portals, and that once there's a connection made, people are more open to rational arguments.   \n",
       "42                                                                                                                                                                                                                                                                                                                                                                                     Random violence is incredibly infectious.   \n",
       "43                                                                                                                                                                                               In Angola, I visited 'HeroRats' that have been trained to sniff out land mines (and, in some countries, diagnose tuberculosis). In a day, they can clear 20 times as much of a minefield as a human, and they work for bananas!   \n",
       "44                                                                                                                                                                                              Since the end of the 1970s, something has gone profoundly wrong in America. Inequality has soared. Educational progress slowed. Incarceration rates quintupled. Family breakdown accelerated. Median household income stagnated.   \n",
       "45                                                                                                                                                                                                                                                               The caricature of Islam as a violent and intolerant religion is horrendously incomplete. Remember that those standing up to Muslim fanatics are mostly Muslims.   \n",
       "46                                                                                                                                                      Utah may well be the most cosmopolitan state in America. Vast numbers of young Mormons - increasingly women as well as men - spend a couple of years abroad as missionaries and return jabbering in Thai or Portuguese and bearing a wealth of international experience.   \n",
       "47                                                                                                                                                                                                                                 Beware of generalizations about any faith because they sometimes amount to the religious equivalent of racial profiling. Hinduism contained both Gandhi and the fanatic who assassinated him.   \n",
       "48                                                                                                                                                                                   For all Trump's criticisms of government, his family wealth came from feeding at the government trough. His father, Fred Trump, leveraged government housing programs into a construction business; the empire was founded on public money.   \n",
       "49                                                                                                                                                                                                                                                             There are very few things I've done just twice in my life, 40 years apart, and one is to backpack on the Pacific Crest Trail across the California/Oregon border.   \n",
       "50                                                                                                       Most moms and dads, they want to be good moms and dads. But it's an incredibly hard job when you are stressed out, when you are poor, when your life is in chaos. And giving them some of the tools to be better parents, to whittle away at that parenting gap, gives those kids a much better starting point in life.   \n",
       "51                                                                                                                            We journalists are a bit like vultures, feasting on war, scandal and disaster. Turn on the news, and you see Syrian refugees, Volkswagen corruption, dysfunctional government. Yet that reflects a selection bias in how we report the news: We cover planes that crash, not planes that take off.   \n",
       "52                                                                                                                                                                                                                                                                         People want the right to die at a time of their own choosing. Too many families have watched helplessly as a relative dies slowly, longing for death.   \n",
       "53                                                                                                                                                                                                        Could a government dare to set out with happiness as its goal? Now that there are accepted scientific proofs, it would be easy to audit the progress of national happiness annually, just as we monitor money and GDP.   \n",
       "54                                                                                                                                                                                                Inequality makes everyone unhappy, the poor most of all, and that is well within the remit of the state. More money gives less extra happiness the richer we get, yet we are addicted to earning and spending more every year.   \n",
       "55                                                                                                                                                                                But how odd that in this heathen nation of empty pews, where churches' bare, ruined choirs are converted into luxury loft living, a Labour government - yes, a Labour government - is deliberately creating a huge expansion of faith schools.   \n",
       "56                                                                                                                                                                                                                                                                   How do you make any sense of history, art or literature without knowing the stories and iconography of your own culture and all the world's main religions?   \n",
       "57                                                                                                                                                                                                                                                                                                                                                           Thresholds of pain, indignity and incapacity are entirely personal.   \n",
       "58                                                                                                                                                                                                                                                       Crime is only the worst example, but it is a paradigm for other Labour policy disasters. No one tells the voters that crime is falling: let them stay scared senseless.   \n",
       "59                                                                                                                                                                                                                                                                                                                          But the single overwhelming reason why jails are bursting is longer sentences given for more crimes.   \n",
       "60                                                                                                                                                                                                                                               So what really works? Treatments in jail do some good, but it's mostly too late: finding a family and a job or just growing older make most prisoners eventually give up crime.   \n",
       "61                                                                                                                                                                                                    Is anyone serious about the politics of happiness? David Cameron dipped a toe in the water, using the word lightly, but denying the hard policies it implies. Labour shies away from it, but should take up the challenge.   \n",
       "62                                                                                                                                                                                                                It is now possible to quantify people's levels of happiness pretty accurately by asking them, by observation, and by measuring electrical activity in the brain, in degrees from terrible pain to sublime joy.   \n",
       "63                                                                                                                                                                                       Happiness is a real, objective phenomenon, scientifically verifiable. That means people and whole societies can now be measured over time and compared accurately with one another. Causes and cures for unhappiness can be quantified.   \n",
       "64                                                                                                                                                                                                                       One in six people suffer depression or a chronic anxiety disorder. These are not the worried well but those in severe mental pain with conditions crippling enough to prevent them living normal lives.   \n",
       "65                                                                                                                                                                                                                                   The strongest predictor of unhappiness is anyone who has had a mental illness in the last 10 years. It is an even stronger predictor of unhappiness than poverty - which also ranks highly.   \n",
       "66                                                                                                                                                                                                                                                                                                                         Working lives are for the state to influence. Unemployment makes people unhappy. So does instability.   \n",
       "67                                                                                                                                                                                                                                                                 My Lords temporal, today is the day to rise up against the regiment of Lords spiritual and proclaim the values of enlightenment, compassion and common sense.   \n",
       "68                                                                                                                                                                                                                                           In the polls, over 80% support the right to die and have done for the last 25 years. Even 80% of practising Catholics and Protestants support it, plus 76% of Church Times readers.   \n",
       "69                                                                                                                                                                                                                                                                                                                                               Most people come to fear not death itself, but the many terrible ways of dying.   \n",
       "70                                                                                                                                                                                                                                                                                                                                                                The best care on earth cannot prevent us all dying in the end.   \n",
       "71                                                                                                                                                                                                                                                                                                                                                      Openness about death has led to greater care about all aspects of dying.   \n",
       "72                                                                                                                                            My mother begged doctors to end her life. She was beyond the physical ability to swallow enough of the weak morphine pills she had around her. When she knew she was dying I promised to make sure she could go at a time of her choosing, but it was impossible. I couldn't help.   \n",
       "73                                                                                                                                                                                                                                                                                        There is all the difference in the world between teaching children about religion and handing them over to be taught by the religious.   \n",
       "74                                                                                                                                                                                                                                                                                                               This is indeed a clash of civilisations, not between Islam and Christendom but between reason and superstition.   \n",
       "75                                                                                                                                                                                                                                                                                                                        But instead of standing up for reason, our government is handing education over to the world of faith.   \n",
       "76                If nothing is done to counter present trends, the major fault line in American politics will no longer be between Democrats and Republicans, liberals and conservatives. It will be between the 'establishment'—political insiders, power brokers, the heads of American business, Wall Street, and the mainstream media—and an increasingly mad-as-hell populace determined to 'take back America' from them.   \n",
       "77                                                                                                                                                                                                                                                                                                                                                     Government doesn’t \"intrude\" on the \"free market.\" It creates the market.   \n",
       "78                                                                                                                                                                                                                                                                                             Those who argue for \"less government\" area really arguing for a different government—often one that favors them or their patrons.   \n",
       "79  The critical things to watch out for aren't the rare big events, such as the 2008 bailout of the Street itself, but the ongoing multitude of small rule changes that continuously alter the economic game. ... The \"free market\" is a myth that prevents us from examining these rule changes and asking whom they serve. The myth is therefore highly useful to those who do not wish such an examination to be undertaken.   \n",
       "80                                                                                                                                                                                                                                                                                                                                                               If I'm sitting around exulting over traffic data, I'm an idiot.   \n",
       "81                                                                                                                                                                                                                                                                                                                                                            The thing to remember about being young is you eventually get old.   \n",
       "82                                                                                                                                                                                                    One of my big beliefs about Washington is that we highly overstate the power of individuals and highly underrate seeing Washington as a system, in general, but, in particular, we highly underrate the power of Congress.   \n",
       "83                                                                                                                                                                                                                                                                                                     The idea of changing and fixing the problem of how news is presented on the Internet has been recognized for a long time.   \n",
       "84                                                                                                                                                                                                                                                                                 When I talk to people about 'KnowMore', it is as an experiment. The biggest thing I'll say there is that we've learned a lot from 'KnowMore'.   \n",
       "85                                                                                                                                                                                                                                                                          My only advice is to try to get the job that's most like the job you want, rather than the one that's more prestigious. Always try to be the talent.   \n",
       "86                                                                                                                                                                                                                                          Every newspaper in the country covers stories that other newspapers cover. Every industry is filled with people who are competing to do the best job providing a particular service.   \n",
       "87                                                                                                                                                                                                                                                                                                           I think it's weird that the news cedes so much ground to Wikipedia. That isn't true in other informational sectors.   \n",
       "88                                                                                                                                                                                                                                                                                                                               My career wouldn't exist without blogs, electronic text, hyperlinks, and mass online audiences.   \n",
       "89                                                                                                                                                                                                                                                                                                                    It's not a good idea to conceptualize a static relationship with long-standing policies, like health care.   \n",
       "90                                                                                                                                                                                                          When I first came to Washington, what I admired most was that people were just really, really smart with a tremendous amount of intellectual horsepower and the ability to look at an issue and say something fresh.   \n",
       "91                                                                                                                                                                                                        When you're trying to come up with a good approach to reporting on the bleeding edge of where the conversation's moving, you're just leaving a lot of people who aren't on the bleeding edge of that conversation out.   \n",
       "92                                                                                                                                                                                                                              People set newspapers on fire; they use them for wrapping fish. The Internet does not have that property. What I don't think we've gotten is that you can make things last longer than in print.   \n",
       "\n",
       "   Contain Political Orientation  auto  \n",
       "0       no           Progressive     0  \n",
       "1       no           Progressive     0  \n",
       "2       no           Progressive     0  \n",
       "3       no           Progressive     0  \n",
       "4       no           Progressive     0  \n",
       "5       no           Progressive     0  \n",
       "6       no           Progressive     0  \n",
       "7       no           Progressive     0  \n",
       "8       no           Progressive     0  \n",
       "9       no           Progressive     0  \n",
       "10      no           Progressive     0  \n",
       "11      no           Progressive     0  \n",
       "12      no           Progressive     0  \n",
       "13      no           Progressive     0  \n",
       "14      no           Progressive     0  \n",
       "15      no           Progressive     0  \n",
       "16      no           Progressive     0  \n",
       "17      no           Progressive     0  \n",
       "18      no           Progressive     0  \n",
       "19      no           Progressive     0  \n",
       "20      no           Progressive     0  \n",
       "21      no           Progressive     0  \n",
       "22      no           Progressive     0  \n",
       "23      no           Progressive     0  \n",
       "24      no           Progressive     0  \n",
       "25      no           Progressive     0  \n",
       "26      no           Progressive     0  \n",
       "27      no           Progressive     0  \n",
       "28      no           Progressive     0  \n",
       "29      no           Progressive     0  \n",
       "30      no           Progressive     0  \n",
       "31      no           Progressive     0  \n",
       "32      no           Progressive     0  \n",
       "33      no           Progressive     0  \n",
       "34      no           Progressive     0  \n",
       "35      no           Progressive     0  \n",
       "36      no           Progressive     0  \n",
       "37      no           Progressive     0  \n",
       "38      no           Progressive     0  \n",
       "39      no           Progressive     0  \n",
       "40      no           Progressive     0  \n",
       "41      no           Progressive     0  \n",
       "42      no           Progressive     0  \n",
       "43      no           Progressive     0  \n",
       "44      no           Progressive     0  \n",
       "45      no           Progressive     0  \n",
       "46      no           Progressive     0  \n",
       "47      no           Progressive     0  \n",
       "48      no           Progressive     0  \n",
       "49      no           Progressive     0  \n",
       "50      no           Progressive     0  \n",
       "51      no           Progressive     0  \n",
       "52      no           Progressive     0  \n",
       "53      no           Progressive     0  \n",
       "54      no           Progressive     0  \n",
       "55      no           Progressive     0  \n",
       "56      no           Progressive     0  \n",
       "57      no           Progressive     0  \n",
       "58      no           Progressive     0  \n",
       "59      no           Progressive     0  \n",
       "60      no           Progressive     0  \n",
       "61      no           Progressive     0  \n",
       "62      no           Progressive     0  \n",
       "63      no           Progressive     0  \n",
       "64      no           Progressive     0  \n",
       "65      no           Progressive     0  \n",
       "66      no           Progressive     0  \n",
       "67      no           Progressive     0  \n",
       "68      no           Progressive     0  \n",
       "69      no           Progressive     0  \n",
       "70      no           Progressive     0  \n",
       "71      no           Progressive     0  \n",
       "72      no           Progressive     0  \n",
       "73      no           Progressive     0  \n",
       "74      no           Progressive     0  \n",
       "75      no           Progressive     0  \n",
       "76      no           Progressive     0  \n",
       "77      no           Progressive     0  \n",
       "78      no           Progressive     0  \n",
       "79      no           Progressive     0  \n",
       "80      no           Progressive     0  \n",
       "81      no           Progressive     0  \n",
       "82      no           Progressive     0  \n",
       "83      no           Progressive     0  \n",
       "84      no           Progressive     0  \n",
       "85      no           Progressive     0  \n",
       "86      no           Progressive     0  \n",
       "87      no           Progressive     0  \n",
       "88      no           Progressive     0  \n",
       "89      no           Progressive     0  \n",
       "90      no           Progressive     0  \n",
       "91      no           Progressive     0  \n",
       "92      no           Progressive     0  "
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CL_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "CL_test=CL_test.drop(CL_test.index[[4,32,88]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "CL_test['texicity'] = CL_test.Quotes.apply(lambda x : preprocess_text(x, \n",
    "                                                                      lowercase=True, \n",
    "                                                                      no_punct=True, \n",
    "                                                                      no_accents=True, \n",
    "                                                                      no_contractions=True,\n",
    "                                                                     no_numbers=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "CL_test = tok_stop(CL_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "CL_test.to_csv('CL_test_clean.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Robert Reich\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n",
      "28\n",
      "Nicholas Kristof\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "Polly Toynbee\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "Ezra Klein\n",
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n"
     ]
    }
   ],
   "source": [
    "fake = {}\n",
    "#count = 0\n",
    "for name in CL_test.Author.unique():\n",
    "    print(name)\n",
    "    conta = []\n",
    "    for i,text in enumerate(CL_test[CL_test.Author==name].texicity) :\n",
    "        print(i)\n",
    "        count = 0\n",
    "        for i in range(len(CL_test[CL_test.Author==name])):\n",
    "            if text in CL_test[CL_test.Author==name].iloc[i,5]:\n",
    "                count += 1\n",
    "\n",
    "\n",
    "        conta.append(count)\n",
    "    fake[name] = conta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Robert Reich  done\n",
      "Nicholas Kristof  done\n",
      "Polly Toynbee  done\n",
      "Ezra Klein  done\n"
     ]
    }
   ],
   "source": [
    "CL_test['rep'] = 0\n",
    "for name in CL_test.Author.unique():\n",
    "    print(name, ' done')\n",
    "    CL_test.loc[CL_test.Author==name, 'rep'] = fake[name]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    90\n",
       "Name: rep, dtype: int64"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CL_test.rep.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "CL_train.to_csv('CL_train_clean.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}