eda_vinay_controversial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv(\"C:\\\\Users\\\\Hannah\\\\Desktop\\\\UT_Austin\\\\Analytics_MSBA\\\\Fall_19\\\\Data_mgmt_intro\\\\Final_project\\\\worldnews_0619.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>body</th>\n",
       "      <th>author</th>\n",
       "      <th>created_utc</th>\n",
       "      <th>subreddit_id</th>\n",
       "      <th>link_id</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>score</th>\n",
       "      <th>retrieved_on</th>\n",
       "      <th>controversiality</th>\n",
       "      <th>gilded</th>\n",
       "      <th>id</th>\n",
       "      <th>subreddit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Boarders huh</td>\n",
       "      <td>J_Man2743</td>\n",
       "      <td>1559518235</td>\n",
       "      <td>t5_2qh13</td>\n",
       "      <td>t3_a78kyo</td>\n",
       "      <td>t3_a78kyo</td>\n",
       "      <td>0</td>\n",
       "      <td>1568784725</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>epuwwc7</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[removed]</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>1561011745</td>\n",
       "      <td>t5_2qh13</td>\n",
       "      <td>t3_aeu81p</td>\n",
       "      <td>t1_erl6n3x</td>\n",
       "      <td>0</td>\n",
       "      <td>1570329691</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>erm9r84</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I think you know less when you think that Stal...</td>\n",
       "      <td>steeritupld</td>\n",
       "      <td>1560168854</td>\n",
       "      <td>t5_2qh13</td>\n",
       "      <td>t3_andfac</td>\n",
       "      <td>t1_eqmsv60</td>\n",
       "      <td>0</td>\n",
       "      <td>1569348202</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>eqmx6le</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>So why are you using it as an argument against...</td>\n",
       "      <td>arvada14</td>\n",
       "      <td>1560403679</td>\n",
       "      <td>t5_2qh13</td>\n",
       "      <td>t3_aqio89</td>\n",
       "      <td>t1_eggt2jk</td>\n",
       "      <td>0</td>\n",
       "      <td>1569783807</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>eqzooom</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>There is no round up ready wheat in the commer...</td>\n",
       "      <td>arvada14</td>\n",
       "      <td>1560400159</td>\n",
       "      <td>t5_2qh13</td>\n",
       "      <td>t3_aqio89</td>\n",
       "      <td>t1_eghowah</td>\n",
       "      <td>0</td>\n",
       "      <td>1569779899</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>eqzhngv</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                body       author  \\\n",
       "0                                       Boarders huh    J_Man2743   \n",
       "1                                          [removed]    [deleted]   \n",
       "2  I think you know less when you think that Stal...  steeritupld   \n",
       "3  So why are you using it as an argument against...     arvada14   \n",
       "4  There is no round up ready wheat in the commer...     arvada14   \n",
       "\n",
       "   created_utc subreddit_id    link_id   parent_id  score  retrieved_on  \\\n",
       "0   1559518235     t5_2qh13  t3_a78kyo   t3_a78kyo      0    1568784725   \n",
       "1   1561011745     t5_2qh13  t3_aeu81p  t1_erl6n3x      0    1570329691   \n",
       "2   1560168854     t5_2qh13  t3_andfac  t1_eqmsv60      0    1569348202   \n",
       "3   1560403679     t5_2qh13  t3_aqio89  t1_eggt2jk      0    1569783807   \n",
       "4   1560400159     t5_2qh13  t3_aqio89  t1_eghowah      0    1569779899   \n",
       "\n",
       "   controversiality  gilded       id  subreddit  \n",
       "0                 0       0  epuwwc7  worldnews  \n",
       "1                 0       0  erm9r84  worldnews  \n",
       "2                 0       0  eqmx6le  worldnews  \n",
       "3                 0       0  eqzooom  worldnews  \n",
       "4                 0       0  eqzhngv  worldnews  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[deleted]           123493\n",
       "AutoModerator         5601\n",
       "autotldr              3904\n",
       "FblthpLives           1152\n",
       "Capitalist_Model       830\n",
       "Name: author, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['author'].value_counts()[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1=df[df['body']!=\"[removed]\"]\n",
    "df2=df1[df1['body']!=\"[deleted]\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>body</th>\n",
       "      <th>author</th>\n",
       "      <th>created_utc</th>\n",
       "      <th>subreddit_id</th>\n",
       "      <th>link_id</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>score</th>\n",
       "      <th>retrieved_on</th>\n",
       "      <th>controversiality</th>\n",
       "      <th>gilded</th>\n",
       "      <th>id</th>\n",
       "      <th>subreddit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Boarders huh</td>\n",
       "      <td>J_Man2743</td>\n",
       "      <td>1559518235</td>\n",
       "      <td>t5_2qh13</td>\n",
       "      <td>t3_a78kyo</td>\n",
       "      <td>t3_a78kyo</td>\n",
       "      <td>0</td>\n",
       "      <td>1568784725</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>epuwwc7</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I think you know less when you think that Stal...</td>\n",
       "      <td>steeritupld</td>\n",
       "      <td>1560168854</td>\n",
       "      <td>t5_2qh13</td>\n",
       "      <td>t3_andfac</td>\n",
       "      <td>t1_eqmsv60</td>\n",
       "      <td>0</td>\n",
       "      <td>1569348202</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>eqmx6le</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>So why are you using it as an argument against...</td>\n",
       "      <td>arvada14</td>\n",
       "      <td>1560403679</td>\n",
       "      <td>t5_2qh13</td>\n",
       "      <td>t3_aqio89</td>\n",
       "      <td>t1_eggt2jk</td>\n",
       "      <td>0</td>\n",
       "      <td>1569783807</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>eqzooom</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>There is no round up ready wheat in the commer...</td>\n",
       "      <td>arvada14</td>\n",
       "      <td>1560400159</td>\n",
       "      <td>t5_2qh13</td>\n",
       "      <td>t3_aqio89</td>\n",
       "      <td>t1_eghowah</td>\n",
       "      <td>0</td>\n",
       "      <td>1569779899</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>eqzhngv</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Curious...If there ever winds up being proven ...</td>\n",
       "      <td>zenwalrus</td>\n",
       "      <td>1559812945</td>\n",
       "      <td>t5_2qh13</td>\n",
       "      <td>t3_atqudb</td>\n",
       "      <td>t1_eh3b69n</td>\n",
       "      <td>0</td>\n",
       "      <td>1568978537</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>eq5skxs</td>\n",
       "      <td>worldnews</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                body       author  \\\n",
       "0                                       Boarders huh    J_Man2743   \n",
       "2  I think you know less when you think that Stal...  steeritupld   \n",
       "3  So why are you using it as an argument against...     arvada14   \n",
       "4  There is no round up ready wheat in the commer...     arvada14   \n",
       "5  Curious...If there ever winds up being proven ...    zenwalrus   \n",
       "\n",
       "   created_utc subreddit_id    link_id   parent_id  score  retrieved_on  \\\n",
       "0   1559518235     t5_2qh13  t3_a78kyo   t3_a78kyo      0    1568784725   \n",
       "2   1560168854     t5_2qh13  t3_andfac  t1_eqmsv60      0    1569348202   \n",
       "3   1560403679     t5_2qh13  t3_aqio89  t1_eggt2jk      0    1569783807   \n",
       "4   1560400159     t5_2qh13  t3_aqio89  t1_eghowah      0    1569779899   \n",
       "5   1559812945     t5_2qh13  t3_atqudb  t1_eh3b69n      0    1568978537   \n",
       "\n",
       "   controversiality  gilded       id  subreddit  \n",
       "0                 0       0  epuwwc7  worldnews  \n",
       "2                 0       0  eqmx6le  worldnews  \n",
       "3                 0       0  eqzooom  worldnews  \n",
       "4                 0       0  eqzhngv  worldnews  \n",
       "5                 0       0  eq5skxs  worldnews  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df2[pd.notnull(df2['body'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "c=(pd.Series(' '.join(df2['body']).lower().split())).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Hannah\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "\n",
    "# You will have to download the set of stop words the first time\n",
    "import nltk\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "s=set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_sentence = [] \n",
    "  \n",
    "for w in c: \n",
    "    if w not in s: \n",
    "        filtered_sentence.append(w) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('people', 129041),\n",
       " ('like', 102564),\n",
       " ('would', 94037),\n",
       " ('us', 72032),\n",
       " ('one', 62265),\n",
       " ('think', 60980),\n",
       " ('get', 58791),\n",
       " ('even', 57851),\n",
       " ('&gt;', 54062),\n",
       " ('trump', 44404),\n",
       " ('also', 44326),\n",
       " (\"i'm\", 44024),\n",
       " ('know', 41835),\n",
       " (\"that's\", 40965),\n",
       " ('want', 40312),\n",
       " ('make', 38736),\n",
       " ('could', 38620),\n",
       " ('much', 36247),\n",
       " ('going', 35703),\n",
       " ('really', 35154)]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "Counter(filtered_sentence).most_common(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3=df2[df2['controversiality']==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "c=(pd.Series(' '.join(df3['body']).lower().split())).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_sentence_cont = [] \n",
    "  \n",
    "for w in c: \n",
    "    if w not in s: \n",
    "        filtered_sentence_cont.append(w) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('people', 12386),\n",
       " ('like', 8797),\n",
       " ('us', 7423),\n",
       " ('would', 7361),\n",
       " ('trump', 5071),\n",
       " ('think', 5041),\n",
       " ('one', 4936),\n",
       " ('get', 4762),\n",
       " ('even', 4670),\n",
       " ('&gt;', 3460),\n",
       " (\"i'm\", 3392),\n",
       " ('right', 3327),\n",
       " ('want', 3323),\n",
       " (\"that's\", 3279),\n",
       " ('know', 3272),\n",
       " ('make', 3213),\n",
       " ('also', 3154),\n",
       " ('iran', 3106),\n",
       " ('going', 2990),\n",
       " ('really', 2832)]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "Counter(filtered_sentence_cont).most_common(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}