3_reddit-scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import requests.auth\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from nltk import word_tokenize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generate a link to query the API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Criteria (full list and API documentation here: https://pushshift.io/api-parameters/)\n",
    "\n",
    "criteria=dict()\n",
    "\n",
    "#type (comment or submission)\n",
    "search_type='submission'\n",
    "\n",
    "#search term\n",
    "criteria['q']='halloween'\n",
    "\n",
    "#start time (m (minute), h (hour), d (day))\n",
    "criteria['before']='0d'\n",
    "\n",
    "#end time (m (minute), h (hour), d (day))\n",
    "criteria['after']='30d'\n",
    "\n",
    "#size of results (max=1000)\n",
    "criteria['size']='1000'\n",
    "\n",
    "#sort results by score\n",
    "criteria['sort_type']='score'\n",
    "criteria['sort']='desc'\n",
    "\n",
    "#subreddit\n",
    "criteria['subreddit']='AskReddit'\n",
    "\n",
    "#score threshold\n",
    "criteria['score']='>15'\n",
    "\n",
    "#author\n",
    "#criteria['author']='user1992'\n",
    "\n",
    "#function for generate the search URL from criteria\n",
    "def get_url(search_type,criteria):   \n",
    "    url='https://api.pushshift.io/reddit/search/' + search_type + '/?'\n",
    "    for crit in criteria:\n",
    "        if crit=='score':\n",
    "            url=url + crit + criteria[crit] + '&'\n",
    "        else:\n",
    "            url=url + crit + '=' + criteria[crit] + '&'\n",
    "    return url\n",
    "\n",
    "#generate the url \n",
    "url=get_url(search_type,criteria)\n",
    "\n",
    "print(url)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find submission about candy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#request data\n",
    "submissions = requests.get(url).json()['data']\n",
    "\n",
    "#loop through submissions\n",
    "for submission in submissions:\n",
    "    \n",
    "    #get submission title\n",
    "    title=submission['title']\n",
    "    \n",
    "    #if candy in title, get submission link_id and break loop\n",
    "    if 'candy' in title:\n",
    "        print(title)\n",
    "        \n",
    "        #get link ID to query comments\n",
    "        link_id=submission['id']       \n",
    "        \n",
    "        break\n",
    "        \n",
    "\n",
    "#Now search comments in this link with new url\n",
    "criteria=dict()\n",
    "\n",
    "#submission link_id\n",
    "criteria['link_id']=link_id\n",
    "\n",
    "#size\n",
    "criteria['size']='1000'\n",
    "\n",
    "#type\n",
    "search_type='comment'\n",
    "\n",
    "#get URL\n",
    "url=get_url(search_type,criteria)\n",
    "\n",
    "#request data\n",
    "comments = requests.get(url).json()['data']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check the comments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for comment in comments:\n",
    "    \n",
    "    #print if comment is a parent (top level) comment\n",
    "    if comment['link_id']==comment['parent_id']:\n",
    "    \n",
    "        print(comment['body'])\n",
    "        print('\\n')\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Do sentiment analysis in two different subreddits with AFINN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#analyze sentiment with afinn\n",
    "from afinn import Afinn\n",
    "afinn = Afinn()\n",
    "\n",
    "#initialize search criteria\n",
    "criteria1=dict()\n",
    "criteria2=dict()\n",
    "\n",
    "#type (comment or submission)\n",
    "search_type='submission'\n",
    "\n",
    "#start time (m (minute), h (hour), d (day))\n",
    "criteria1['before']='0d'\n",
    "criteria2['before']='0d'\n",
    "\n",
    "#end time (m (minute), h (hour), d (day))\n",
    "criteria1['after']='300d'\n",
    "criteria2['after']='300d'\n",
    "\n",
    "#size of results (max=1000)\n",
    "criteria1['size']='1000'\n",
    "criteria2['size']='1000'\n",
    "\n",
    "#sort\n",
    "criteria1['sort_type']='score'\n",
    "criteria1['sort']='desc'\n",
    "criteria2['sort_type']='score'\n",
    "criteria2['sort']='desc'\n",
    "\n",
    "#subreddits\n",
    "criteria1['subreddit']='tifu'\n",
    "criteria2['subreddit']='WholesomeMemes'\n",
    "\n",
    "#get urls\n",
    "url1=get_url(search_type,criteria1)\n",
    "url2=get_url(search_type,criteria2)\n",
    "\n",
    "#get submissions\n",
    "submissions1 = requests.get(url1).json()['data']\n",
    "submissions2 = requests.get(url2).json()['data']\n",
    "\n",
    "sentiments1=[]\n",
    "sentiments2=[]\n",
    "\n",
    "#get sentiments for subreddit1 submissions\n",
    "for submission in submissions1:\n",
    "    post=submission['title']\n",
    "    sentiment=afinn.score(post) / len(post.split(' '))\n",
    "    sentiments1.append(sentiment)\n",
    "\n",
    "#get sentiments for subreddit2 submissions\n",
    "for submission in submissions2:\n",
    "    post=submission['title']\n",
    "    sentiment=afinn.score(post) / len(post.split(' '))\n",
    "    sentiments2.append(sentiment)\n",
    "    \n",
    "print('AFINN SCORE:')\n",
    "print(criteria1['subreddit'] + ': ' + str(np.mean(sentiments1)))\n",
    "print(criteria2['subreddit'] + ': ' + str(np.mean(sentiments2)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}