01_RedditScraper.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Daksh_Hacklytics.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "CNUgpcOSDwgv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import requests\n",
        "import json\n",
        "import pandas as pd\n",
        "import time"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mqAYVXX9EUdu",
        "colab_type": "code",
        "outputId": "06bfb83f-33c7-4ffb-b97d-7c28a80b55b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 434
        }
      },
      "source": [
        "after = 0\n",
        "sub = \"uber\"\n",
        "size = 1000\n",
        "score = 20\n",
        "newdf = pd.DataFrame(columns=[\"title\",\"score\",\"created_utc\"])\n",
        "print(newdf.head())\n",
        "\n",
        "while(True):\n",
        "  url = \"https://api.pushshift.io/reddit/search/submission/?subreddit={}&score=%3E{}&size={}&sort=asc&sort_type=created_utc&after={}\".format(sub, score, size, after)\n",
        "  r = requests.get(url)\n",
        "  print(r)\n",
        "  data = r.json()\n",
        "  # if newdf.shape[0] == 0:\n",
        "  #   newdf = pd.DataFrame(data[\"data\"]).filter([\"title\",\"score\",\"created_utc\"], axis=1)\n",
        "  # else:\n",
        "  newdf = newdf.append(data[\"data\"], ignore_index=True).filter([\"title\",\"score\",\"created_utc\"], axis=1)\n",
        "  print(newdf.shape)\n",
        "  after = newdf[\"created_utc\"][newdf.shape[0] - 1]\n",
        "  \n",
        "  if (int(newdf.shape[0]) < 999):\n",
        "    break\n",
        "  \n",
        "newdf.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Empty DataFrame\n",
            "Columns: [title, score, created_utc]\n",
            "Index: []\n",
            "<Response [200]>\n",
            "(645, 3)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/pandas/core/frame.py:7138: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
            "of pandas will change to not sort by default.\n",
            "\n",
            "To accept the future behavior, pass 'sort=False'.\n",
            "\n",
            "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
            "\n",
            "  sort=sort,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title</th>\n",
              "      <th>score</th>\n",
              "      <th>created_utc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>PSA from an Uber driver on rating your driver.</td>\n",
              "      <td>22</td>\n",
              "      <td>1406423484</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Is rating a driver a 4 actually like telling U...</td>\n",
              "      <td>22</td>\n",
              "      <td>1422892697</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>My recommendations on getting your driver exac...</td>\n",
              "      <td>21</td>\n",
              "      <td>1422907306</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Uber should make comments mandatory for any ra...</td>\n",
              "      <td>41</td>\n",
              "      <td>1425586306</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>With uber banned, Las Vegas taxis are a comple...</td>\n",
              "      <td>22</td>\n",
              "      <td>1426197083</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                               title score created_utc\n",
              "0     PSA from an Uber driver on rating your driver.    22  1406423484\n",
              "1  Is rating a driver a 4 actually like telling U...    22  1422892697\n",
              "2  My recommendations on getting your driver exac...    21  1422907306\n",
              "3  Uber should make comments mandatory for any ra...    41  1425586306\n",
              "4  With uber banned, Las Vegas taxis are a comple...    22  1426197083"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N8w0Xlkr3iF3",
        "colab_type": "code",
        "outputId": "3c83930a-4d47-47e7-9df7-9035ab420bcb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "print(newdf['created_utc'][newdf.shape[0] - 1])\n",
        "print(newdf.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1578157177\n",
            "(645, 3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0BB_3HTTTf0A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "newdf.to_csv(\"reddits.csv\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1rhpHVjqTkiT",
        "colab_type": "code",
        "outputId": "1e731a7a-259d-47b4-ce83-9733c44a74e5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        }
      },
      "source": [
        "newdf.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title</th>\n",
              "      <th>score</th>\n",
              "      <th>created_utc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>PSA from an Uber driver on rating your driver.</td>\n",
              "      <td>22</td>\n",
              "      <td>1406423484</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Is rating a driver a 4 actually like telling U...</td>\n",
              "      <td>22</td>\n",
              "      <td>1422892697</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>My recommendations on getting your driver exac...</td>\n",
              "      <td>21</td>\n",
              "      <td>1422907306</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Uber should make comments mandatory for any ra...</td>\n",
              "      <td>41</td>\n",
              "      <td>1425586306</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>With uber banned, Las Vegas taxis are a comple...</td>\n",
              "      <td>22</td>\n",
              "      <td>1426197083</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                               title score created_utc\n",
              "0     PSA from an Uber driver on rating your driver.    22  1406423484\n",
              "1  Is rating a driver a 4 actually like telling U...    22  1422892697\n",
              "2  My recommendations on getting your driver exac...    21  1422907306\n",
              "3  Uber should make comments mandatory for any ra...    41  1425586306\n",
              "4  With uber banned, Las Vegas taxis are a comple...    22  1426197083"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aLfGTTKaTnTQ",
        "colab_type": "code",
        "outputId": "40f0e467-26fa-4a3f-fb00-af194cdf56a9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import numpy as np\n",
        "from math import log\n",
        "\n",
        "copyDf = newdf.copy()\n",
        "print(copyDf.score.max())\n",
        "copyDf.score = copyDf.score.apply(log) / log(45586)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "203\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lwHOH_2WWId8",
        "colab_type": "code",
        "outputId": "4e1e65a7-011e-4886-be5f-6058bfa43035",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        }
      },
      "source": [
        "copyDf.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title</th>\n",
              "      <th>score</th>\n",
              "      <th>created_utc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>PSA from an Uber driver on rating your driver.</td>\n",
              "      <td>0.288146</td>\n",
              "      <td>1406423484</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Is rating a driver a 4 actually like telling U...</td>\n",
              "      <td>0.288146</td>\n",
              "      <td>1422892697</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>My recommendations on getting your driver exac...</td>\n",
              "      <td>0.283809</td>\n",
              "      <td>1422907306</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Uber should make comments mandatory for any ra...</td>\n",
              "      <td>0.346178</td>\n",
              "      <td>1425586306</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>With uber banned, Las Vegas taxis are a comple...</td>\n",
              "      <td>0.288146</td>\n",
              "      <td>1426197083</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                               title     score created_utc\n",
              "0     PSA from an Uber driver on rating your driver.  0.288146  1406423484\n",
              "1  Is rating a driver a 4 actually like telling U...  0.288146  1422892697\n",
              "2  My recommendations on getting your driver exac...  0.283809  1422907306\n",
              "3  Uber should make comments mandatory for any ra...  0.346178  1425586306\n",
              "4  With uber banned, Las Vegas taxis are a comple...  0.288146  1426197083"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OyC6gmD0YVqD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "copyDf.to_csv(\"redditsNew.csv\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QucssIF-Y0NQ",
        "colab_type": "code",
        "outputId": "d5f64378-5b2a-4326-bb5f-1918f2fecbbc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284
        }
      },
      "source": [
        "copyDf.describe()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>645.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>0.329332</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>0.038137</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.283809</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>0.300062</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>0.320115</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>0.348424</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>0.495295</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            score\n",
              "count  645.000000\n",
              "mean     0.329332\n",
              "std      0.038137\n",
              "min      0.283809\n",
              "25%      0.300062\n",
              "50%      0.320115\n",
              "75%      0.348424\n",
              "max      0.495295"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7jlzGPkKZA8t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}