NB1_Filtering_BDP Final Project_Kshitijm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d33cdd43-40d5-478a-aefe-7c3f711f1325",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.8.15 | packaged by conda-forge | (default, Nov 22 2022, 08:46:39) \n",
      "[GCC 10.4.0]\n",
      "3.1.3\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "print(sys.version)\n",
    "print(spark.version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e53028db-b351-4cd7-ab10-827d9f9abafc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "pd.set_option('display.max_colwidth', None)\n",
    "pd.reset_option('display.max_rows')\n",
    "from itertools import compress \n",
    "from pyspark.sql.functions import *\n",
    "from pyspark.sql.types import *\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "warnings.filterwarnings(action='ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4fd0ee04-b91e-45f5-a184-dec5b0bd416e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: regex in /opt/conda/miniconda3/lib/python3.8/site-packages (2021.4.4)\n",
      "Collecting regex\n",
      "  Downloading regex-2022.10.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (772 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m772.3/772.3 kB\u001b[0m \u001b[31m13.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: regex\n",
      "  Attempting uninstall: regex\n",
      "    Found existing installation: regex 2021.4.4\n",
      "    Uninstalling regex-2021.4.4:\n",
      "      Successfully uninstalled regex-2021.4.4\n",
      "Successfully installed regex-2022.10.31\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install -U regex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4606267d-2dd0-4863-906a-73126aa37757",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import re\n",
    "# from pyspark.ml.feature import MinHashLSH\n",
    "# from pyspark.ml.feature import CountVectorizer,  IDF, CountVectorizerModel, Tokenizer, RegexTokenizer, StopWordsRemover\n",
    "# from pyspark import SparkContext\n",
    "# from pyspark.sql import SQLContext\n",
    "# from pyspark.sql import Row\n",
    "# import nltk\n",
    "# nltk.download('stopwords')\n",
    "# from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f77729c5-f7da-4e59-9d85-f919c05b390e",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option(\"max_colwidth\", 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2ee08f51-a4fe-4613-b5e6-e14d532cb1c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import storage\n",
    "# !pip install gcsfs --upgrade"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51e4207f-d6a0-4738-9826-c5cffa716816",
   "metadata": {},
   "source": [
    "### Setting Up Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ef2cbb3a-bdd6-406b-9614-32c1fab5415c",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark.conf.set(\"spark.sql.repl.eagerEval.enabled\",True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c15421e9-e94d-4b63-8a72-2315a664fb9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def list_blobs(bucket_name, folder_name):\n",
    "    gcs_client = storage.Client()\n",
    "    bucket = gcs_client.bucket(bucket_name)\n",
    "    blobs = list(bucket.list_blobs(prefix=folder_name))\n",
    "\n",
    "    for blob in blobs:\n",
    "        print(blob.name + '\\t' + str(blob.size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b9c53846-8887-4e5d-9b82-ea2875308a28",
   "metadata": {},
   "outputs": [],
   "source": [
    "# List all files in given COS directory\n",
    "def list_blobs_pd(bucket_name, folder_name):\n",
    "    gcs_client = storage.Client()\n",
    "    bucket = gcs_client.bucket(bucket_name)\n",
    "    blobs = list(bucket.list_blobs(prefix=folder_name))\n",
    "\n",
    "    blob_name = []\n",
    "    blob_size = []\n",
    "    \n",
    "    for blob in blobs:\n",
    "        blob_name.append(blob.name)\n",
    "        blob_size.append(blob.size)\n",
    "\n",
    "    blobs_df = pd.DataFrame(list(zip(blob_name, blob_size)), columns=['Name','Size'])\n",
    "\n",
    "    blobs_df.style.format({\"Size\": \"{:,.0f}\"}) \n",
    "    \n",
    "    return blobs_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "380396e3-d2ec-485a-b025-004b78c109ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Delete folder from COS bucket\n",
    "def delete_folder(bucket_name, folder_name):\n",
    "    gcs_client = storage.Client()\n",
    "    bucket = gcs_client.bucket(bucket_name)\n",
    "    blobs = list(bucket.list_blobs(prefix=folder_name))\n",
    "\n",
    "    for blob in blobs:\n",
    "        blob.delete()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c666991a-9cfc-439f-a4d0-1dcbdada05fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reading data from open bucket, avaible to all students\n",
    "bucket_read = 'msca-bdp-tweets'\n",
    "\n",
    "# Saving results into individual bucket, students must update to their own bucket\n",
    "bucket_write = 'msca-bdp-students-bucket/shared_data/kshitijm'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7478d8d2-2a95-479b-99ae-9371ac9cd9ce",
   "metadata": {},
   "source": [
    "### Accessing Main Database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "98fea783-4b0c-40e9-a521-c7f4def99d87",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ls: `msca-bdp-students-bucket/shared_data/kshitijm/Final_Project-FiltData1': No such file or directory\n"
     ]
    }
   ],
   "source": [
    "#!hadoop fs -ls 'gs://msca-bdp-tweets/final_project'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9227469-05ca-4778-82f1-d5853c1135e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>final_project/</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>final_project/_SUCCESS</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>final_project/part-00000-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json</td>\n",
       "      <td>4500466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>final_project/part-00001-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json</td>\n",
       "      <td>4107431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>final_project/part-00002-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json</td>\n",
       "      <td>4672123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50692</th>\n",
       "      <td>final_project/part-50690-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json</td>\n",
       "      <td>11562361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50693</th>\n",
       "      <td>final_project/part-50691-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json</td>\n",
       "      <td>9132693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50694</th>\n",
       "      <td>final_project/part-50692-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json</td>\n",
       "      <td>15376390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50695</th>\n",
       "      <td>final_project/part-50693-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json</td>\n",
       "      <td>8586044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50696</th>\n",
       "      <td>final_project/part-50694-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json</td>\n",
       "      <td>14317778</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50697 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                          Name  \\\n",
       "0                                                               final_project/   \n",
       "1                                                       final_project/_SUCCESS   \n",
       "2      final_project/part-00000-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json   \n",
       "3      final_project/part-00001-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json   \n",
       "4      final_project/part-00002-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json   \n",
       "...                                                                        ...   \n",
       "50692  final_project/part-50690-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json   \n",
       "50693  final_project/part-50691-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json   \n",
       "50694  final_project/part-50692-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json   \n",
       "50695  final_project/part-50693-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json   \n",
       "50696  final_project/part-50694-aa6d3cb4-7022-4df2-9921-218307589ce2-c000.json   \n",
       "\n",
       "           Size  \n",
       "0             0  \n",
       "1             0  \n",
       "2       4500466  \n",
       "3       4107431  \n",
       "4       4672123  \n",
       "...         ...  \n",
       "50692  11562361  \n",
       "50693   9132693  \n",
       "50694  15376390  \n",
       "50695   8586044  \n",
       "50696  14317778  \n",
       "\n",
       "[50697 rows x 2 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_blobs_pd(bucket_read,'final_project')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0766fa35-95d6-4904-9909-bc9d05841345",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/03/03 04:30:02 WARN org.apache.spark.sql.execution.datasources.SharedInMemoryCache: Evicting cached table partition metadata from memory due to size constraints (spark.sql.hive.filesourcePartitionFileCacheSize = 262144000 bytes). This may impact query planning performance.\n",
      "23/03/03 04:35:27 WARN org.apache.spark.sql.catalyst.util.package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n"
     ]
    }
   ],
   "source": [
    "twitter_df= spark.read.json('gs://msca-bdp-tweets/final_project')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "14b1816f-a4f8-42f4-9fad-35c7dae1ddb2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "99994342"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#twitter_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f2ed466d-585e-492b-8b10-7387d6885b27",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>coordinates</th><th>created_at</th><th>display_text_range</th><th>entities</th><th>extended_entities</th><th>extended_tweet</th><th>favorite_count</th><th>favorited</th><th>filter_level</th><th>geo</th><th>id</th><th>id_str</th><th>in_reply_to_screen_name</th><th>in_reply_to_status_id</th><th>in_reply_to_status_id_str</th><th>in_reply_to_user_id</th><th>in_reply_to_user_id_str</th><th>is_quote_status</th><th>lang</th><th>place</th><th>possibly_sensitive</th><th>quote_count</th><th>quoted_status</th><th>quoted_status_id</th><th>quoted_status_id_str</th><th>quoted_status_permalink</th><th>quoted_text</th><th>reply_count</th><th>retweet_count</th><th>retweeted</th><th>retweeted_from</th><th>retweeted_status</th><th>source</th><th>text</th><th>timestamp_ms</th><th>truncated</th><th>tweet_text</th><th>user</th><th>withheld_in_countries</th></tr>\n",
       "<tr><td>null</td><td>Wed May 25 00:46:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1529262624952569856</td><td>1529262624952569856</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>nicolesjchung</td><td>{null, Tue May 24...</td><td>&lt;a href=&quot;http://t...</td><td>RT @nicolesjchung...</td><td>1653439596771</td><td>false</td><td>it&#x27;s school. they...</td><td>{false, Thu Dec 0...</td><td>null</td></tr>\n",
       "<tr><td>null</td><td>Wed May 25 00:46:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1529262625057607680</td><td>1529262625057607680</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>KingJames</td><td>{null, Tue May 24...</td><td>&lt;a href=&quot;http://t...</td><td>RT @KingJames: My...</td><td>1653439596796</td><td>false</td><td>My thoughts and p...</td><td>{false, Thu Oct 1...</td><td>null</td></tr>\n",
       "<tr><td>null</td><td>Wed May 25 00:46:...</td><td>[27, 122]</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1529262625099419648</td><td>1529262625099419648</td><td>SaraIsaacson4</td><td>1529205912182968323</td><td>1529205912182968323</td><td>1112921930015956993</td><td>1112921930015956993</td><td>false</td><td>en</td><td>{{[[[-105.053666,...</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td></td><td>Leahgreenb I am m...</td><td>null</td><td>&lt;a href=&quot;http://t...</td><td>@SaraIsaacson4 @L...</td><td>1653439596806</td><td>false</td><td>@SaraIsaacson4 @L...</td><td>{false, Sun Feb 2...</td><td>null</td></tr>\n",
       "<tr><td>null</td><td>Wed May 25 00:46:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1529262625296613376</td><td>1529262625296613376</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>priore_ashley</td><td>{null, Wed May 25...</td><td>&lt;a href=&quot;http://t...</td><td>RT @priore_ashley...</td><td>1653439596853</td><td>false</td><td>Gun violence epid...</td><td>{false, Tue Aug 0...</td><td>null</td></tr>\n",
       "<tr><td>null</td><td>Wed May 25 00:46:...</td><td>null</td><td>{[], null, [], [{...</td><td>null</td><td>{[0, 253], {[], n...</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1529262625166708739</td><td>1529262625166708739</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>false</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td></td><td>null</td><td>null</td><td>&lt;a href=&quot;http://t...</td><td>WATCH LIVE: Presi...</td><td>1653439596822</td><td>true</td><td>WATCH LIVE: Presi...</td><td>{false, Sat Oct 2...</td><td>null</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+-----------+--------------------+------------------+--------------------+-----------------+--------------------+--------------+---------+------------+----+-------------------+-------------------+-----------------------+---------------------+-------------------------+-------------------+-----------------------+---------------+----+-----+------------------+-----------+--------------------+-------------------+--------------------+-----------------------+--------------------+-----------+-------------+---------+--------------+--------------------+--------------------+--------------------+-------------+---------+--------------------+--------------------+---------------------+\n",
       "|coordinates|          created_at|display_text_range|            entities|extended_entities|      extended_tweet|favorite_count|favorited|filter_level| geo|                 id|             id_str|in_reply_to_screen_name|in_reply_to_status_id|in_reply_to_status_id_str|in_reply_to_user_id|in_reply_to_user_id_str|is_quote_status|lang|place|possibly_sensitive|quote_count|       quoted_status|   quoted_status_id|quoted_status_id_str|quoted_status_permalink|         quoted_text|reply_count|retweet_count|retweeted|retweeted_from|    retweeted_status|              source|                text| timestamp_ms|truncated|          tweet_text|                user|withheld_in_countries|\n",
       "+-----------+--------------------+------------------+--------------------+-----------------+--------------------+--------------+---------+------------+----+-------------------+-------------------+-----------------------+---------------------+-------------------------+-------------------+-----------------------+---------------+----+-----+------------------+-----------+--------------------+-------------------+--------------------+-----------------------+--------------------+-----------+-------------+---------+--------------+--------------------+--------------------+--------------------+-------------+---------+--------------------+--------------------+---------------------+\n",
       "|       null|Wed May 25 21:22:...|              null|{[], null, [], []...|             null|                null|             0|    false|         low|null|1529573716455202817|1529573716455202817|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|   AaronParnas|{null, Wed May 25...|<a href=\"http://t...|RT @AaronParnas: ...|1653513766765|    false|If you are more u...|{false, Sat Nov 0...|                 null|\n",
       "|       null|Wed May 25 21:22:...|              null|{[], null, [], [{...|             null|{[0, 228], {[], n...|             0|    false|         low|null|1529573716602064901|1529573716602064901|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|         |          null|                null|<a href=\"https://...|Fun fact: In Germ...|1653513766800|     true|Fun fact: In Germ...|{false, Fri Jan 1...|                 null|\n",
       "|       null|Wed May 25 21:22:...|              null|{[], null, [], []...|             null|                null|             0|    false|         low|null|1529573716921040896|1529573716921040896|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|   MoneyMiaaaa|{null, Tue May 24...|<a href=\"https://...|RT @MoneyMiaaaa: ...|1653513766876|    false|26 school shootin...|{false, Tue Dec 1...|                 null|\n",
       "|       null|Wed May 25 21:22:...|              null|{[], null, [], []...|             null|                null|             0|    false|         low|null|1529573717583478787|1529573717583478787|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|   MoneyMiaaaa|{null, Tue May 24...|<a href=\"http://t...|RT @MoneyMiaaaa: ...|1653513767034|    false|26 school shootin...|{false, Fri Feb 1...|                 null|\n",
       "|       null|Wed May 25 21:22:...|              null|{[], null, [], [{...|             null|{[0, 261], {[], [...|             0|    false|         low|null|1529573717243830275|1529573717243830275|                   null|                 null|                     null|               null|                   null|           true|  en| null|             false|          0|{null, Sat Apr 02...|1510361661085036544| 1510361661085036544|   {twitter.com/0law...|For Sale \n",
       "\n",
       "A plot...|          0|            0|         |          null|                null|<a href=\"http://t...|FOR SALE\n",
       "\n",
       "Mixed u...|1653513766953|     true|FOR SALE\n",
       "\n",
       "Mixed u...|{false, Thu Nov 1...|                 null|\n",
       "+-----------+--------------------+------------------+--------------------+-----------------+--------------------+--------------+---------+------------+----+-------------------+-------------------+-----------------------+---------------------+-------------------------+-------------------+-----------------------+---------------+----+-----+------------------+-----------+--------------------+-------------------+--------------------+-----------------------+--------------------+-----------+-------------+---------+--------------+--------------------+--------------------+--------------------+-------------+---------+--------------------+--------------------+---------------------+"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_df.limit(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6996e71c-5334-494d-89d1-8ec97f2f451c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>coordinates</th><th>created_at</th><th>display_text_range</th><th>entities</th><th>extended_entities</th><th>extended_tweet</th><th>favorite_count</th><th>favorited</th><th>filter_level</th><th>geo</th><th>id</th><th>id_str</th><th>in_reply_to_screen_name</th><th>in_reply_to_status_id</th><th>in_reply_to_status_id_str</th><th>in_reply_to_user_id</th><th>in_reply_to_user_id_str</th><th>is_quote_status</th><th>lang</th><th>place</th><th>possibly_sensitive</th><th>quote_count</th><th>quoted_status</th><th>quoted_status_id</th><th>quoted_status_id_str</th><th>quoted_status_permalink</th><th>quoted_text</th><th>reply_count</th><th>retweet_count</th><th>retweeted</th><th>retweeted_from</th><th>retweeted_status</th><th>source</th><th>text</th><th>timestamp_ms</th><th>truncated</th><th>tweet_text</th><th>user</th><th>withheld_in_countries</th></tr>\n",
       "<tr><td>null</td><td>Mon Nov 07 16:39:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1589658862785028097</td><td>1589658862785028097</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>LifeNewsHQ</td><td>{null, Mon Nov 07...</td><td>&lt;a href=&quot;http://t...</td><td>RT @LifeNewsHQ: B...</td><td>1667839181978</td><td>false</td><td>BREAKING:  A fede...</td><td>{false, Fri Mar 0...</td><td>null</td></tr>\n",
       "<tr><td>null</td><td>Mon Nov 07 16:39:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1589658863376429056</td><td>1589658863376429056</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>robbystarbuck</td><td>{null, Sun Nov 06...</td><td>&lt;a href=&quot;http://t...</td><td>RT @robbystarbuck...</td><td>1667839182119</td><td>false</td><td>This is Dana Ness...</td><td>{false, Tue Apr 2...</td><td>null</td></tr>\n",
       "<tr><td>null</td><td>Mon Nov 07 16:39:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1589658863489675264</td><td>1589658863489675264</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>MarinaPurkiss</td><td>{null, Mon Nov 07...</td><td>&lt;a href=&quot;https://...</td><td>RT @MarinaPurkiss...</td><td>1667839182146</td><td>false</td><td>Many of those in ...</td><td>{false, Tue Feb 0...</td><td>null</td></tr>\n",
       "<tr><td>null</td><td>Mon Nov 07 16:39:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1589658864173355008</td><td>1589658864173355008</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>twit_fluffy</td><td>{null, Mon Nov 07...</td><td>&lt;a href=&quot;https://...</td><td>RT @twit_fluffy: ...</td><td>1667839182309</td><td>false</td><td>A new art that I ...</td><td>{false, Sun Nov 0...</td><td>null</td></tr>\n",
       "<tr><td>null</td><td>Mon Nov 07 16:39:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1589658865355829248</td><td>1589658865355829248</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>true</td><td>en</td><td>null</td><td>null</td><td>0</td><td>{null, Sun Nov 06...</td><td>1589202057168359425</td><td>1589202057168359425</td><td>{twitter.com/BBCN...</td><td>Keir Starmer: Too...</td><td>0</td><td>0</td><td>RT</td><td>NesrineMalik</td><td>{null, Sun Nov 06...</td><td>&lt;a href=&quot;https://...</td><td>RT @NesrineMalik:...</td><td>1667839182591</td><td>false</td><td>There is actually...</td><td>{false, Fri Nov 0...</td><td>null</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+-----------+--------------------+------------------+--------------------+-----------------+--------------------+--------------+---------+------------+----+-------------------+-------------------+-----------------------+---------------------+-------------------------+-------------------+-----------------------+---------------+----+-----+------------------+-----------+--------------------+-------------------+--------------------+-----------------------+--------------------+-----------+-------------+---------+--------------------+--------------------+--------------------+--------------------+-------------+---------+--------------------+--------------------+---------------------+\n",
       "|coordinates|          created_at|display_text_range|            entities|extended_entities|      extended_tweet|favorite_count|favorited|filter_level| geo|                 id|             id_str|in_reply_to_screen_name|in_reply_to_status_id|in_reply_to_status_id_str|in_reply_to_user_id|in_reply_to_user_id_str|is_quote_status|lang|place|possibly_sensitive|quote_count|       quoted_status|   quoted_status_id|quoted_status_id_str|quoted_status_permalink|         quoted_text|reply_count|retweet_count|retweeted|      retweeted_from|    retweeted_status|              source|                text| timestamp_ms|truncated|          tweet_text|                user|withheld_in_countries|\n",
       "+-----------+--------------------+------------------+--------------------+-----------------+--------------------+--------------+---------+------------+----+-------------------+-------------------+-----------------------+---------------------+-------------------------+-------------------+-----------------------+---------------+----+-----+------------------+-----------+--------------------+-------------------+--------------------+-----------------------+--------------------+-----------+-------------+---------+--------------------+--------------------+--------------------+--------------------+-------------+---------+--------------------+--------------------+---------------------+\n",
       "|       null|Wed May 25 17:29:...|              null|{[], null, [], []...|             null|                null|             0|    false|         low|null|1529514980768456705|1529514980768456705|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|     MalloryMcMorrow|{null, Wed May 25...|<a href=\"http://t...|RT @MalloryMcMorr...|1653499763086|    false|It’s apparently t...|{false, Sat May 2...|                 null|\n",
       "|       null|Wed May 25 17:29:...|              null|{[], null, [], []...|             null|                null|             0|    false|         low|null|1529514983452823555|1529514983452823555|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|         sourimagine|{null, Wed May 25...|<a href=\"http://t...|RT @sourimagine: ...|1653499763726|    false|today marks two y...|{false, Sat Aug 0...|                 null|\n",
       "|       null|Wed May 25 17:29:...|              null|{[], null, [], []...|             null|                null|             0|    false|         low|null|1529514983834456068|1529514983834456068|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|       ResisterSis20|{null, Wed May 25...|<a href=\"https://...|RT @ResisterSis20...|1653499763817|    false|No. No. No. I am ...|{false, Wed Mar 1...|                 null|\n",
       "|       null|Wed May 25 17:29:...|              null|{[], null, [], []...|             null|                null|             0|    false|         low|null|1529514983725449218|1529514983725449218|                   null|                 null|                     null|               null|                   null|           true|  en| null|              null|          0|{null, Wed May 25...|1529486533643870208| 1529486533643870208|   {twitter.com/real...|What’s drives an ...|          0|            0|       RT|         FordFischer|{null, Wed May 25...|<a href=\"http://t...|RT @FordFischer: ...|1653499763791|    false|Photos from Reddi...|{false, Sun Feb 1...|                 null|\n",
       "|       null|Wed May 25 17:29:...|              null|{[], null, [], [{...|             null|{[0, 232], {[], n...|             0|    false|         low|null|1529514983842893824|1529514983842893824|                   null|                 null|                     null|               null|                   null|           true|  en| null|              null|          0|{null, Wed May 25...|1529504180364353537| 1529504180364353537|   {twitter.com/yode...|Liberals - kids c...|          0|            0|         | 7  and we sent U...|                null|<a href=\"http://t...|You cam change yo...|1653499763819|     true|You cam change yo...|{false, Tue Jul 0...|                 null|\n",
       "+-----------+--------------------+------------------+--------------------+-----------------+--------------------+--------------+---------+------------+----+-------------------+-------------------+-----------------------+---------------------+-------------------------+-------------------+-----------------------+---------------+----+-----+------------------+-----------+--------------------+-------------------+--------------------+-----------------------+--------------------+-----------+-------------+---------+--------------------+--------------------+--------------------+--------------------+-------------+---------+--------------------+--------------------+---------------------+"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#twitter_df.limit(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4f74c417-f891-47c7-8825-a8681df94cef",
   "metadata": {},
   "outputs": [],
   "source": [
    "#twitter_df.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8a9b9c1-01d7-4043-82ff-cd9d641633b5",
   "metadata": {},
   "source": [
    "### First Filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "033279f8-df4f-4f1c-b222-98d6f17c521a",
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter_df_filt = twitter_df\\\n",
    ".withColumn('tweet_text', lower('tweet_text'))\\\n",
    ".withColumn('stripped', regexp_replace(col(\"tweet_text\"),\"[\\$#,&%\\\".]\",\"\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc0f3498-7e67-4481-83b0-a6d10d108d32",
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter_df_filt = twitter_df_filt.filter(twitter_df_filt.lang == 'en')\n",
    "twitter_df_filt.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f348a4e2-8ad9-4b4a-86d4-5c5e02a14fb3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'primary school|schools|education|k12|high school|teacher|higher secondary|higher education|senior secondary|sophomore|math|mathematics|science|physics|chemistry|biology|humanities|history|philosophy|alma mater|academia|educating|teaching|curriculum|online learning|educational|textbook|kindergarten|schooling|k-12|social-emotional learning|training|knowledge|scholarship|literacy|schooling|tuition|undergraduate|academic|course|curriculum|graduate|stem|phd|classwork|classroom|preschool|educationist|syllabus|middle school|secondary school|undergrad|book ban|student loan|student aid|school shootings|school shooting'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "edu_words = [\"primary school\",\"schools\",\"education\",\"k12\",\"high school\",\"teacher\",\"higher secondary\",\"higher education\",\n",
    "             \"senior secondary\",\"sophomore\",\"math\",\"mathematics\",\"science\",\"physics\",\"chemistry\",\"biology\",\"humanities\",\n",
    "             \"history\",\"philosophy\",\"alma mater\",\"academia\",\"educating\",\"teaching\",\"curriculum\",\"online learning\",\n",
    "             \"educational\",\"textbook\",\"kindergarten\",\"schooling\",\"k-12\",\"social-emotional learning\",\"training\",\"knowledge\",\n",
    "             \"scholarship\",\"literacy\",\"schooling\",\"tuition\",\"undergraduate\",\"academic\",\"course\",\"curriculum\",\"graduate\",\"stem\",\"phd\",\n",
    "             \"classwork\",\"classroom\",\"preschool\",\"educationist\",\"syllabus\",\"middle school\",\"secondary school\",\"undergrad\",\"book ban\",\n",
    "             \"student loan\",\"student aid\",\"school shootings\",\"school shooting\"]\n",
    "'|'.join(edu_words)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b01a90cf-8037-4559-b693-f7e03f22c3a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter_df_filt = twitter_df_filt.filter(col('tweet_text').rlike('|'.join(edu_words)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b9799e8-22bb-4362-880c-545055a83ce9",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/03/03 05:38:43 WARN org.apache.spark.deploy.yarn.YarnAllocator: Container from a bad node: container_1677802855224_0004_01_000013 on host: hub-msca-bdp-dphub-students-kshitijm-sw-6kvx.c.msca-bdp-students.internal. Exit status: 143. Diagnostics: [2023-03-03 05:38:43.937]Container killed on request. Exit code is 143\n",
      "[2023-03-03 05:38:43.941]Container exited with a non-zero exit code 143. \n",
      "[2023-03-03 05:38:43.942]Killed by external signal\n",
      ".\n",
      "23/03/03 05:38:43 WARN org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnSchedulerEndpoint: Requesting driver to remove executor 13 for reason Container from a bad node: container_1677802855224_0004_01_000013 on host: hub-msca-bdp-dphub-students-kshitijm-sw-6kvx.c.msca-bdp-students.internal. Exit status: 143. Diagnostics: [2023-03-03 05:38:43.937]Container killed on request. Exit code is 143\n",
      "[2023-03-03 05:38:43.941]Container exited with a non-zero exit code 143. \n",
      "[2023-03-03 05:38:43.942]Killed by external signal\n",
      ".\n",
      "23/03/03 05:38:43 ERROR org.apache.spark.scheduler.cluster.YarnScheduler: Lost executor 13 on hub-msca-bdp-dphub-students-kshitijm-sw-6kvx.c.msca-bdp-students.internal: Container from a bad node: container_1677802855224_0004_01_000013 on host: hub-msca-bdp-dphub-students-kshitijm-sw-6kvx.c.msca-bdp-students.internal. Exit status: 143. Diagnostics: [2023-03-03 05:38:43.937]Container killed on request. Exit code is 143\n",
      "[2023-03-03 05:38:43.941]Container exited with a non-zero exit code 143. \n",
      "[2023-03-03 05:38:43.942]Killed by external signal\n",
      ".\n",
      "23/03/03 05:38:43 WARN org.apache.spark.scheduler.TaskSetManager: Lost task 5734.0 in stage 2.0 (TID 21475) (hub-msca-bdp-dphub-students-kshitijm-sw-6kvx.c.msca-bdp-students.internal executor 13): ExecutorLostFailure (executor 13 exited caused by one of the running tasks) Reason: Container from a bad node: container_1677802855224_0004_01_000013 on host: hub-msca-bdp-dphub-students-kshitijm-sw-6kvx.c.msca-bdp-students.internal. Exit status: 143. Diagnostics: [2023-03-03 05:38:43.937]Container killed on request. Exit code is 143\n",
      "[2023-03-03 05:38:43.941]Container exited with a non-zero exit code 143. \n",
      "[2023-03-03 05:38:43.942]Killed by external signal\n",
      ".\n",
      "23/03/03 05:38:43 WARN org.apache.spark.scheduler.TaskSetManager: Lost task 5732.0 in stage 2.0 (TID 21473) (hub-msca-bdp-dphub-students-kshitijm-sw-6kvx.c.msca-bdp-students.internal executor 13): ExecutorLostFailure (executor 13 exited caused by one of the running tasks) Reason: Container from a bad node: container_1677802855224_0004_01_000013 on host: hub-msca-bdp-dphub-students-kshitijm-sw-6kvx.c.msca-bdp-students.internal. Exit status: 143. Diagnostics: [2023-03-03 05:38:43.937]Container killed on request. Exit code is 143\n",
      "[2023-03-03 05:38:43.941]Container exited with a non-zero exit code 143. \n",
      "[2023-03-03 05:38:43.942]Killed by external signal\n",
      ".\n",
      "23/03/03 05:41:06 WARN org.apache.spark.scheduler.TaskSetManager: Lost task 0.0 in stage 4.0 (TID 21484) (hub-msca-bdp-dphub-students-kshitijm-w-0.c.msca-bdp-students.internal executor 22): FetchFailed(BlockManagerId(13, hub-msca-bdp-dphub-students-kshitijm-sw-6kvx.c.msca-bdp-students.internal, 7337, None), shuffleId=0, mapIndex=0, mapId=15741, reduceId=0, message=\n",
      "org.apache.spark.shuffle.FetchFailedException: Failed to connect to hub-msca-bdp-dphub-students-kshitijm-sw-6kvx.c.msca-bdp-students.internal:7337\n",
      "\tat org.apache.spark.storage.ShuffleBlockFetcherIterator.throwFetchFailedException(ShuffleBlockFetcherIterator.scala:775)\n",
      "\tat org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:690)\n",
      "\tat org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:70)\n",
      "\tat org.apache.spark.util.CompletionIterator.next(CompletionIterator.scala:29)\n",
      "\tat scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:486)\n",
      "\tat scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:492)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)\n",
      "\tat org.apache.spark.util.CompletionIterator.hasNext(CompletionIterator.scala:31)\n",
      "\tat org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)\n",
      "\tat scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460)\n",
      "\tat org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage2.agg_doAggregateWithoutKey_0$(Unknown Source)\n",
      "\tat org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage2.processNext(Unknown Source)\n",
      "\tat org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)\n",
      "\tat org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:755)\n",
      "\tat org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:345)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898)\n",
      "\tat org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898)\n",
      "\tat org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)\n",
      "\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373)\n",
      "\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:337)\n",
      "\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)\n",
      "\tat org.apache.spark.scheduler.Task.run(Task.scala:131)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:505)\n",
      "\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)\n",
      "\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:508)\n",
      "\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n",
      "\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n",
      "\tat java.lang.Thread.run(Thread.java:750)\n",
      "Caused by: java.io.IOException: Failed to connect to hub-msca-bdp-dphub-students-kshitijm-sw-6kvx.c.msca-bdp-students.internal:7337\n",
      "\tat org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:287)\n",
      "\tat org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:218)\n",
      "\tat org.apache.spark.network.shuffle.ExternalBlockStoreClient.lambda$fetchBlocks$0(ExternalBlockStoreClient.java:105)\n",
      "\tat org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:153)\n",
      "\tat org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(RetryingBlockFetcher.java:181)\n",
      "\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)\n",
      "\tat java.util.concurrent.FutureTask.run(FutureTask.java:266)\n",
      "\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n",
      "\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n",
      "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n",
      "\t... 1 more\n",
      "Caused by: java.net.UnknownHostException: hub-msca-bdp-dphub-students-kshitijm-sw-6kvx.c.msca-bdp-students.internal\n",
      "\tat java.net.InetAddress$CachedAddresses.get(InetAddress.java:764)\n",
      "\tat java.net.InetAddress.getAllByName0(InetAddress.java:1282)\n",
      "\tat java.net.InetAddress.getAllByName(InetAddress.java:1140)\n",
      "\tat java.net.InetAddress.getAllByName(InetAddress.java:1064)\n",
      "\tat java.net.InetAddress.getByName(InetAddress.java:1014)\n",
      "\tat io.netty.util.internal.SocketUtils$8.run(SocketUtils.java:156)\n",
      "\tat io.netty.util.internal.SocketUtils$8.run(SocketUtils.java:153)\n",
      "\tat java.security.AccessController.doPrivileged(Native Method)\n",
      "\tat io.netty.util.internal.SocketUtils.addressByName(SocketUtils.java:153)\n",
      "\tat io.netty.resolver.DefaultNameResolver.doResolve(DefaultNameResolver.java:41)\n",
      "\tat io.netty.resolver.SimpleNameResolver.resolve(SimpleNameResolver.java:61)\n",
      "\tat io.netty.resolver.SimpleNameResolver.resolve(SimpleNameResolver.java:53)\n",
      "\tat io.netty.resolver.InetSocketAddressResolver.doResolve(InetSocketAddressResolver.java:55)\n",
      "\tat io.netty.resolver.InetSocketAddressResolver.doResolve(InetSocketAddressResolver.java:31)\n",
      "\tat io.netty.resolver.AbstractAddressResolver.resolve(AbstractAddressResolver.java:106)\n",
      "\tat io.netty.bootstrap.Bootstrap.doResolveAndConnect0(Bootstrap.java:206)\n",
      "\tat io.netty.bootstrap.Bootstrap.access$000(Bootstrap.java:46)\n",
      "\tat io.netty.bootstrap.Bootstrap$1.operationComplete(Bootstrap.java:180)\n",
      "\tat io.netty.bootstrap.Bootstrap$1.operationComplete(Bootstrap.java:166)\n",
      "\tat io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577)\n",
      "\tat io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:551)\n",
      "\tat io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490)\n",
      "\tat io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615)\n",
      "\tat io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:604)\n",
      "\tat io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104)\n",
      "\tat io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84)\n",
      "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.safeSetSuccess(AbstractChannel.java:984)\n",
      "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.register0(AbstractChannel.java:504)\n",
      "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.access$200(AbstractChannel.java:417)\n",
      "\tat io.netty.channel.AbstractChannel$AbstractUnsafe$1.run(AbstractChannel.java:474)\n",
      "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n",
      "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n",
      "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n",
      "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n",
      "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n",
      "\t... 2 more\n",
      "\n",
      ")\n",
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "39848346"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_df_filt.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "529c2b20-55ea-4fec-bb25-d306a6a546c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "twitter_df_filt.write.format(\"parquet\").\\\n",
    "mode('overwrite').\\\n",
    "save('gs://' + 'msca-bdp-students-bucket/shared_data/kshitijm' + '/Final_Project-FiltData0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67b58288-14f3-4cdd-8a47-a8938c29da92",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>coordinates</th><th>created_at</th><th>display_text_range</th><th>entities</th><th>extended_entities</th><th>extended_tweet</th><th>favorite_count</th><th>favorited</th><th>filter_level</th><th>geo</th><th>id</th><th>id_str</th><th>in_reply_to_screen_name</th><th>in_reply_to_status_id</th><th>in_reply_to_status_id_str</th><th>in_reply_to_user_id</th><th>in_reply_to_user_id_str</th><th>is_quote_status</th><th>lang</th><th>place</th><th>possibly_sensitive</th><th>quote_count</th><th>quoted_status</th><th>quoted_status_id</th><th>quoted_status_id_str</th><th>quoted_status_permalink</th><th>quoted_text</th><th>reply_count</th><th>retweet_count</th><th>retweeted</th><th>retweeted_from</th><th>retweeted_status</th><th>source</th><th>text</th><th>timestamp_ms</th><th>truncated</th><th>tweet_text</th><th>user</th><th>withheld_in_countries</th><th>stripped</th></tr>\n",
       "<tr><td>null</td><td>Sat Oct 22 08:14:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1583733489350111232</td><td>1583733489350111232</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>sirmobbsalot</td><td>{null, Thu Oct 20...</td><td>&lt;a href=&quot;http://t...</td><td>RT @sirmobbsalot:...</td><td>1666426462863</td><td>false</td><td>process the teach...</td><td>{false, Sun Nov 0...</td><td>null</td><td>process the teach...</td></tr>\n",
       "<tr><td>null</td><td>Sat Oct 22 08:14:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1583733538888638464</td><td>1583733538888638464</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>DrFrankRoberts</td><td>{null, Fri Oct 21...</td><td>&lt;a href=&quot;http://t...</td><td>RT @DrFrankRobert...</td><td>1666426474674</td><td>false</td><td>my rule: any stud...</td><td>{false, Thu Sep 0...</td><td>null</td><td>my rule: any stud...</td></tr>\n",
       "<tr><td>null</td><td>Sat Oct 22 08:15:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1583733735790235648</td><td>1583733735790235648</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>IAGovernor</td><td>{null, Thu Oct 20...</td><td>&lt;a href=&quot;http://t...</td><td>RT @IAGovernor: T...</td><td>1666426521619</td><td>false</td><td>the covid vaccine...</td><td>{false, Mon Jul 1...</td><td>null</td><td>the covid vaccine...</td></tr>\n",
       "<tr><td>null</td><td>Sat Oct 22 08:16:...</td><td>null</td><td>{[{[28, 43], ONEP...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1583733931161317376</td><td>1583733931161317376</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>AmericaNewsroom</td><td>{null, Wed Oct 19...</td><td>&lt;a href=&quot;http://t...</td><td>RT @AmericaNewsro...</td><td>1666426568199</td><td>false</td><td>watch: #onepillca...</td><td>{false, Sat Oct 2...</td><td>null</td><td>watch: onepillcan...</td></tr>\n",
       "<tr><td>null</td><td>Sat Oct 22 08:16:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1583734006583300097</td><td>1583734006583300097</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>MailOnline</td><td>{null, Sat Oct 22...</td><td>&lt;a href=&quot;http://t...</td><td>RT @MailOnline: H...</td><td>1666426586181</td><td>false</td><td>horrific moment f...</td><td>{false, Thu Feb 1...</td><td>null</td><td>horrific moment f...</td></tr>\n",
       "<tr><td>null</td><td>Sat Oct 22 08:16:...</td><td>[14, 140]</td><td>{[], null, [], [{...</td><td>null</td><td>{[14, 293], {[{[2...</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1583734094390665216</td><td>1583734094390665216</td><td>DmytroKuleba</td><td>1583726855819235329</td><td>1583726855819235329</td><td>2885493519</td><td>2885493519</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td></td><td>null</td><td>null</td><td>&lt;a href=&quot;http://t...</td><td>@DmytroKuleba Und...</td><td>1666426607116</td><td>true</td><td>@dmytrokuleba und...</td><td>{false, Fri Oct 0...</td><td>null</td><td>@dmytrokuleba und...</td></tr>\n",
       "<tr><td>null</td><td>Sat Oct 22 08:17:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1583734263614099457</td><td>1583734263614099457</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>adjiboussodieng</td><td>{null, Fri Oct 21...</td><td>&lt;a href=&quot;http://t...</td><td>RT @adjiboussodie...</td><td>1666426647462</td><td>false</td><td>it&#x27;s grad school ...</td><td>{false, Wed Dec 0...</td><td>null</td><td>it&#x27;s grad school ...</td></tr>\n",
       "<tr><td>null</td><td>Sat Oct 22 08:20:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1583735117549572096</td><td>1583735117549572096</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>_oluwaseun9</td><td>{null, Sat Oct 22...</td><td>&lt;a href=&quot;http://t...</td><td>RT @_oluwaseun9: ...</td><td>1666426851056</td><td>false</td><td>another big oppor...</td><td>{false, Thu Jul 0...</td><td>null</td><td>another big oppor...</td></tr>\n",
       "<tr><td>null</td><td>Sat Oct 22 08:21:...</td><td>null</td><td>{[], null, [], [{...</td><td>null</td><td>{[0, 156], {[], n...</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1583735183085166593</td><td>1583735183085166593</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>false</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td></td><td>null</td><td>null</td><td>&lt;a href=&quot;http://t...</td><td>thinking about wh...</td><td>1666426866681</td><td>true</td><td>thinking about wh...</td><td>{false, Sun Aug 0...</td><td>null</td><td>thinking about wh...</td></tr>\n",
       "<tr><td>null</td><td>Sat Oct 22 08:21:...</td><td>null</td><td>{[], null, [], [{...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1583735209052102662</td><td>1583735209052102662</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>false</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td></td><td>null</td><td>null</td><td>&lt;a href=&quot;https://...</td><td>Non Teaching Staf...</td><td>1666426872872</td><td>false</td><td>non teaching staf...</td><td>{false, Sat May 1...</td><td>null</td><td>non teaching staf...</td></tr>\n",
       "<tr><td>null</td><td>Sat Oct 22 08:21:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1583735249238134785</td><td>1583735249238134785</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>Beany_1</td><td>{null, Fri Oct 21...</td><td>&lt;a href=&quot;http://t...</td><td>RT @Beany_1: All ...</td><td>1666426882453</td><td>false</td><td>all the tories ke...</td><td>{false, Sun Oct 0...</td><td>null</td><td>all the tories ke...</td></tr>\n",
       "<tr><td>null</td><td>Sun Jan 08 17:15:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1612135938414579723</td><td>1612135938414579723</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>QasimRashid</td><td>{null, Sat Jan 07...</td><td>&lt;a href=&quot;http://t...</td><td>RT @QasimRashid: ...</td><td>1673198134412</td><td>false</td><td>“the problem is t...</td><td>{false, Wed Oct 0...</td><td>null</td><td>“the problem is t...</td></tr>\n",
       "<tr><td>null</td><td>Sun Jan 08 17:15:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1612135973047111680</td><td>1612135973047111680</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>Prathkum</td><td>{null, Sun Jan 08...</td><td>&lt;a href=&quot;http://t...</td><td>RT @Prathkum: 7 w...</td><td>1673198142669</td><td>false</td><td>7 web development...</td><td>{false, Mon Nov 2...</td><td>null</td><td>7 web development...</td></tr>\n",
       "<tr><td>null</td><td>Sun Jan 08 17:16:...</td><td>null</td><td>{[], null, [], [{...</td><td>null</td><td>{[0, 215], {[], n...</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1612136104647602176</td><td>1612136104647602176</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>false</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td></td><td>null</td><td>null</td><td>&lt;a href=&quot;https://...</td><td>Hundreds of Chica...</td><td>1673198174045</td><td>true</td><td>hundreds of chica...</td><td>{false, Wed Oct 0...</td><td>null</td><td>hundreds of chica...</td></tr>\n",
       "<tr><td>null</td><td>Sun Jan 08 17:16:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1612136160654159872</td><td>1612136160654159872</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td></td><td>null</td><td>null</td><td>&lt;a href=&quot;https://...</td><td>LMAO Indiana high...</td><td>1673198187398</td><td>false</td><td>lmao indiana high...</td><td>{false, Fri May 2...</td><td>null</td><td>lmao indiana high...</td></tr>\n",
       "<tr><td>null</td><td>Sun Jan 08 17:16:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1612136214903287808</td><td>1612136214903287808</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>Rainmaker1973</td><td>{null, Sun Jan 08...</td><td>&lt;a href=&quot;http://t...</td><td>RT @Rainmaker1973...</td><td>1673198200332</td><td>false</td><td>striped eel catfi...</td><td>{false, Tue Apr 1...</td><td>null</td><td>striped eel catfi...</td></tr>\n",
       "<tr><td>null</td><td>Sun Jan 08 17:17:...</td><td>null</td><td>{[], null, [], []...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1612136299061997568</td><td>1612136299061997568</td><td>manojsinha_</td><td>null</td><td>null</td><td>2427057066</td><td>2427057066</td><td>false</td><td>en</td><td>{{[[[74.634785, 3...</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td></td><td>null</td><td>null</td><td>&lt;a href=&quot;http://t...</td><td>@manojsinha_ dear...</td><td>1673198220397</td><td>false</td><td>@manojsinha_ dear...</td><td>{false, Sat Jul 1...</td><td>null</td><td>@manojsinha_ dear...</td></tr>\n",
       "<tr><td>null</td><td>Sun Jan 08 17:17:...</td><td>[32, 140]</td><td>{[], null, [], [{...</td><td>null</td><td>{[32, 152], {[], ...</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1612136322243911682</td><td>1612136322243911682</td><td>highbrow_nobrow</td><td>1611497912252436482</td><td>1611497912252436482</td><td>4018888580</td><td>4018888580</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td></td><td>TheNakedParty A h...</td><td>null</td><td>&lt;a href=&quot;https://...</td><td>@highbrow_nobrow ...</td><td>1673198225924</td><td>true</td><td>@highbrow_nobrow ...</td><td>{false, Mon Oct 2...</td><td>null</td><td>@highbrow_nobrow ...</td></tr>\n",
       "<tr><td>null</td><td>Sun Jan 08 17:17:...</td><td>null</td><td>{[{[17, 30], ment...</td><td>null</td><td>null</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1612136420507787264</td><td>1612136420507787264</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>null</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td>RT</td><td>withjazmine</td><td>{null, Sun Jan 08...</td><td>&lt;a href=&quot;http://t...</td><td>RT @withjazmine: ...</td><td>1673198249352</td><td>false</td><td>#mentalhealth is ...</td><td>{false, Sun Feb 1...</td><td>null</td><td>mentalhealth is i...</td></tr>\n",
       "<tr><td>null</td><td>Sun Jan 08 17:17:...</td><td>null</td><td>{[], null, [], [{...</td><td>null</td><td>{[0, 205], {[{[14...</td><td>0</td><td>false</td><td>low</td><td>null</td><td>1612136429538410496</td><td>1612136429538410496</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>en</td><td>null</td><td>false</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>0</td><td>0</td><td></td><td>null</td><td>null</td><td>&lt;a href=&quot;https://...</td><td>Riverside vs New ...</td><td>1673198251505</td><td>true</td><td>riverside vs new ...</td><td>{false, Mon Aug 2...</td><td>null</td><td>riverside vs new ...</td></tr>\n",
       "</table>\n",
       "only showing top 20 rows\n"
      ],
      "text/plain": [
       "+-----------+--------------------+------------------+--------------------+--------------------+--------------------+--------------+---------+------------+----+-------------------+-------------------+-----------------------+---------------------+-------------------------+-------------------+-----------------------+---------------+----+-----+------------------+-----------+--------------------+-------------------+--------------------+-----------------------+--------------------+-----------+-------------+---------+--------------------+--------------------+--------------------+--------------------+-------------+---------+--------------------+--------------------+---------------------+--------------------+\n",
       "|coordinates|          created_at|display_text_range|            entities|   extended_entities|      extended_tweet|favorite_count|favorited|filter_level| geo|                 id|             id_str|in_reply_to_screen_name|in_reply_to_status_id|in_reply_to_status_id_str|in_reply_to_user_id|in_reply_to_user_id_str|is_quote_status|lang|place|possibly_sensitive|quote_count|       quoted_status|   quoted_status_id|quoted_status_id_str|quoted_status_permalink|         quoted_text|reply_count|retweet_count|retweeted|      retweeted_from|    retweeted_status|              source|                text| timestamp_ms|truncated|          tweet_text|                user|withheld_in_countries|            stripped|\n",
       "+-----------+--------------------+------------------+--------------------+--------------------+--------------------+--------------+---------+------------+----+-------------------+-------------------+-----------------------+---------------------+-------------------------+-------------------+-----------------------+---------------+----+-----+------------------+-----------+--------------------+-------------------+--------------------+-----------------------+--------------------+-----------+-------------+---------+--------------------+--------------------+--------------------+--------------------+-------------+---------+--------------------+--------------------+---------------------+--------------------+\n",
       "|       null|Mon Nov 07 18:05:...|              null|{[{[19, 28], DIAA...|                null|                null|             0|    false|         low|null|1589680510288424960|1589680510288424960|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|       dianamarie265|{null, Sun Nov 06...|<a href=\"http://t...|RT @dianamarie265...|1667844343145|    false|#diaa2022 #levelo...|{false, Fri Aug 2...|                 null|diaa2022 levelone...|\n",
       "|       null|Mon Nov 07 18:05:...|         [28, 140]|{[], null, [], [{...|                null|{[28, 306], {[], ...|             0|    false|         low|null|1589680563837100032|1589680563837100032|           libsoftiktok|  1589679742433005568|      1589679742433005568|1326229737551912960|    1326229737551912960|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|         |MooreSchools 1). ...|                null|<a href=\"http://t...|@libsoftiktok @Mo...|1667844355912|     true|@libsoftiktok @mo...|{false, Thu Jun 0...|                 null|@libsoftiktok @mo...|\n",
       "|       null|Mon Nov 07 18:05:...|              null|{[], null, [], []...|                null|                null|             0|    false|         low|null|1589680578848526336|1589680578848526336|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|        RobSchneider|{null, Mon Nov 07...|<a href=\"https://...|RT @RobSchneider:...|1667844359491|    false|dear former calif...|{false, Tue Jun 3...|                 null|dear former calif...|\n",
       "|       null|Mon Nov 07 18:06:...|              null|{[], null, [], []...|                null|                null|             0|    false|         low|null|1589680624327348225|1589680624327348225|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|      piper4missouri|{null, Mon Nov 07...|<a href=\"http://t...|RT @piper4missour...|1667844370334|    false|￼when republican ...|{false, Sun Nov 2...|                 null|￼when republican ...|\n",
       "|       null|Mon Nov 07 18:06:...|          [14, 98]|{[], null, [], []...|                null|                null|             0|    false|         low|null|1589680626076377090|1589680626076377090|           dirtbagqueer|  1589680237834829826|      1589680237834829826|          467078620|              467078620|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|         |                null|                null|<a href=\"http://t...|@dirtbagqueer He ...|1667844370751|    false|@dirtbagqueer he ...|{false, Tue Sep 2...|                 null|@dirtbagqueer he ...|\n",
       "|       null|Mon Nov 07 18:06:...|              null|{[], null, [], []...|                null|                null|             0|    false|         low|null|1589680668203966466|1589680668203966466|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|           JipseeCat|{null, Mon Nov 07...|<a href=\"http://t...|RT @JipseeCat: @G...|1667844380795|    false|@govrondesantis y...|{false, Fri Aug 1...|                 null|@govrondesantis y...|\n",
       "|       null|Mon Nov 07 18:06:...|              null|{[], null, [], []...|                null|                null|             0|    false|         low|null|1589680672511496192|1589680672511496192|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|          Jim_Jordan|{null, Mon Nov 07...|<a href=\"http://t...|RT @Jim_Jordan: R...|1667844381822|    false|record inflation....|{false, Sat Apr 0...|                 null|record inflation ...|\n",
       "|       null|Mon Nov 07 18:07:...|              null|{[{[116, 127], Me...|                null|                null|             0|    false|         low|null|1589680867353722880|1589680867353722880|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|     NYTimesLearning|{null, Mon Oct 24...|<a href=\"http://t...|RT @NYTimesLearni...|1667844428276|    false|what don’t adults...|{false, Tue Feb 1...|                 null|what don’t adults...|\n",
       "|       null|Mon Nov 07 18:07:...|              null|{[{[61, 71], writ...|                null|                null|             0|    false|         low|null|1589680981786562561|1589680981786562561|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|         |                null|                null|<a href=\"https://...|History Professor...|1667844455559|    false|history professor...|{false, Sat Sep 1...|                 null|history professor...|\n",
       "|       null|Mon Nov 07 18:08:...|              null|{[], null, [], [{...|                null|{[0, 179], {[], n...|             0|    false|         low|null|1589681085461454848|1589681085461454848|                   null|                 null|                     null|               null|                   null|          false|  en| null|             false|          0|                null|               null|                null|                   null|                null|          0|            0|         |                null|                null|<a href=\"https://...|Barbara Nakles, l...|1667844480277|     true|barbara nakles, l...|{false, Thu Sep 2...|                 null|barbara nakles lo...|\n",
       "|       null|Mon Nov 07 18:08:...|              null|{[], null, [], []...|                null|                null|             0|    false|         low|null|1589681211772805120|1589681211772805120|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|          FieldYates|{null, Mon Nov 07...|<a href=\"https://...|RT @FieldYates: A...|1667844510392|    false|as best as i can ...|{false, Sun Sep 1...|                 null|as best as i can ...|\n",
       "|       null|Mon Nov 07 18:08:...|              null|{[], null, [], []...|                null|                null|             0|    false|         low|null|1589681234204315648|1589681234204315648|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|         harperlee07|{null, Mon Nov 07...|<a href=\"http://t...|RT @harperlee07: ...|1667844515740|    false|i grew up in smal...|{false, Wed Aug 2...|                 null|i grew up in smal...|\n",
       "|       null|Mon Nov 07 18:08:...|              null|{[], null, [], []...|                null|                null|             0|    false|         low|null|1589681292605804545|1589681292605804545|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|        RobSchneider|{null, Mon Nov 07...|<a href=\"https://...|RT @RobSchneider:...|1667844529664|    false|dear former calif...|{false, Tue Jun 0...|                 null|dear former calif...|\n",
       "|       null|Mon Nov 07 18:08:...|              null|{[], null, [], []...|                null|                null|             0|    false|         low|null|1589681297131126785|1589681297131126785|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|          FieldYates|{null, Mon Nov 07...|<a href=\"http://t...|RT @FieldYates: A...|1667844530743|    false|as best as i can ...|{false, Fri Jan 1...|                 null|as best as i can ...|\n",
       "|       null|Mon Nov 07 18:08:...|              null|{[], null, [], []...|                null|                null|             0|    false|         low|null|1589681306560262145|1589681306560262145|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|             afPE_PE|{null, Mon Nov 07...|<a href=\"http://t...|RT @afPE_PE: ‼️ B...|1667844532991|    false|‼️ breaking‼️\n",
       "\n",
       "up...|{false, Mon Nov 2...|                 null|‼️ breaking‼️\n",
       "\n",
       "up...|\n",
       "|       null|Mon Nov 07 18:09:...|              null|{[], null, [], []...|                null|                null|             0|    false|         low|null|1589681376798085120|1589681376798085120|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|       amandaBdennis|{null, Mon Nov 07...|<a href=\"https://...|RT @amandaBdennis...|1667844549737|    false|he's 4th gen rais...|{false, Sun Nov 0...|                 null|he's 4th gen rais...|\n",
       "|       null|Mon Nov 07 18:09:...|              null|{[], null, [], []...|                null|                null|             0|    false|         low|null|1589681410331541505|1589681410331541505|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|        libsoftiktok|{null, Mon Nov 07...|<a href=\"http://t...|RT @libsoftiktok:...|1667844557732|    false|graphic content\n",
       "\n",
       "...|{false, Thu Oct 1...|                 null|graphic content\n",
       "\n",
       "...|\n",
       "|       null|Mon Nov 07 18:09:...|          [0, 110]|{[], [{null, null...|{[{null, null, pi...|                null|             0|    false|         low|null|1589681421093830657|1589681421093830657|                   null|                 null|                     null|               null|                   null|          false|  en| null|             false|          0|                null|               null|                null|                   null|                null|          0|            0|         |                null|                null|<a href=\"http://w...|Winterized irriga...|1667844560298|    false|winterized irriga...|{false, Thu Jan 0...|                 null|winterized irriga...|\n",
       "|       null|Wed May 25 16:56:...|              null|{[], null, [], []...|                null|                null|             0|    false|         low|null|1529506599202541568|1529506599202541568|                   null|                 null|                     null|               null|                   null|          false|  en| null|              null|          0|                null|               null|                null|                   null|                null|          0|            0|       RT|         jamient1776|{null, Wed May 25...|<a href=\"http://t...|RT @jamient1776: ...|1653497764765|    false|19 little kids an...|{false, Sun Mar 0...|                 null|19 little kids an...|\n",
       "|       null|Wed May 25 16:56:...|              null|{[], null, [], []...|                null|                null|             0|    false|         low|null|1529506623869267968|1529506623869267968|                   null|                 null|                     null|               null|                   null|           true|  en| null|              null|          0|{null, Wed May 25...|1529459086152110082| 1529459086152110082|   {twitter.com/fdrl...|Tragedies Like Th...|          0|            0|       RT|           talkwordy|{null, Wed May 25...|<a href=\"http://t...|RT @talkwordy: “w...|1653497770646|    false|“what is the most...|{false, Sun Jun 0...|                 null|“what is the most...|\n",
       "+-----------+--------------------+------------------+--------------------+--------------------+--------------------+--------------+---------+------------+----+-------------------+-------------------+-----------------------+---------------------+-------------------------+-------------------+-----------------------+---------------+----+-----+------------------+-----------+--------------------+-------------------+--------------------+-----------------------+--------------------+-----------+-------------+---------+--------------------+--------------------+--------------------+--------------------+-------------+---------+--------------------+--------------------+---------------------+--------------------+\n",
       "only showing top 20 rows"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "education_text_sample = twitter_df_filt.sample(withReplacement=False, fraction=0.025, seed=12345)\n",
    "education_text_sample.limit(40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c30b89c1-e682-4d43-8ca6-1ab9e2ec61e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "996858"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "education_text_sample.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7db6e713-8a5f-4349-b838-52e9feaa6062",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|tweet_text                                                                                                                                                                                                                                                                                                      |\n",
      "+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|fun fact: in germany, homeschooling is effectively prohibited. you must send your child to an accredited school. exceptions are made for \"the travelling people\" and a few others, but even there, the state decides the curriculum.                                                                            |\n",
      "|a gunman just got apprehended in richardson texas at a high school.                                                                                                                                                                                                                                             |\n",
      "|there are over 860,000 abortions each year. the average high school houses about 850 students. that’s enough children to fill up nearly 1012 high schools to capacity that people are willingly killing each year. guns aren’t the problem. evil is.                                                            |\n",
      "|jackson christian vs. columbia academy | high school softball\n",
      "eagles vs. bulldogs\n",
      "wattch game : https://t.co/vcdryf0hvt\n",
      "columbia academy (columbia, tn) varsity softball team has a neutral playoff game vs. jackson christian (jackson, tn) today                                                              |\n",
      "|instead of more cops or arming teachers i propose we train an elite squad of heavily-armed paramility raccoons to patrol the ceilings of schools &amp; rain down upon shooters with tactical precision. this will be known as the raccoon armed battalion in case evil strikes (aka rabies)                     |\n",
      "|@foxnews how about making schools like prison! armed guards, bulletproof windows, iron gates, metal detectors, body cavity searches. this would do wonders for childrens mental health                                                                                                                          |\n",
      "|in 2022, more children have been killed in school shootings than police officers have been shot in the line of duty.                                                                                                                                                                                            |\n",
      "|26 school shootings this yr and they worried about abortions. ridiculous                                                                                                                                                                                                                                        |\n",
      "|26 school shootings this yr and they worried about abortions. ridiculous                                                                                                                                                                                                                                        |\n",
      "|at least 19 children and two adults were killed at robb elementary school in uvalde, texas, another mass shooting at a us school. here are some of the deadliest k-12 school shootings since 1999. https://t.co/utvynljiko https://t.co/gcxvlybf7q                                                              |\n",
      "|@johnrich why do we let cosplay fantasists, insurrectionist, nazis, white supremacists, conspiracy nutbags, sexual predators, and kids access to arsenals of near military grade weapons from which they annihilate schools, churches, concerts, bars and other gatherings on a nigh weekly basis?              |\n",
      "|i was in high school when columbine happened and i remember the biggest action they took was immediately banning marilyn manson and trench coats                                                                                                                                                                |\n",
      "|🧵i've reached out to wisconsin's 2022 statewide candidates to ask what policies they support to prevent shootings and other violence at schools.\n",
      "\n",
      "story tk, but i'll share responses as they come in here:                                                                                                     |\n",
      "|wild how many people have asked for a school shooting episode of the show i write. people are that deeply removed from demanding more from the politicians they've elected and are instead demanding \"entertainment.\" i can't ask \"are yall ok\" anymore because the answer is \"no.\"                             |\n",
      "|what's having school shooting episodes in shows gonna do other than desensitise people further? politicians don't give a fuck about y'all, wake up!                                                                                                                                                             |\n",
      "|persecution continues of uk mum sarah paxman for stand against covid-infested schools - world socialist web site https://t.co/cdg671pk3d                                                                                                                                                                        |\n",
      "|remember when parkland happened and the nra, through trump, tried to convince us that guns in the school room would stop gun violence? there was an officer with a gun that encountered the shooter; yet, 19 children &amp; 2 teachers still died. that officer did not stop the gunman.                        |\n",
      "|@juliettekayyem republicans block every measure to reduce school shootings.\n",
      "\n",
      "always remember that republicans have done nothing to stop school shootings since columbine.\n",
      "\n",
      "this is a policy choice by republicans to have school shootings.                                                                     |\n",
      "|today's show will be up in an hour or so\n",
      "\n",
      "any money from it will be going to the texas elementary school shooting victims fund. if you'd like to donate as well here is the link. https://t.co/rflu8ykppv                                                                                                       |\n",
      "|“you will go to college for free!”\n",
      "\n",
      "pete kadens, a chicago millionaire who retired at 40, created a charity called hope chicago that will pay for in-state college tuition, room and board, and books for 30,000 chicago high school students. https://t.co/0ikgy3hwjf https://t.co/o0d5dekfvq                  |\n",
      "|@doehlerd85 discipline and spanking doesn’t cause kids to shoot other kids. it’s teaches respect for authority. i guarantee there wouldn’t as much school shootings if parents disciplined in a living manner.                                                                                                  |\n",
      "|@aaronparnas how bout if you are disgusted with beto and heartbroken about the school shooting … but you’re a lib                                                                                                                                                                                               |\n",
      "|schools scared to death.\n",
      "the truth is, one education under desks,\n",
      "stooped low from bullets;\n",
      "that plunge when we ask\n",
      "where our children\n",
      "shall live\n",
      "&amp; how\n",
      "&amp; if                                                                                                                                            |\n",
      "|“protecting” kids from masks, from lgbtq people, from history, from getting health care but not from getting shot and killed at school.\n",
      "\n",
      "got it.                                                                                                                                                                |\n",
      "|we grieve for the 21 who were killed in texas today. but grieving is not enough—we could have, and must now, take commonsense actions to prevent these tragedies, like establishing single points of entry into schools, armed guards, trained staff, mental health services &amp; more                         |\n",
      "|texas. where the governor was more concerned about keeping masks out of schools than guns.                                                                                                                                                                                                                      |\n",
      "|evanston township high school graduate refuses to give up indigenous adornments on cap, barred from walking across stage https://t.co/1ntgqfcmkc                                                                                                                                                                |\n",
      "|if you put more effort into keeping crt out of schools than armed gunman, you are part of the problem. if you are more worried about your kids learning that gay and trans people exist, than them being gunned down and murdered, you are part of the problem.                                                 |\n",
      "|7/30/20\n",
      "a push at the federal level to remove police from schools, murphy lead the charge\n",
      "\n",
      "@chrismurphyct, along with 3 other lawmakers, introduced a bill that would pave the way to remove school resource officers &amp; replace them w/ a counseling system\n",
      "\n",
      "https://t.co/y2mmedhrih https://t.co/arcq451mmg|\n",
      "|i was in high school when columbine happened and i remember the biggest action they took was immediately banning marilyn manson and trench coats                                                                                                                                                                |\n",
      "|there is no such thing as being “pro-life” while supporting laws that let children be shot in their schools, elders in grocery stores, worshippers in their houses of faith, survivors by abusers, or anyone in a crowded place.\n",
      "\n",
      "it is an idolatry of violence. and it must end.                               |\n",
      "|@wanda41618925 @fuhknjo i agree all schools around the country need those no matter what! and more security and all gated                                                                                                                                                                                       |\n",
      "|thread of gofundmes/petitions/etc for the uvalde elementary school shooting:                                                                                                                                                                                                                                    |\n",
      "|@theonlyjfalz @mjmckean the buffalo grocery store had an armed security guard — a former cop. he’s dead. \n",
      "\n",
      "plus, the logistics of putting armed guards in 100,000 public schools in the u.s. &amp; the costs when teachers already spend their own money for basic classroom supplies make this a non-starter.  |\n",
      "|26 years ago, a gunman entered\n",
      "dunblane primary school in scotland,\n",
      "killing 16 kids and a teacher. the uk\n",
      "govt responded by enacting tight gun\n",
      "control legislation. in the 9400+ days\n",
      "since, there have been a total of o\n",
      "school shootings in the uk. #uvalde                                                   |\n",
      "|bruh i'm looking at the history of school shootings. mind-blowing, kids brought guns to school until the 70s. \n",
      "\n",
      "only 3 school mass shootings between 1903-1966.\n",
      "\n",
      "what happened?\n",
      "(didnt take off until the 90s)                                                                                                  |\n",
      "|@kyleramsvig massive strikes and walkouts are the answer. schools, stores, factories, public buildings. grind the economy to a halt once a week until legislation banning assault weapons is passed. start this monday, memorial day.                                                                           |\n",
      "|a deputy sheriff lost his daughter in the school shooting.                                                                                                                                                                                                                                                      |\n",
      "|“protecting” kids from masks, from lgbtq people, from history, from getting health care but not from getting shot and killed at school.\n",
      "\n",
      "got it.                                                                                                                                                                |\n",
      "|fourth-grade teacher eva mireles.\n",
      "she had worked for the uvalde school district for 17 years. \n",
      "\n",
      "she was reportedly shot and killed while trying to protect her students. \n",
      "\n",
      "she leaves behind a spouse and a daughter in college. https://t.co/cdsnz2v5lk                                                        |\n",
      "+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "only showing top 40 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "education_text_sample.select('tweet_text').show(40,truncate=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf167344-cba7-41e4-9b42-9442fd12deaf",
   "metadata": {},
   "source": [
    "## Filtering 2 - Removing sports words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a970a941-a765-4990-a670-2b41f742d566",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/03/03 13:14:08 WARN org.apache.spark.sql.catalyst.util.package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n"
     ]
    }
   ],
   "source": [
    "twitter_df_filt= spark.read.parquet('gs://msca-bdp-students-bucket/shared_data/kshitijm/Final_Project-FiltData0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3a3775eb-abd5-466e-9f91-b92008b53540",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_rm = [\"baseball\",\"volleyball\",\"association football\",\"varsity\",\"high school softball\",\"playoff\",\"varsity sports\",\n",
    "           \"university sports\",\"vs.\",\"college football\",\"college basketball\",\"basketball\",\"track and field\",\"swimming\",\n",
    "           \"lacrosse\",\"cricket\",\"handball\",\"professional sports\",\"ultimate frisbee\",\"frisbee\",\"sport\",\"sports\",\"sporting\",\n",
    "           \"competition\",\"competitions\",\"tournament\",\"tournaments\",\"championship\",\"championships\",\"march madness\",\n",
    "           \"ncaa\",\"national collegiate athletic association\",\"games\",\"game\",\"matches\",\"cheerleaders\",\"cheerleader\",\"mascot\",\n",
    "          \"team color\",\"athletic conference\",\"athletic conferences\",\"athletics department\"]\n",
    "\n",
    "#score, scores\n",
    "\n",
    "dict_rm1='|'.join([\"(\" + c +\")\" for c in dict_rm])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "69a0caec-d260-4518-a38e-51392324defc",
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter_df_filt2 = twitter_df_filt.where(~twitter_df_filt['tweet_text'].rlike(dict_rm1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cd4b9bd4-d634-4633-b2f3-e31e0ca224c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "36344266"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_df_filt2.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0671d294-3ad3-421e-a2bd-45fd15c06117",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 1:===================================>                  (434 + 48) / 662]\r"
     ]
    }
   ],
   "source": [
    "twitter_df_filt2.write.format(\"parquet\").\\\n",
    "mode('overwrite').\\\n",
    "save('gs://' + 'msca-bdp-students-bucket/shared_data/kshitijm' + '/Final_Project-FiltData1')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc1a27ec-53af-454c-bac2-ce64a5cb5163",
   "metadata": {},
   "source": [
    "### Saving a sample for further analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "30182c00-248d-4478-a0c6-c48749657b2f",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/03/03 13:48:35 WARN org.apache.spark.deploy.yarn.YarnAllocator: Container from a bad node: container_1677847511085_0002_01_000025 on host: hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal. Exit status: 143. Diagnostics: [2023-03-03 13:48:35.923]Container killed on request. Exit code is 143\n",
      "[2023-03-03 13:48:35.924]Container exited with a non-zero exit code 143. \n",
      "[2023-03-03 13:48:35.926]Killed by external signal\n",
      ".\n",
      "23/03/03 13:48:35 WARN org.apache.spark.deploy.yarn.YarnAllocator: Container from a bad node: container_1677847511085_0002_01_000026 on host: hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal. Exit status: 143. Diagnostics: [2023-03-03 13:48:35.924]Container killed on request. Exit code is 143\n",
      "[2023-03-03 13:48:35.926]Container exited with a non-zero exit code 143. \n",
      "[2023-03-03 13:48:35.927]Killed by external signal\n",
      ".\n",
      "23/03/03 13:48:35 WARN org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnSchedulerEndpoint: Requesting driver to remove executor 25 for reason Container from a bad node: container_1677847511085_0002_01_000025 on host: hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal. Exit status: 143. Diagnostics: [2023-03-03 13:48:35.923]Container killed on request. Exit code is 143\n",
      "[2023-03-03 13:48:35.924]Container exited with a non-zero exit code 143. \n",
      "[2023-03-03 13:48:35.926]Killed by external signal\n",
      ".\n",
      "23/03/03 13:48:35 WARN org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnSchedulerEndpoint: Requesting driver to remove executor 26 for reason Container from a bad node: container_1677847511085_0002_01_000026 on host: hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal. Exit status: 143. Diagnostics: [2023-03-03 13:48:35.924]Container killed on request. Exit code is 143\n",
      "[2023-03-03 13:48:35.926]Container exited with a non-zero exit code 143. \n",
      "[2023-03-03 13:48:35.927]Killed by external signal\n",
      ".\n",
      "23/03/03 13:48:35 ERROR org.apache.spark.scheduler.cluster.YarnScheduler: Lost executor 25 on hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal: Container from a bad node: container_1677847511085_0002_01_000025 on host: hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal. Exit status: 143. Diagnostics: [2023-03-03 13:48:35.923]Container killed on request. Exit code is 143\n",
      "[2023-03-03 13:48:35.924]Container exited with a non-zero exit code 143. \n",
      "[2023-03-03 13:48:35.926]Killed by external signal\n",
      ".\n",
      "23/03/03 13:48:35 WARN org.apache.spark.scheduler.TaskSetManager: Lost task 612.0 in stage 2.0 (TID 1275) (hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal executor 25): ExecutorLostFailure (executor 25 exited caused by one of the running tasks) Reason: Container from a bad node: container_1677847511085_0002_01_000025 on host: hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal. Exit status: 143. Diagnostics: [2023-03-03 13:48:35.923]Container killed on request. Exit code is 143\n",
      "[2023-03-03 13:48:35.924]Container exited with a non-zero exit code 143. \n",
      "[2023-03-03 13:48:35.926]Killed by external signal\n",
      ".\n",
      "23/03/03 13:48:35 WARN org.apache.spark.scheduler.TaskSetManager: Lost task 614.0 in stage 2.0 (TID 1277) (hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal executor 25): ExecutorLostFailure (executor 25 exited caused by one of the running tasks) Reason: Container from a bad node: container_1677847511085_0002_01_000025 on host: hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal. Exit status: 143. Diagnostics: [2023-03-03 13:48:35.923]Container killed on request. Exit code is 143\n",
      "[2023-03-03 13:48:35.924]Container exited with a non-zero exit code 143. \n",
      "[2023-03-03 13:48:35.926]Killed by external signal\n",
      ".\n",
      "23/03/03 13:48:35 ERROR org.apache.spark.scheduler.cluster.YarnScheduler: Lost executor 26 on hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal: Container from a bad node: container_1677847511085_0002_01_000026 on host: hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal. Exit status: 143. Diagnostics: [2023-03-03 13:48:35.924]Container killed on request. Exit code is 143\n",
      "[2023-03-03 13:48:35.926]Container exited with a non-zero exit code 143. \n",
      "[2023-03-03 13:48:35.927]Killed by external signal\n",
      ".\n",
      "23/03/03 13:48:35 WARN org.apache.spark.scheduler.TaskSetManager: Lost task 619.0 in stage 2.0 (TID 1282) (hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal executor 26): ExecutorLostFailure (executor 26 exited caused by one of the running tasks) Reason: Container from a bad node: container_1677847511085_0002_01_000026 on host: hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal. Exit status: 143. Diagnostics: [2023-03-03 13:48:35.924]Container killed on request. Exit code is 143\n",
      "[2023-03-03 13:48:35.926]Container exited with a non-zero exit code 143. \n",
      "[2023-03-03 13:48:35.927]Killed by external signal\n",
      ".\n",
      "23/03/03 13:48:35 WARN org.apache.spark.scheduler.TaskSetManager: Lost task 620.0 in stage 2.0 (TID 1283) (hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal executor 26): ExecutorLostFailure (executor 26 exited caused by one of the running tasks) Reason: Container from a bad node: container_1677847511085_0002_01_000026 on host: hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal. Exit status: 143. Diagnostics: [2023-03-03 13:48:35.924]Container killed on request. Exit code is 143\n",
      "[2023-03-03 13:48:35.926]Container exited with a non-zero exit code 143. \n",
      "[2023-03-03 13:48:35.927]Killed by external signal\n",
      ".\n",
      "23/03/03 13:48:36 WARN org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnSchedulerEndpoint: Requesting driver to remove executor 40 for reason Container marked as failed: container_1677847511085_0002_01_000040 on host: hub-msca-bdp-dphub-students-kshitijm-sw-qspf.c.msca-bdp-students.internal. Exit status: -100. Diagnostics: Container released on a *lost* node.\n",
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "twitter_df_filt2_sample = twitter_df_filt2.sample(withReplacement=False, fraction=0.025, seed=12345)\n",
    "twitter_df_filt2_sample.write.format(\"parquet\").\\\n",
    "mode('overwrite').\\\n",
    "save('gs://' + 'msca-bdp-students-bucket/shared_data/kshitijm' + '/Final_Project-FiltData1_Sample')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77763a5d-1eb9-4ef2-8a7c-0f061e6abe10",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}