NB6_Unique_BDP Final Project_Kshitijm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ac0ef949-10cf-4d41-9f62-86f9024085d4",
   "metadata": {},
   "source": [
    "## Notebook 6 - Duplication Analysis (without groups)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6e40c91c-8c51-4c06-9b4b-a8f2e62cb721",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.8.15 | packaged by conda-forge | (default, Nov 22 2022, 08:46:39) \n",
      "[GCC 10.4.0]\n",
      "3.1.3\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "print(sys.version)\n",
    "print(spark.version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e53028db-b351-4cd7-ab10-827d9f9abafc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "pd.set_option('display.max_colwidth', None)\n",
    "pd.reset_option('display.max_rows')\n",
    "from itertools import compress \n",
    "from pyspark.sql.functions import *\n",
    "from pyspark.sql.types import *\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "warnings.filterwarnings(action='ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5cb04042-11fd-424d-a342-051de225aae2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: regex in /opt/conda/miniconda3/lib/python3.8/site-packages (2021.4.4)\n",
      "Collecting regex\n",
      "  Downloading regex-2022.10.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (772 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m772.3/772.3 kB\u001b[0m \u001b[31m12.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: regex\n",
      "  Attempting uninstall: regex\n",
      "    Found existing installation: regex 2021.4.4\n",
      "    Uninstalling regex-2021.4.4:\n",
      "      Successfully uninstalled regex-2021.4.4\n",
      "Successfully installed regex-2022.10.31\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mNote: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "#pip install -U regex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d5dfa47f-5b38-4bfe-93c5-8760978cc6fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
     ]
    }
   ],
   "source": [
    "## Adding ML and NLP packages\n",
    "\n",
    "import re\n",
    "from pyspark.ml.feature import MinHashLSH\n",
    "from pyspark.ml.feature import CountVectorizer,  IDF, CountVectorizerModel, Tokenizer, RegexTokenizer, StopWordsRemover\n",
    "from pyspark import SparkContext\n",
    "from pyspark.sql import SQLContext\n",
    "from pyspark.sql import Row\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c48f9017-7c18-4e6a-ba51-f0b14e7af5b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "StopWords = stopwords.words(\"english\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bb532e0c-caae-4c85-a3b6-23d2e6ebf289",
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%config InlineBackend.figure_format = 'svg'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f77729c5-f7da-4e59-9d85-f919c05b390e",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option(\"max_colwidth\", 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2ee08f51-a4fe-4613-b5e6-e14d532cb1c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import storage\n",
    "# !pip install gcsfs --upgrade"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51e4207f-d6a0-4738-9826-c5cffa716816",
   "metadata": {},
   "source": [
    "### Setting Up Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ef2cbb3a-bdd6-406b-9614-32c1fab5415c",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark.conf.set(\"spark.sql.repl.eagerEval.enabled\",True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7478d8d2-2a95-479b-99ae-9371ac9cd9ce",
   "metadata": {},
   "source": [
    "### Accessing Filtered Database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0766fa35-95d6-4904-9909-bc9d05841345",
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter_df7= spark.read.parquet('gs://msca-bdp-students-bucket/shared_data/kshitijm/Final_Project-FiltData3')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "14b1816f-a4f8-42f4-9fad-35c7dae1ddb2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/03/08 20:40:17 WARN org.apache.spark.sql.catalyst.util.package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n",
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>id</th><th>created_at</th><th>text</th><th>tweet_text</th><th>tweet_coordinates</th><th>tweet_country</th><th>tweet_country_code</th><th>tweet_place_full_name</th><th>tweet_place_type</th><th>is_reply</th><th>is_quote</th><th>is_retweeted</th><th>reply_count</th><th>quote_count</th><th>retweet_count</th><th>favorite_count</th><th>rp_original_id</th><th>rp_user_id</th><th>rp_original_user</th><th>qu_original_id</th><th>rt_original_id</th><th>rt_original_user</th><th>account_id</th><th>account_name</th><th>account_description</th><th>account_location</th><th>is_verified</th><th>total_followers</th><th>total_friends</th><th>total_listed</th><th>total_favourites</th><th>total_tweets</th><th>account_created_at</th><th>account_profile_picture</th><th>hashtags</th><th>tweet_type</th></tr>\n",
       "<tr><td>1566378682318462976</td><td>Sun Sep 04 10:52:...</td><td>RT @C_A_Graham: H...</td><td>had another conve...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td>RT</td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1566034383559065601</td><td>C_A_Graham</td><td>483864332</td><td>erin_gilbey</td><td>Storyteller and 1...</td><td>Birmingham, England</td><td>false</td><td>442</td><td>601</td><td>8</td><td>15433</td><td>8431</td><td>Sun Feb 05 13:37:...</td><td>https://pbs.twimg...</td><td>[]</td><td>retweet</td></tr>\n",
       "<tr><td>1566378688698097664</td><td>Sun Sep 04 10:52:...</td><td>RT @Oludeewon: I ...</td><td>i hope you have s...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td>RT</td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1566165377796771840</td><td>Oludeewon</td><td>984182548791164928</td><td>Odeshiye</td><td>♎\n",
       "...So A Child M...</td><td>Ghana</td><td>false</td><td>4902</td><td>4613</td><td>3</td><td>149571</td><td>92052</td><td>Wed Apr 11 21:32:...</td><td>https://pbs.twimg...</td><td>[]</td><td>retweet</td></tr>\n",
       "<tr><td>1566378715709427719</td><td>Sun Sep 04 10:52:...</td><td>RT @JackPosobiec:...</td><td>breaking: @trinit...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td>RT</td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1565431877460189189</td><td>JackPosobiec</td><td>1558859689789644800</td><td>EuroAmerican87</td><td>American by birth...</td><td>Anywhere but here...</td><td>false</td><td>6</td><td>34</td><td>1</td><td>50</td><td>89</td><td>Sun Aug 14 16:54:...</td><td>https://pbs.twimg...</td><td>[]</td><td>retweet</td></tr>\n",
       "<tr><td>1566378716887916546</td><td>Sun Sep 04 10:52:...</td><td>RT @amishra77: Th...</td><td>the appalling con...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td>RT</td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1566378309520596993</td><td>amishra77</td><td>1279643153474224130</td><td>Bharath12450842</td><td>One of the 900 mi...</td><td>null</td><td>false</td><td>193</td><td>502</td><td>0</td><td>6244</td><td>9146</td><td>Sun Jul 05 05:09:...</td><td>https://pbs.twimg...</td><td>[]</td><td>retweet</td></tr>\n",
       "<tr><td>1566378724135555072</td><td>Sun Sep 04 10:52:...</td><td>RT @imzarvinnn: S...</td><td>senior high schoo...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td>RT</td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1566034947831660545</td><td>imzarvinnn</td><td>1530893925803622401</td><td>lvchdvmp</td><td>I&#x27;m counting on y...</td><td>archen&#x27;s heart</td><td>false</td><td>341</td><td>458</td><td>0</td><td>11632</td><td>5624</td><td>Sun May 29 12:49:...</td><td>https://pbs.twimg...</td><td>[]</td><td>retweet</td></tr>\n",
       "<tr><td>1566378734529101824</td><td>Sun Sep 04 10:52:...</td><td>RT @imzarvinnn: S...</td><td>senior high schoo...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td>RT</td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1566034947831660545</td><td>imzarvinnn</td><td>1398282464854827012</td><td>doyiea_</td><td>she/her          ...</td><td>null</td><td>false</td><td>1</td><td>91</td><td>0</td><td>59</td><td>621</td><td>Fri May 28 14:18:...</td><td>https://pbs.twimg...</td><td>[]</td><td>retweet</td></tr>\n",
       "<tr><td>1566378741508734976</td><td>Sun Sep 04 10:52:...</td><td>You can also tag ...</td><td>you can also tag ...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>true</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>1566378738438533120</td><td>1299666479344807937</td><td>thxteacher</td><td>null</td><td>null</td><td>null</td><td>1299666479344807937</td><td>thxteacher</td><td>Our Teachers are ...</td><td>null</td><td>false</td><td>902</td><td>69</td><td>1</td><td>325</td><td>1257</td><td>Sat Aug 29 11:13:...</td><td>https://pbs.twimg...</td><td>[TeachersDay, Tha...</td><td>reply</td></tr>\n",
       "<tr><td>1566378743710744576</td><td>Sun Sep 04 10:52:...</td><td>RT @cutedevil5934...</td><td>&quot;teachers play a ...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td>RT</td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1566366649912938498</td><td>cutedevil5934</td><td>769516988335067137</td><td>shoaibAlam28</td><td>http://Instagram....</td><td>Jamui, India</td><td>false</td><td>85</td><td>146</td><td>0</td><td>1434</td><td>5245</td><td>Sat Aug 27 12:48:...</td><td>https://pbs.twimg...</td><td>[]</td><td>retweet</td></tr>\n",
       "<tr><td>1566378745539469312</td><td>Sun Sep 04 10:52:...</td><td>@jenqissante ! 🧚...</td><td>@jenqissante ! 🧚...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>true</td><td>true</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>1566373182810234882</td><td>1547185772708909056</td><td>jenqissante</td><td>1496397166339932162</td><td>null</td><td>null</td><td>1307586299893702656</td><td>PaperupPh</td><td>more than 1k+ wor...</td><td>10AM-10PM</td><td>false</td><td>520</td><td>182</td><td>2</td><td>95</td><td>820</td><td>Sun Sep 20 07:44:...</td><td>https://pbs.twimg...</td><td>[]</td><td>quote</td></tr>\n",
       "<tr><td>1566378750090027008</td><td>Sun Sep 04 10:52:...</td><td>RT @SethAbramson:...</td><td>like... one of hi...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td>RT</td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1566236400785264640</td><td>SethAbramson</td><td>1269070500082450437</td><td>ResisterNell</td><td>Activist, #Resist...</td><td>Kentucky </td><td>false</td><td>9745</td><td>10379</td><td>16</td><td>463255</td><td>333552</td><td>Sat Jun 06 00:57:...</td><td>https://pbs.twimg...</td><td>[]</td><td>retweet</td></tr>\n",
       "<tr><td>1566378757585248257</td><td>Sun Sep 04 10:52:...</td><td>@MayraFlores2022 ...</td><td>@mayraflores2022 ...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>true</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>1566050202410164224</td><td>89469296</td><td>MayraFlores2022</td><td>null</td><td>null</td><td>null</td><td>190656345</td><td>bmhochberg</td><td>Retired from IT. ...</td><td>Marlboro, NJ</td><td>false</td><td>274</td><td>147</td><td>2</td><td>2279</td><td>13104</td><td>Tue Sep 14 14:24:...</td><td>https://pbs.twimg...</td><td>[]</td><td>reply</td></tr>\n",
       "<tr><td>1566378766691188736</td><td>Sun Sep 04 10:52:...</td><td>RT @divbyzero: My...</td><td>my son, a first-y...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td>RT</td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1566198145897451521</td><td>divbyzero</td><td>2656346922</td><td>FraserProf</td><td>academic economis...</td><td>Birmingham/Leicester</td><td>false</td><td>454</td><td>433</td><td>2</td><td>15686</td><td>10789</td><td>Fri Jul 18 09:49:...</td><td>https://pbs.twimg...</td><td>[]</td><td>retweet</td></tr>\n",
       "<tr><td>1566378775038029824</td><td>Sun Sep 04 10:52:...</td><td>@sachi04100 ! 🧚 ...</td><td>@sachi04100 ! 🧚 ...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>true</td><td>true</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>1566060522809675776</td><td>1555157163072712704</td><td>sachi04100</td><td>1496397166339932162</td><td>null</td><td>null</td><td>1307586299893702656</td><td>PaperupPh</td><td>more than 1k+ wor...</td><td>10AM-10PM</td><td>false</td><td>520</td><td>182</td><td>2</td><td>95</td><td>821</td><td>Sun Sep 20 07:44:...</td><td>https://pbs.twimg...</td><td>[]</td><td>quote</td></tr>\n",
       "<tr><td>1566378776396763136</td><td>Sun Sep 04 10:52:...</td><td>RT @Kimmie219: Gu...</td><td>guess what&#x27;s back...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>true</td><td>RT</td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>1566173018178609155</td><td>1566217351674171393</td><td>Kimmie219</td><td>1556987805678682112</td><td>MsGerner_3</td><td>✨3rd Grade teache...</td><td>NJ</td><td>false</td><td>259</td><td>387</td><td>1</td><td>2032</td><td>2362</td><td>Tue Aug 09 12:56:...</td><td>https://pbs.twimg...</td><td>[]</td><td>retweet</td></tr>\n",
       "<tr><td>1566378788845506560</td><td>Sun Sep 04 10:52:...</td><td>Today english edu...</td><td>today english edu...</td><td>null</td><td>Poland</td><td>PL</td><td>Wroclaw, Poland</td><td>city</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1464093317965230080</td><td>BabichZarzycka</td><td>Aristocrat</td><td>Wrocław, Polska</td><td>false</td><td>43</td><td>168</td><td>0</td><td>359</td><td>2507</td><td>Fri Nov 26 04:47:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+-------------------+--------------------+--------------------+--------------------+-----------------+-------------+------------------+---------------------+----------------+--------+--------+------------+-----------+-----------+-------------+--------------+-------------------+-------------------+----------------+-------------------+-------------------+----------------+-------------------+---------------+--------------------+--------------------+-----------+---------------+-------------+------------+----------------+------------+--------------------+-----------------------+--------------------+--------------+\n",
       "|                 id|          created_at|                text|          tweet_text|tweet_coordinates|tweet_country|tweet_country_code|tweet_place_full_name|tweet_place_type|is_reply|is_quote|is_retweeted|reply_count|quote_count|retweet_count|favorite_count|     rp_original_id|         rp_user_id|rp_original_user|     qu_original_id|     rt_original_id|rt_original_user|         account_id|   account_name| account_description|    account_location|is_verified|total_followers|total_friends|total_listed|total_favourites|total_tweets|  account_created_at|account_profile_picture|            hashtags|    tweet_type|\n",
       "+-------------------+--------------------+--------------------+--------------------+-----------------+-------------+------------------+---------------------+----------------+--------+--------+------------+-----------+-----------+-------------+--------------+-------------------+-------------------+----------------+-------------------+-------------------+----------------+-------------------+---------------+--------------------+--------------------+-----------+---------------+-------------+------------+----------------+------------+--------------------+-----------------------+--------------------+--------------+\n",
       "|1566378682318462976|Sun Sep 04 10:52:...|RT @C_A_Graham: H...|had another conve...|             null|         null|              null|                 null|            null|   false|   false|          RT|          0|          0|            0|             0|               null|               null|            null|               null|1566034383559065601|      C_A_Graham|          483864332|    erin_gilbey|Storyteller and 1...| Birmingham, England|      false|            442|          601|           8|           15433|        8431|Sun Feb 05 13:37:...|   https://pbs.twimg...|                  []|       retweet|\n",
       "|1566378688698097664|Sun Sep 04 10:52:...|RT @Oludeewon: I ...|i hope you have s...|             null|         null|              null|                 null|            null|   false|   false|          RT|          0|          0|            0|             0|               null|               null|            null|               null|1566165377796771840|       Oludeewon| 984182548791164928|       Odeshiye|♎\n",
       "...So A Child M...|               Ghana|      false|           4902|         4613|           3|          149571|       92052|Wed Apr 11 21:32:...|   https://pbs.twimg...|                  []|       retweet|\n",
       "|1566378715709427719|Sun Sep 04 10:52:...|RT @JackPosobiec:...|breaking: @trinit...|             null|         null|              null|                 null|            null|   false|   false|          RT|          0|          0|            0|             0|               null|               null|            null|               null|1565431877460189189|    JackPosobiec|1558859689789644800| EuroAmerican87|American by birth...|Anywhere but here...|      false|              6|           34|           1|              50|          89|Sun Aug 14 16:54:...|   https://pbs.twimg...|                  []|       retweet|\n",
       "|1566378716887916546|Sun Sep 04 10:52:...|RT @amishra77: Th...|the appalling con...|             null|         null|              null|                 null|            null|   false|   false|          RT|          0|          0|            0|             0|               null|               null|            null|               null|1566378309520596993|       amishra77|1279643153474224130|Bharath12450842|One of the 900 mi...|                null|      false|            193|          502|           0|            6244|        9146|Sun Jul 05 05:09:...|   https://pbs.twimg...|                  []|       retweet|\n",
       "|1566378724135555072|Sun Sep 04 10:52:...|RT @imzarvinnn: S...|senior high schoo...|             null|         null|              null|                 null|            null|   false|   false|          RT|          0|          0|            0|             0|               null|               null|            null|               null|1566034947831660545|      imzarvinnn|1530893925803622401|       lvchdvmp|I'm counting on y...|      archen's heart|      false|            341|          458|           0|           11632|        5624|Sun May 29 12:49:...|   https://pbs.twimg...|                  []|       retweet|\n",
       "|1566378734529101824|Sun Sep 04 10:52:...|RT @imzarvinnn: S...|senior high schoo...|             null|         null|              null|                 null|            null|   false|   false|          RT|          0|          0|            0|             0|               null|               null|            null|               null|1566034947831660545|      imzarvinnn|1398282464854827012|        doyiea_|she/her          ...|                null|      false|              1|           91|           0|              59|         621|Fri May 28 14:18:...|   https://pbs.twimg...|                  []|       retweet|\n",
       "|1566378741508734976|Sun Sep 04 10:52:...|You can also tag ...|you can also tag ...|             null|         null|              null|                 null|            null|    true|   false|            |          0|          0|            0|             0|1566378738438533120|1299666479344807937|      thxteacher|               null|               null|            null|1299666479344807937|     thxteacher|Our Teachers are ...|                null|      false|            902|           69|           1|             325|        1257|Sat Aug 29 11:13:...|   https://pbs.twimg...|[TeachersDay, Tha...|         reply|\n",
       "|1566378743710744576|Sun Sep 04 10:52:...|RT @cutedevil5934...|\"teachers play a ...|             null|         null|              null|                 null|            null|   false|   false|          RT|          0|          0|            0|             0|               null|               null|            null|               null|1566366649912938498|   cutedevil5934| 769516988335067137|   shoaibAlam28|http://Instagram....|        Jamui, India|      false|             85|          146|           0|            1434|        5245|Sat Aug 27 12:48:...|   https://pbs.twimg...|                  []|       retweet|\n",
       "|1566378745539469312|Sun Sep 04 10:52:...|@jenqissante ! 🧚...|@jenqissante ! 🧚...|             null|         null|              null|                 null|            null|    true|    true|            |          0|          0|            0|             0|1566373182810234882|1547185772708909056|     jenqissante|1496397166339932162|               null|            null|1307586299893702656|      PaperupPh|more than 1k+ wor...|           10AM-10PM|      false|            520|          182|           2|              95|         820|Sun Sep 20 07:44:...|   https://pbs.twimg...|                  []|         quote|\n",
       "|1566378750090027008|Sun Sep 04 10:52:...|RT @SethAbramson:...|like... one of hi...|             null|         null|              null|                 null|            null|   false|   false|          RT|          0|          0|            0|             0|               null|               null|            null|               null|1566236400785264640|    SethAbramson|1269070500082450437|   ResisterNell|Activist, #Resist...|           Kentucky |      false|           9745|        10379|          16|          463255|      333552|Sat Jun 06 00:57:...|   https://pbs.twimg...|                  []|       retweet|\n",
       "|1566378757585248257|Sun Sep 04 10:52:...|@MayraFlores2022 ...|@mayraflores2022 ...|             null|         null|              null|                 null|            null|    true|   false|            |          0|          0|            0|             0|1566050202410164224|           89469296| MayraFlores2022|               null|               null|            null|          190656345|     bmhochberg|Retired from IT. ...|        Marlboro, NJ|      false|            274|          147|           2|            2279|       13104|Tue Sep 14 14:24:...|   https://pbs.twimg...|                  []|         reply|\n",
       "|1566378766691188736|Sun Sep 04 10:52:...|RT @divbyzero: My...|my son, a first-y...|             null|         null|              null|                 null|            null|   false|   false|          RT|          0|          0|            0|             0|               null|               null|            null|               null|1566198145897451521|       divbyzero|         2656346922|     FraserProf|academic economis...|Birmingham/Leicester|      false|            454|          433|           2|           15686|       10789|Fri Jul 18 09:49:...|   https://pbs.twimg...|                  []|       retweet|\n",
       "|1566378775038029824|Sun Sep 04 10:52:...|@sachi04100 ! 🧚 ...|@sachi04100 ! 🧚 ...|             null|         null|              null|                 null|            null|    true|    true|            |          0|          0|            0|             0|1566060522809675776|1555157163072712704|      sachi04100|1496397166339932162|               null|            null|1307586299893702656|      PaperupPh|more than 1k+ wor...|           10AM-10PM|      false|            520|          182|           2|              95|         821|Sun Sep 20 07:44:...|   https://pbs.twimg...|                  []|         quote|\n",
       "|1566378776396763136|Sun Sep 04 10:52:...|RT @Kimmie219: Gu...|guess what's back...|             null|         null|              null|                 null|            null|   false|    true|          RT|          0|          0|            0|             0|               null|               null|            null|1566173018178609155|1566217351674171393|       Kimmie219|1556987805678682112|     MsGerner_3|✨3rd Grade teache...|                  NJ|      false|            259|          387|           1|            2032|        2362|Tue Aug 09 12:56:...|   https://pbs.twimg...|                  []|       retweet|\n",
       "|1566378788845506560|Sun Sep 04 10:52:...|Today english edu...|today english edu...|             null|       Poland|                PL|      Wroclaw, Poland|            city|   false|   false|            |          0|          0|            0|             0|               null|               null|            null|               null|               null|            null|1464093317965230080| BabichZarzycka|          Aristocrat|     Wrocław, Polska|      false|             43|          168|           0|             359|        2507|Fri Nov 26 04:47:...|   https://pbs.twimg...|                  []|original_tweet|\n",
       "+-------------------+--------------------+--------------------+--------------------+-----------------+-------------+------------------+---------------------+----------------+--------+--------+------------+-----------+-----------+-------------+--------------+-------------------+-------------------+----------------+-------------------+-------------------+----------------+-------------------+---------------+--------------------+--------------------+-----------+---------------+-------------+------------+----------------+------------+--------------------+-----------------------+--------------------+--------------+"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_df7.limit(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "85097067-0e45-4361-a87e-fc3072ee4584",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>id</th><th>created_at</th><th>text</th><th>tweet_text</th><th>tweet_coordinates</th><th>tweet_country</th><th>tweet_country_code</th><th>tweet_place_full_name</th><th>tweet_place_type</th><th>is_reply</th><th>is_quote</th><th>is_retweeted</th><th>reply_count</th><th>quote_count</th><th>retweet_count</th><th>favorite_count</th><th>rp_original_id</th><th>rp_user_id</th><th>rp_original_user</th><th>qu_original_id</th><th>rt_original_id</th><th>rt_original_user</th><th>account_id</th><th>account_name</th><th>account_description</th><th>account_location</th><th>is_verified</th><th>total_followers</th><th>total_friends</th><th>total_listed</th><th>total_favourites</th><th>total_tweets</th><th>account_created_at</th><th>account_profile_picture</th><th>hashtags</th><th>tweet_type</th></tr>\n",
       "<tr><td>1539335994981330944</td><td>Tue Jun 21 19:54:...</td><td>Andy Vermaut shar...</td><td>andy vermaut shar...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>283604227</td><td>AndyVermaut</td><td>Human Rights Jour...</td><td>Diksmuide, België</td><td>false</td><td>31047</td><td>34144</td><td>57</td><td>1904</td><td>1774810</td><td>Sun Apr 17 16:39:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td></tr>\n",
       "<tr><td>1539336023624359936</td><td>Tue Jun 21 19:54:...</td><td>The second NIXLA ...</td><td>the second nixla ...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>116000946</td><td>DAWPHD</td><td>A scholar/leader/...</td><td>Atlanta, GA</td><td>false</td><td>1533</td><td>264</td><td>13</td><td>883</td><td>1387</td><td>Sat Feb 20 19:23:...</td><td>https://pbs.twimg...</td><td>[DEI]</td><td>original_tweet</td></tr>\n",
       "<tr><td>1539336046755864584</td><td>Tue Jun 21 19:54:...</td><td>Matching with my ...</td><td>matching with my ...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>348565639</td><td>bekah_thehedge</td><td>Travel Photograph...</td><td>TX-AR-🇨🇳-901-TX</td><td>false</td><td>424</td><td>407</td><td>2</td><td>25548</td><td>6077</td><td>Thu Aug 04 16:55:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td></tr>\n",
       "<tr><td>1539336063554138116</td><td>Tue Jun 21 19:54:...</td><td>OFFICIALLY ENROLL...</td><td>officially enroll...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>94268196</td><td>Luvv_mrs_brii</td><td>(She/Her)\n",
       "🇨🇦🇯?...</td><td>Canada</td><td>false</td><td>16</td><td>81</td><td>0</td><td>1255</td><td>846</td><td>Thu Dec 03 06:02:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td></tr>\n",
       "<tr><td>1539336104733720576</td><td>Tue Jun 21 19:55:...</td><td>The Biomere West ...</td><td>the biomere west ...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1500920481628753923</td><td>BiomereCRO</td><td>Biomere is a prem...</td><td>null</td><td>false</td><td>91</td><td>87</td><td>0</td><td>34</td><td>80</td><td>Mon Mar 07 19:45:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td></tr>\n",
       "<tr><td>1539336132399353856</td><td>Tue Jun 21 19:55:...</td><td>Wishing all our p...</td><td>wishing all our p...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1258354571635802113</td><td>StirlingPE</td><td>Stirling Council ...</td><td>null</td><td>false</td><td>184</td><td>110</td><td>0</td><td>707</td><td>357</td><td>Thu May 07 11:16:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td></tr>\n",
       "<tr><td>1539336141286961154</td><td>Tue Jun 21 19:55:...</td><td>This means ALL re...</td><td>this means all re...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1070914240280584192</td><td>KrisA37039217</td><td>#ModernaVaccine \n",
       "...</td><td>Metro Denver</td><td>false</td><td>409</td><td>678</td><td>1</td><td>20411</td><td>22584</td><td>Fri Dec 07 05:33:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td></tr>\n",
       "<tr><td>1539336157443538944</td><td>Tue Jun 21 19:55:...</td><td>Supreme Court rul...</td><td>supreme court rul...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1452760304161525765</td><td>ArOkTxNm1</td><td>The Constitution ...</td><td>null</td><td>false</td><td>999</td><td>2323</td><td>2</td><td>19761</td><td>35426</td><td>Mon Oct 25 22:13:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td></tr>\n",
       "<tr><td>1539336158848720898</td><td>Tue Jun 21 19:55:...</td><td>someone on one of...</td><td>someone on one of...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>259002403</td><td>ashconway2</td><td>#RTR</td><td>everywhere like s...</td><td>false</td><td>519</td><td>486</td><td>2</td><td>18951</td><td>28795</td><td>Mon Feb 28 23:35:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td></tr>\n",
       "<tr><td>1539336165010067459</td><td>Tue Jun 21 19:55:...</td><td>The final Middle ...</td><td>the final middle ...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1169343518172766209</td><td>MGS_Chemistry</td><td>Tweets by The Man...</td><td>Manchester, England</td><td>false</td><td>263</td><td>198</td><td>0</td><td>119</td><td>217</td><td>Wed Sep 04 20:16:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td></tr>\n",
       "<tr><td>1539336176066252804</td><td>Tue Jun 21 19:55:...</td><td>Supreme Court say...</td><td>supreme court say...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1476165846057644034</td><td>usasharenews</td><td>usa share news</td><td>United States</td><td>false</td><td>321</td><td>997</td><td>2</td><td>0</td><td>306179</td><td>Wed Dec 29 12:19:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td></tr>\n",
       "<tr><td>1539336197683785731</td><td>Tue Jun 21 19:55:...</td><td>Double Dip Today!...</td><td>double dip today!...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>59794037</td><td>ElmhurstU_FB</td><td>The official acco...</td><td>Elmhurst, IL</td><td>false</td><td>6402</td><td>652</td><td>19</td><td>2412</td><td>3610</td><td>Fri Jul 24 14:21:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td></tr>\n",
       "<tr><td>1539336237743542276</td><td>Tue Jun 21 19:55:...</td><td>I’ve had my stint...</td><td>i’ve had my stint...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1365170569386348544</td><td>Daftpunkster32</td><td>22, Tx. (CRH “The...</td><td>lurking in your m...</td><td>false</td><td>17</td><td>111</td><td>0</td><td>3334</td><td>725</td><td>Fri Feb 26 05:23:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td></tr>\n",
       "<tr><td>1539336255611080705</td><td>Tue Jun 21 19:55:...</td><td>triplane over qua...</td><td>triplane over qua...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1079997377807769600</td><td>realxtime</td><td>i watched a lot o...</td><td>arc-v pit</td><td>false</td><td>1702</td><td>1650</td><td>36</td><td>88346</td><td>144612</td><td>Tue Jan 01 07:06:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td></tr>\n",
       "<tr><td>1539336281196462083</td><td>Tue Jun 21 19:55:...</td><td>Last Friday EHS m...</td><td>last friday ehs m...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>104473859</td><td>edinahistory</td><td>Where history is ...</td><td>Edina, MN</td><td>false</td><td>603</td><td>258</td><td>24</td><td>60</td><td>1600</td><td>Wed Jan 13 12:54:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+-------------------+--------------------+--------------------+--------------------+-----------------+-------------+------------------+---------------------+----------------+--------+--------+------------+-----------+-----------+-------------+--------------+--------------+------------------+----------------+--------------+--------------+----------------+-------------------+---------------+--------------------+--------------------+-----------+---------------+-------------+------------+----------------+------------+--------------------+-----------------------+--------------------+--------------+\n",
       "|                 id|          created_at|                text|          tweet_text|tweet_coordinates|tweet_country|tweet_country_code|tweet_place_full_name|tweet_place_type|is_reply|is_quote|is_retweeted|reply_count|quote_count|retweet_count|favorite_count|rp_original_id|        rp_user_id|rp_original_user|qu_original_id|rt_original_id|rt_original_user|         account_id|   account_name| account_description|    account_location|is_verified|total_followers|total_friends|total_listed|total_favourites|total_tweets|  account_created_at|account_profile_picture|            hashtags|    tweet_type|\n",
       "+-------------------+--------------------+--------------------+--------------------+-----------------+-------------+------------------+---------------------+----------------+--------+--------+------------+-----------+-----------+-------------+--------------+--------------+------------------+----------------+--------------+--------------+----------------+-------------------+---------------+--------------------+--------------------+-----------+---------------+-------------+------------+----------------+------------+--------------------+-----------------------+--------------------+--------------+\n",
       "|1566378788845506560|Sun Sep 04 10:52:...|Today english edu...|today english edu...|             null|       Poland|                PL|      Wroclaw, Poland|            city|   false|   false|            |          0|          0|            0|             0|          null|              null|            null|          null|          null|            null|1464093317965230080| BabichZarzycka|          Aristocrat|     Wrocław, Polska|      false|             43|          168|           0|             359|        2507|Fri Nov 26 04:47:...|   https://pbs.twimg...|                  []|original_tweet|\n",
       "|1566378804402323458|Sun Sep 04 10:53:...|@ceo_karnataka @E...|@ceo_karnataka @e...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|882836250054602752|   ceo_karnataka|          null|          null|            null|1004869458991636481|     aqeeltweet|A simple Indian ?...|     Gulbarga, India|      false|           3695|         4286|           0|            3135|        3105|Thu Jun 07 23:35:...|   https://pbs.twimg...|                  []|original_tweet|\n",
       "|1566378865068498947|Sun Sep 04 10:53:...|Resistances Osada...|resistances osada...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|              null|            null|          null|          null|            null|         2298538838|      fahmiswag|Academics accurac...|المملكة العربية ا...|      false|             18|           77|           0|              13|        2604|Sat Jan 18 21:20:...|   https://pbs.twimg...|                  []|original_tweet|\n",
       "|1566379116017901568|Sun Sep 04 10:54:...|PSC 4400 Methodis...|psc 4400 methodis...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|              null|            null|          null|          null|            null| 879496394691805184|   EssayPaperUK|I AM The Documentary|      Long Beach, CA|      false|            763|         2505|           1|             334|      212343|Tue Jun 27 00:27:...|   https://pbs.twimg...|                  []|original_tweet|\n",
       "|1566379138956423169|Sun Sep 04 10:54:...|something about h...|something about h...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|              null|            null|          null|          null|            null|1414908966845370371|        usualsy|lee seyoung, namg...|          6 ongoings|      false|            820|          389|           9|            6628|       22758|Tue Jul 13 11:26:...|   https://pbs.twimg...|                  []|original_tweet|\n",
       "|1566379279440650240|Sun Sep 04 10:54:...|Man charged in ab...|man charged in ab...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|              null|            null|          null|          null|            null|1328063334172794882|    OlumoRocktv|Global News & Ent...|             Ireland|      false|            316|          323|           0|              28|       15580|Sun Nov 15 19:52:...|   https://pbs.twimg...|                  []|original_tweet|\n",
       "|1566379420201410560|Sun Sep 04 10:55:...|Tips on How to Av...|tips on how to av...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|              null|            null|          null|          null|            null| 889327405126397957|getthatrightgtr|Helping You Know ...|              Global|      false|            755|          422|           9|             660|      632488|Mon Jul 24 03:32:...|   https://pbs.twimg...|[education, learn...|original_tweet|\n",
       "|1566379426945835009|Sun Sep 04 10:55:...|thats my emotiona...|thats my emotiona...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|              null|            null|          null|          null|            null|1555246771944701957| suppiluliumass|for legal reasons...|                null|      false|             21|          160|           0|             208|          70|Thu Aug 04 17:38:...|   https://pbs.twimg...|                  []|original_tweet|\n",
       "|1566379643413880837|Sun Sep 04 10:56:...|With Prof. Lisa B...|with prof. lisa b...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|              null|            null|          null|          null|            null|          158387764|ShaikhaBinjasim|You can find me a...|              Kuwait|       true|          76796|          659|         143|            6833|       35252|Tue Jun 22 14:33:...|   https://pbs.twimg...|                  []|original_tweet|\n",
       "|1566379715438321666|Sun Sep 04 10:56:...|let me listen to ...|let me listen to ...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|              null|            null|          null|          null|            null| 803857296409427968|     gotsvnteen|s/her, 199x | hea...|     exo got7 svt ts|      false|           1096|          366|           9|           54611|       54101|Wed Nov 30 07:04:...|   https://pbs.twimg...|                  []|original_tweet|\n",
       "|1566379736321908738|Sun Sep 04 10:56:...|Our teacher unive...|our teacher unive...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|              null|            null|          null|          null|            null|         2628139479| Dominicpaul771|Adversity is the ...|            السعودية|      false|            110|           92|           0|               1|         366|Sat Jun 21 10:12:...|   https://pbs.twimg...|                  []|original_tweet|\n",
       "|1566379789405093891|Sun Sep 04 10:56:...|COMPARE AND CONTR...|compare and contr...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|              null|            null|          null|          null|            null|1372970774546485253|      USAGirl47|EXTREME-ULTRA-NUC...|                 USA|      false|          33491|        17909|          18|           33597|       19343|Fri Mar 19 17:58:...|   https://pbs.twimg...|                  []|original_tweet|\n",
       "|1566379790457872386|Sun Sep 04 10:56:...|if i had a one wi...|if i had a one wi...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|              null|            null|          null|          null|            null|1564944569644597249|JustChillinH3r3|            ask 4 dm|                 Ari|      false|              8|           81|           0|               8|           7|Wed Aug 31 11:54:...|   https://pbs.twimg...|                  []|original_tweet|\n",
       "|1566379863224770561|Sun Sep 04 10:57:...|Roman Lightning I...|roman lightning i...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|              null|            null|          null|          null|            null|          280670454|  TheMagentaFox|Accelerators Acco...|المملكة العربية ا...|      false|              2|            3|           0|               7|        6459|Mon Apr 11 20:18:...|   https://pbs.twimg...|                  []|original_tweet|\n",
       "|1566379933471051777|Sun Sep 04 10:57:...|We had an educati...|we had an educati...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|              null|            null|          null|          null|            null|1482388453841113095|       grecopke|Greener Communiti...|                null|      false|             35|           13|           0|              12|          28|Sat Jan 15 16:25:...|   https://pbs.twimg...|           [kajiado]|original_tweet|\n",
       "+-------------------+--------------------+--------------------+--------------------+-----------------+-------------+------------------+---------------------+----------------+--------+--------+------------+-----------+-----------+-------------+--------------+--------------+------------------+----------------+--------------+--------------+----------------+-------------------+---------------+--------------------+--------------------+-----------+---------------+-------------+------------+----------------+------------+--------------------+-----------------------+--------------------+--------------+"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filtering for only tweets\n",
    "twitter_df7_tweets=twitter_df7.filter(col('tweet_type')=='original_tweet')\n",
    "twitter_df7_tweets.limit(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "728bf566-0e94-46c5-a2f1-d948794166be",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3847631"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Out of 27 million data entries, we only have ~3.8 million new tweets\n",
    "twitter_df7_tweets.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af097aa4-1b74-466d-b2e8-ec4d7b678ef4",
   "metadata": {},
   "source": [
    "### Analyzing Posts from verified users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e9dc6003-a503-47bb-a96c-4662d24a3ec9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/03/09 00:15:12 WARN org.apache.spark.sql.catalyst.util.package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n",
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>id</th><th>created_at</th><th>text</th><th>tweet_text</th><th>tweet_coordinates</th><th>tweet_country</th><th>tweet_country_code</th><th>tweet_place_full_name</th><th>tweet_place_type</th><th>is_reply</th><th>is_quote</th><th>is_retweeted</th><th>reply_count</th><th>quote_count</th><th>retweet_count</th><th>favorite_count</th><th>rp_original_id</th><th>rp_user_id</th><th>rp_original_user</th><th>qu_original_id</th><th>rt_original_id</th><th>rt_original_user</th><th>account_id</th><th>account_name</th><th>account_description</th><th>account_location</th><th>is_verified</th><th>total_followers</th><th>total_friends</th><th>total_listed</th><th>total_favourites</th><th>total_tweets</th><th>account_created_at</th><th>account_profile_picture</th><th>hashtags</th><th>tweet_type</th><th>is_user_govt</th><th>is_user_news_outlet</th><th>is_user_news_journalist</th><th>is_user_edu</th><th>is_user_research</th><th>is_influencer</th><th>user_type</th></tr>\n",
       "<tr><td>1539335995845402624</td><td>Tue Jun 21 19:54:...</td><td>RT @NSEAOnline: P...</td><td>public dollars sh...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>true</td><td>RT</td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>1539262288053833728</td><td>1539316518202138624</td><td>NSEAOnline</td><td>22789766</td><td>NEAToday</td><td>official account ...</td><td>Washington DC</td><td>true</td><td>247738</td><td>11834</td><td>4148</td><td>8048</td><td>27669</td><td>Wed Mar 04 15:51:...</td><td>https://pbs.twimg...</td><td>[]</td><td>retweet</td><td>false</td><td>false</td><td>false</td><td>true</td><td>false</td><td>social_media_infl...</td><td>Educational_Inst</td></tr>\n",
       "<tr><td>1539336083649048577</td><td>Tue Jun 21 19:54:...</td><td>RT @JasonBedrick:...</td><td>@wajahatali what ...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td>RT</td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1539289355449028608</td><td>JasonBedrick</td><td>640893</td><td>EWErickson</td><td>you can listen ev...</td><td>Atlanta, GA</td><td>true</td><td>205722</td><td>2700</td><td>4633</td><td>1384</td><td>14273</td><td>Mon Jan 15 22:54:...</td><td>https://pbs.twimg...</td><td>[]</td><td>retweet</td><td>false</td><td>false</td><td>false</td><td>false</td><td>false</td><td>social_media_infl...</td><td>Influencer</td></tr>\n",
       "<tr><td>1539336297814319104</td><td>Tue Jun 21 19:55:...</td><td>RT @itsSpencerBro...</td><td>scotus handed a b...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td>RT</td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1539328635613630464</td><td>itsSpencerBrown</td><td>48459553</td><td>KatiePavlich</td><td>proud american wo...</td><td>On Instagram/Off ...</td><td>true</td><td>947092</td><td>2009</td><td>4472</td><td>18188</td><td>55802</td><td>Thu Jun 18 19:29:...</td><td>https://pbs.twimg...</td><td>[]</td><td>retweet</td><td>false</td><td>false</td><td>false</td><td>false</td><td>false</td><td>social_media_infl...</td><td>Influencer</td></tr>\n",
       "<tr><td>1539336371936051200</td><td>Tue Jun 21 19:56:...</td><td>One of Louisiana&#x27;...</td><td>one of louisiana&#x27;...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>44426926</td><td>FOX8NOLA</td><td>wvue - fox 8 is y...</td><td>New Orleans</td><td>true</td><td>217913</td><td>2853</td><td>1241</td><td>1350</td><td>207940</td><td>Wed Jun 03 18:57:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td><td>false</td><td>true</td><td>false</td><td>false</td><td>false</td><td>social_media_infl...</td><td>News_Outlet</td></tr>\n",
       "<tr><td>1539336396971859968</td><td>Tue Jun 21 19:56:...</td><td>@hales_ted But th...</td><td>@hales_ted but th...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>true</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>1539335979336470528</td><td>1445063896725819398</td><td>hales_ted</td><td>null</td><td>null</td><td>null</td><td>738538436</td><td>AoDespair</td><td>author journalist...</td><td>Baltimore, MD</td><td>true</td><td>334054</td><td>610</td><td>2732</td><td>5818</td><td>97431</td><td>Sun Aug 05 12:53:...</td><td>https://pbs.twimg...</td><td>[]</td><td>reply</td><td>false</td><td>true</td><td>true</td><td>false</td><td>false</td><td>social_media_infl...</td><td>News_Journalist</td></tr>\n",
       "<tr><td>1539336533509132289</td><td>Tue Jun 21 19:56:...</td><td>Congratulations t...</td><td>congratulations t...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>true</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>1539329196379541509</td><td>16240525</td><td>ASBMB</td><td>null</td><td>null</td><td>null</td><td>16240525</td><td>ASBMB</td><td>american society ...</td><td>Rockville, Maryland</td><td>true</td><td>24963</td><td>4259</td><td>0</td><td>14868</td><td>40004</td><td>Thu Sep 11 14:05:...</td><td>https://pbs.twimg...</td><td>[]</td><td>reply</td><td>false</td><td>false</td><td>false</td><td>false</td><td>true</td><td>micro_influencer</td><td>Research_Inst</td></tr>\n",
       "<tr><td>1539336550730842115</td><td>Tue Jun 21 19:56:...</td><td>.@JimGrossmanAHA ...</td><td>.@jimgrossmanaha ...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>76944176</td><td>AHAhistorians</td><td>the american hist...</td><td>Washington, D.C.</td><td>true</td><td>55451</td><td>2362</td><td>1105</td><td>8000</td><td>32842</td><td>Thu Sep 24 13:54:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td><td>false</td><td>false</td><td>false</td><td>false</td><td>false</td><td>social_media_infl...</td><td>Influencer</td></tr>\n",
       "<tr><td>1539336562814623745</td><td>Tue Jun 21 19:56:...</td><td>RT @FOX8NOLA: One...</td><td>one of louisiana&#x27;...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td>RT</td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1539336371936051200</td><td>FOX8NOLA</td><td>30950114</td><td>jessecbrooks</td><td>digital content p...</td><td>New Orleans, LA</td><td>true</td><td>3442</td><td>1741</td><td>71</td><td>46045</td><td>82432</td><td>Mon Apr 13 20:23:...</td><td>https://pbs.twimg...</td><td>[]</td><td>retweet</td><td>false</td><td>false</td><td>true</td><td>false</td><td>false</td><td>not_influencer</td><td>News_Journalist</td></tr>\n",
       "<tr><td>1539336614966599681</td><td>Tue Jun 21 19:57:...</td><td>Graduation 2022: ...</td><td>graduation 2022: ...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>16477702</td><td>syracusedotcom</td><td>syracuse and cent...</td><td>Syracuse, NY</td><td>true</td><td>102116</td><td>589</td><td>829</td><td>190</td><td>227817</td><td>Sat Sep 27 00:17:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td><td>false</td><td>true</td><td>false</td><td>false</td><td>false</td><td>social_media_infl...</td><td>News_Outlet</td></tr>\n",
       "<tr><td>1539336707958616068</td><td>Tue Jun 21 19:57:...</td><td>@UConnBSB Great n...</td><td>@uconnbsb great n...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>true</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>1539332632739123200</td><td>491571741</td><td>UConnBSB</td><td>null</td><td>null</td><td>null</td><td>3836570273</td><td>jdesnoyersabc6</td><td>chief meteorologi...</td><td>Providence, RI</td><td>true</td><td>3743</td><td>1091</td><td>56</td><td>31024</td><td>26502</td><td>Thu Oct 01 20:26:...</td><td>https://pbs.twimg...</td><td>[Bitter, HookC]</td><td>reply</td><td>false</td><td>true</td><td>true</td><td>false</td><td>false</td><td>not_influencer</td><td>News_Journalist</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+-------------------+--------------------+--------------------+--------------------+-----------------+-------------+------------------+---------------------+----------------+--------+--------+------------+-----------+-----------+-------------+--------------+-------------------+-------------------+----------------+-------------------+-------------------+----------------+----------+--------------+--------------------+--------------------+-----------+---------------+-------------+------------+----------------+------------+--------------------+-----------------------+---------------+--------------+------------+-------------------+-----------------------+-----------+----------------+--------------------+----------------+\n",
       "|                 id|          created_at|                text|          tweet_text|tweet_coordinates|tweet_country|tweet_country_code|tweet_place_full_name|tweet_place_type|is_reply|is_quote|is_retweeted|reply_count|quote_count|retweet_count|favorite_count|     rp_original_id|         rp_user_id|rp_original_user|     qu_original_id|     rt_original_id|rt_original_user|account_id|  account_name| account_description|    account_location|is_verified|total_followers|total_friends|total_listed|total_favourites|total_tweets|  account_created_at|account_profile_picture|       hashtags|    tweet_type|is_user_govt|is_user_news_outlet|is_user_news_journalist|is_user_edu|is_user_research|       is_influencer|       user_type|\n",
       "+-------------------+--------------------+--------------------+--------------------+-----------------+-------------+------------------+---------------------+----------------+--------+--------+------------+-----------+-----------+-------------+--------------+-------------------+-------------------+----------------+-------------------+-------------------+----------------+----------+--------------+--------------------+--------------------+-----------+---------------+-------------+------------+----------------+------------+--------------------+-----------------------+---------------+--------------+------------+-------------------+-----------------------+-----------+----------------+--------------------+----------------+\n",
       "|1539335995845402624|Tue Jun 21 19:54:...|RT @NSEAOnline: P...|public dollars sh...|             null|         null|              null|                 null|            null|   false|    true|          RT|          0|          0|            0|             0|               null|               null|            null|1539262288053833728|1539316518202138624|      NSEAOnline|  22789766|      NEAToday|official account ...|       Washington DC|       true|         247738|        11834|        4148|            8048|       27669|Wed Mar 04 15:51:...|   https://pbs.twimg...|             []|       retweet|       false|              false|                  false|       true|           false|social_media_infl...|Educational_Inst|\n",
       "|1539336083649048577|Tue Jun 21 19:54:...|RT @JasonBedrick:...|@wajahatali what ...|             null|         null|              null|                 null|            null|   false|   false|          RT|          0|          0|            0|             0|               null|               null|            null|               null|1539289355449028608|    JasonBedrick|    640893|    EWErickson|you can listen ev...|         Atlanta, GA|       true|         205722|         2700|        4633|            1384|       14273|Mon Jan 15 22:54:...|   https://pbs.twimg...|             []|       retweet|       false|              false|                  false|      false|           false|social_media_infl...|      Influencer|\n",
       "|1539336297814319104|Tue Jun 21 19:55:...|RT @itsSpencerBro...|scotus handed a b...|             null|         null|              null|                 null|            null|   false|   false|          RT|          0|          0|            0|             0|               null|               null|            null|               null|1539328635613630464| itsSpencerBrown|  48459553|  KatiePavlich|proud american wo...|On Instagram/Off ...|       true|         947092|         2009|        4472|           18188|       55802|Thu Jun 18 19:29:...|   https://pbs.twimg...|             []|       retweet|       false|              false|                  false|      false|           false|social_media_infl...|      Influencer|\n",
       "|1539336371936051200|Tue Jun 21 19:56:...|One of Louisiana'...|one of louisiana'...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|               null|               null|            null|               null|               null|            null|  44426926|      FOX8NOLA|wvue - fox 8 is y...|         New Orleans|       true|         217913|         2853|        1241|            1350|      207940|Wed Jun 03 18:57:...|   https://pbs.twimg...|             []|original_tweet|       false|               true|                  false|      false|           false|social_media_infl...|     News_Outlet|\n",
       "|1539336396971859968|Tue Jun 21 19:56:...|@hales_ted But th...|@hales_ted but th...|             null|         null|              null|                 null|            null|    true|   false|            |          0|          0|            0|             0|1539335979336470528|1445063896725819398|       hales_ted|               null|               null|            null| 738538436|     AoDespair|author journalist...|       Baltimore, MD|       true|         334054|          610|        2732|            5818|       97431|Sun Aug 05 12:53:...|   https://pbs.twimg...|             []|         reply|       false|               true|                   true|      false|           false|social_media_infl...| News_Journalist|\n",
       "|1539336533509132289|Tue Jun 21 19:56:...|Congratulations t...|congratulations t...|             null|         null|              null|                 null|            null|    true|   false|            |          0|          0|            0|             0|1539329196379541509|           16240525|           ASBMB|               null|               null|            null|  16240525|         ASBMB|american society ...| Rockville, Maryland|       true|          24963|         4259|           0|           14868|       40004|Thu Sep 11 14:05:...|   https://pbs.twimg...|             []|         reply|       false|              false|                  false|      false|            true|    micro_influencer|   Research_Inst|\n",
       "|1539336550730842115|Tue Jun 21 19:56:...|.@JimGrossmanAHA ...|.@jimgrossmanaha ...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|               null|               null|            null|               null|               null|            null|  76944176| AHAhistorians|the american hist...|    Washington, D.C.|       true|          55451|         2362|        1105|            8000|       32842|Thu Sep 24 13:54:...|   https://pbs.twimg...|             []|original_tweet|       false|              false|                  false|      false|           false|social_media_infl...|      Influencer|\n",
       "|1539336562814623745|Tue Jun 21 19:56:...|RT @FOX8NOLA: One...|one of louisiana'...|             null|         null|              null|                 null|            null|   false|   false|          RT|          0|          0|            0|             0|               null|               null|            null|               null|1539336371936051200|        FOX8NOLA|  30950114|  jessecbrooks|digital content p...|     New Orleans, LA|       true|           3442|         1741|          71|           46045|       82432|Mon Apr 13 20:23:...|   https://pbs.twimg...|             []|       retweet|       false|              false|                   true|      false|           false|      not_influencer| News_Journalist|\n",
       "|1539336614966599681|Tue Jun 21 19:57:...|Graduation 2022: ...|graduation 2022: ...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|               null|               null|            null|               null|               null|            null|  16477702|syracusedotcom|syracuse and cent...|        Syracuse, NY|       true|         102116|          589|         829|             190|      227817|Sat Sep 27 00:17:...|   https://pbs.twimg...|             []|original_tweet|       false|               true|                  false|      false|           false|social_media_infl...|     News_Outlet|\n",
       "|1539336707958616068|Tue Jun 21 19:57:...|@UConnBSB Great n...|@uconnbsb great n...|             null|         null|              null|                 null|            null|    true|   false|            |          0|          0|            0|             0|1539332632739123200|          491571741|        UConnBSB|               null|               null|            null|3836570273|jdesnoyersabc6|chief meteorologi...|      Providence, RI|       true|           3743|         1091|          56|           31024|       26502|Thu Oct 01 20:26:...|   https://pbs.twimg...|[Bitter, HookC]|         reply|       false|               true|                   true|      false|           false|      not_influencer| News_Journalist|\n",
       "+-------------------+--------------------+--------------------+--------------------+-----------------+-------------+------------------+---------------------+----------------+--------+--------+------------+-----------+-----------+-------------+--------------+-------------------+-------------------+----------------+-------------------+-------------------+----------------+----------+--------------+--------------------+--------------------+-----------+---------------+-------------+------------+----------------+------------+--------------------+-----------------------+---------------+--------------+------------+-------------------+-----------------------+-----------+----------------+--------------------+----------------+"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_df7_ver=spark.read.parquet('gs://msca-bdp-students-bucket/shared_data/kshitijm/Final_Project-FiltData4_Ver_Users_2')\n",
    "twitter_df7_ver.limit(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4705d741-1def-4234-b41e-32c1ea395ec1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "577459"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_df7_ver.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "91f6824d-8845-40cc-ac20-953ab49e2160",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>tweet_type</th><th>count</th></tr>\n",
       "<tr><td>retweet</td><td>221923</td></tr>\n",
       "<tr><td>reply</td><td>59486</td></tr>\n",
       "<tr><td>quote</td><td>23552</td></tr>\n",
       "<tr><td>original_tweet</td><td>272498</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+--------------+------+\n",
       "|    tweet_type| count|\n",
       "+--------------+------+\n",
       "|       retweet|221923|\n",
       "|         reply| 59486|\n",
       "|         quote| 23552|\n",
       "|original_tweet|272498|\n",
       "+--------------+------+"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_df7_ver.select('tweet_type').groupBy('tweet_type').count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0ab5dec-4ae8-463a-83b8-9e497cdc1da7",
   "metadata": {},
   "source": [
    "#### We have 270K original tweets from our verified users, we will look at these"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9305848f-24b7-47e8-8eb6-d0d99b20d746",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>id</th><th>created_at</th><th>text</th><th>tweet_text</th><th>tweet_coordinates</th><th>tweet_country</th><th>tweet_country_code</th><th>tweet_place_full_name</th><th>tweet_place_type</th><th>is_reply</th><th>is_quote</th><th>is_retweeted</th><th>reply_count</th><th>quote_count</th><th>retweet_count</th><th>favorite_count</th><th>rp_original_id</th><th>rp_user_id</th><th>rp_original_user</th><th>qu_original_id</th><th>rt_original_id</th><th>rt_original_user</th><th>account_id</th><th>account_name</th><th>account_description</th><th>account_location</th><th>is_verified</th><th>total_followers</th><th>total_friends</th><th>total_listed</th><th>total_favourites</th><th>total_tweets</th><th>account_created_at</th><th>account_profile_picture</th><th>hashtags</th><th>tweet_type</th><th>is_user_govt</th><th>is_user_news_outlet</th><th>is_user_news_journalist</th><th>is_user_edu</th><th>is_user_research</th><th>is_influencer</th><th>user_type</th></tr>\n",
       "<tr><td>1584203846662127618</td><td>Sun Oct 23 15:23:...</td><td>Nice turnout at t...</td><td>nice turnout at t...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>757862089</td><td>NMalliotakis</td><td>us congress membe...</td><td>New York City</td><td>true</td><td>68778</td><td>12164</td><td>633</td><td>10832</td><td>25607</td><td>Tue Aug 14 20:45:...</td><td>https://pbs.twimg...</td><td>[StatenIsland]</td><td>original_tweet</td><td>true</td><td>false</td><td>false</td><td>false</td><td>false</td><td>social_media_infl...</td><td>Government</td></tr>\n",
       "<tr><td>1584204146000830464</td><td>Sun Oct 23 15:24:...</td><td>Valentine, who te...</td><td>valentine, who te...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>15145138</td><td>ABC13News</td><td>edward r murrow a...</td><td>Lynchburg, VA</td><td>true</td><td>50108</td><td>547</td><td>606</td><td>9035</td><td>185124</td><td>Tue Jun 17 12:47:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td><td>false</td><td>true</td><td>false</td><td>false</td><td>false</td><td>social_media_infl...</td><td>News_Outlet</td></tr>\n",
       "<tr><td>1584204941312548865</td><td>Sun Oct 23 15:27:...</td><td>Wedaga Empowers F...</td><td>wedaga empowers f...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>310483039</td><td>news_ghana</td><td>ghana’s leading o...</td><td>Accra, Ghana</td><td>true</td><td>39745</td><td>4007</td><td>289</td><td>6181</td><td>293561</td><td>Fri Jun 03 20:41:...</td><td>https://pbs.twimg...</td><td>[Education, donates]</td><td>original_tweet</td><td>false</td><td>true</td><td>false</td><td>false</td><td>false</td><td>micro_influencer</td><td>News_Outlet</td></tr>\n",
       "<tr><td>1584205037625962496</td><td>Sun Oct 23 15:28:...</td><td>I learned everyth...</td><td>i learned everyth...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>2763699105</td><td>SamirTabar</td><td>i help run a publ...</td><td>New York, USA</td><td>true</td><td>1233</td><td>404</td><td>15</td><td>626</td><td>426</td><td>Sun Sep 07 05:48:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td><td>false</td><td>true</td><td>false</td><td>false</td><td>false</td><td>not_influencer</td><td>News_Outlet</td></tr>\n",
       "<tr><td>1539223109445861377</td><td>Tue Jun 21 12:26:...</td><td>Forget School Cho...</td><td>forget school cho...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>10168082</td><td>Heritage</td><td>devoted to the pr...</td><td>Washington, DC</td><td>true</td><td>639460</td><td>25929</td><td>10173</td><td>9889</td><td>115026</td><td>Mon Nov 12 01:50:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td><td>true</td><td>false</td><td>false</td><td>false</td><td>false</td><td>social_media_infl...</td><td>Government</td></tr>\n",
       "<tr><td>1539224132151287808</td><td>Tue Jun 21 12:30:...</td><td>Legislators shoul...</td><td>legislators shoul...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>14304170</td><td>PennLive</td><td>a hub mixing what...</td><td>Pennsylvania, USA</td><td>true</td><td>133770</td><td>915</td><td>1087</td><td>5106</td><td>276823</td><td>Fri Apr 04 16:39:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td><td>false</td><td>true</td><td>false</td><td>false</td><td>false</td><td>social_media_infl...</td><td>News_Outlet</td></tr>\n",
       "<tr><td>1539224144377790468</td><td>Tue Jun 21 12:30:...</td><td>State legislators...</td><td>state legislators...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>14323759</td><td>PatriotNews</td><td>news headlines fr...</td><td>Harrisburg, PA</td><td>true</td><td>62366</td><td>128</td><td>835</td><td>163</td><td>135873</td><td>Mon Apr 07 15:21:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td><td>false</td><td>true</td><td>false</td><td>false</td><td>false</td><td>social_media_infl...</td><td>News_Outlet</td></tr>\n",
       "<tr><td>1614410283602018308</td><td>Sat Jan 14 23:53:...</td><td>Law professor Der...</td><td>law professor der...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>3301029022</td><td>The74</td><td>the 74 is a non-p...</td><td>New York, NY</td><td>true</td><td>58506</td><td>324</td><td>885</td><td>819</td><td>50407</td><td>Wed May 27 22:20:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td><td>false</td><td>true</td><td>false</td><td>true</td><td>true</td><td>social_media_infl...</td><td>Educational_Inst</td></tr>\n",
       "<tr><td>1590113435232223235</td><td>Tue Nov 08 22:46:...</td><td>Voters will get t...</td><td>voters will get t...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>18456975</td><td>WVVA</td><td>wvva offers local...</td><td>Bluefield, West V...</td><td>true</td><td>11901</td><td>394</td><td>280</td><td>256</td><td>45017</td><td>Mon Dec 29 19:28:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td><td>false</td><td>true</td><td>false</td><td>false</td><td>false</td><td>micro_influencer</td><td>News_Outlet</td></tr>\n",
       "<tr><td>1590114100423852033</td><td>Tue Nov 08 22:48:...</td><td>St. Johns County ...</td><td>st. johns county ...</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>false</td><td>false</td><td></td><td>0</td><td>0</td><td>0</td><td>0</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>263481613</td><td>BachmanANjax</td><td>action news jax i...</td><td>Jacksonville, FL</td><td>true</td><td>13559</td><td>2929</td><td>215</td><td>36648</td><td>47434</td><td>Thu Mar 10 03:37:...</td><td>https://pbs.twimg...</td><td>[]</td><td>original_tweet</td><td>false</td><td>true</td><td>false</td><td>false</td><td>false</td><td>micro_influencer</td><td>News_Outlet</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+-------------------+--------------------+--------------------+--------------------+-----------------+-------------+------------------+---------------------+----------------+--------+--------+------------+-----------+-----------+-------------+--------------+--------------+----------+----------------+--------------+--------------+----------------+----------+---------------+--------------------+----------------+-----------+---------------+-------------+------------+----------------+------------+--------------------+-----------------------+--------+--------------+------------+-------------------+-----------------------+-----------+----------------+--------------------+----------------+\n",
       "|                 id|          created_at|                text|          tweet_text|tweet_coordinates|tweet_country|tweet_country_code|tweet_place_full_name|tweet_place_type|is_reply|is_quote|is_retweeted|reply_count|quote_count|retweet_count|favorite_count|rp_original_id|rp_user_id|rp_original_user|qu_original_id|rt_original_id|rt_original_user|account_id|   account_name| account_description|account_location|is_verified|total_followers|total_friends|total_listed|total_favourites|total_tweets|  account_created_at|account_profile_picture|hashtags|    tweet_type|is_user_govt|is_user_news_outlet|is_user_news_journalist|is_user_edu|is_user_research|       is_influencer|       user_type|\n",
       "+-------------------+--------------------+--------------------+--------------------+-----------------+-------------+------------------+---------------------+----------------+--------+--------+------------+-----------+-----------+-------------+--------------+--------------+----------+----------------+--------------+--------------+----------------+----------+---------------+--------------------+----------------+-----------+---------------+-------------+------------+----------------+------------+--------------------+-----------------------+--------+--------------+------------+-------------------+-----------------------+-----------+----------------+--------------------+----------------+\n",
       "|1539336371936051200|Tue Jun 21 19:56:...|One of Louisiana'...|one of louisiana'...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|      null|            null|          null|          null|            null|  44426926|       FOX8NOLA|wvue - fox 8 is y...|     New Orleans|       true|         217913|         2853|        1241|            1350|      207940|Wed Jun 03 18:57:...|   https://pbs.twimg...|      []|original_tweet|       false|               true|                  false|      false|           false|social_media_infl...|     News_Outlet|\n",
       "|1539336550730842115|Tue Jun 21 19:56:...|.@JimGrossmanAHA ...|.@jimgrossmanaha ...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|      null|            null|          null|          null|            null|  76944176|  AHAhistorians|the american hist...|Washington, D.C.|       true|          55451|         2362|        1105|            8000|       32842|Thu Sep 24 13:54:...|   https://pbs.twimg...|      []|original_tweet|       false|              false|                  false|      false|           false|social_media_infl...|      Influencer|\n",
       "|1539336614966599681|Tue Jun 21 19:57:...|Graduation 2022: ...|graduation 2022: ...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|      null|            null|          null|          null|            null|  16477702| syracusedotcom|syracuse and cent...|    Syracuse, NY|       true|         102116|          589|         829|             190|      227817|Sat Sep 27 00:17:...|   https://pbs.twimg...|      []|original_tweet|       false|               true|                  false|      false|           false|social_media_infl...|     News_Outlet|\n",
       "|1539336871049846784|Tue Jun 21 19:58:...|New York families...|new york families...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|      null|            null|          null|          null|            null|1668256296|         wealth|helping you manag...|            null|       true|          80526|         1530|         895|             185|       28156|Tue Aug 13 16:57:...|   https://pbs.twimg...|      []|original_tweet|       false|               true|                  false|      false|           false|social_media_infl...|     News_Outlet|\n",
       "|1527065374851743745|Wed May 18 23:15:...|🎓 The @SamChampi...|🎓 the @samchampi...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|      null|            null|          null|          null|            null|  72585833|      BoerneISD|award-winning sch...|      Boerne, TX|       true|           6366|          377|          93|           24375|       20355|Tue Sep 08 15:52:...|   https://pbs.twimg...|      []|original_tweet|        true|               true|                  false|       true|           false|    micro_influencer|Educational_Inst|\n",
       "|1527065651810078720|Wed May 18 23:16:...|Day 1 of State Tr...|day 1 of state tr...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|      null|            null|          null|          null|            null| 743900947|NicoleWeavingTV|jersey girl | pro...|     Lincoln, NE|       true|           1163|         1108|          12|           20860|        8022|Tue Aug 07 22:42:...|   https://pbs.twimg...|      []|original_tweet|       false|               true|                   true|       true|            true|      not_influencer|Educational_Inst|\n",
       "|1530319226434723840|Fri May 27 22:45:...|Texas School Shoo...|texas school shoo...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|      null|            null|          null|          null|            null|  22630309|  InsideEdition|inside edition is...|            null|       true|          89968|          721|         603|            7446|       59806|Tue Mar 03 14:49:...|   https://pbs.twimg...|      []|original_tweet|       false|               true|                  false|      false|           false|social_media_infl...|     News_Outlet|\n",
       "|1530319227332317185|Fri May 27 22:45:...|After a gunman ki...|after a gunman ki...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|      null|            null|          null|          null|            null|   8940342|   HoustonChron|news headlines an...|     Houston, TX|       true|         669193|          156|        5231|             444|      307465|Mon Sep 17 22:42:...|   https://pbs.twimg...|      []|original_tweet|       false|               true|                   true|      false|           false|social_media_infl...| News_Journalist|\n",
       "|1530319424510513154|Fri May 27 22:45:...|Glorious. A press...|glorious. a press...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|      null|            null|          null|          null|            null| 108557666| ClaraDFMarques|columnist  editor...|       Singapore|       true|           5637|          781|         138|            5546|        8119|Tue Jan 26 09:15:...|   https://pbs.twimg...|      []|original_tweet|       false|              false|                   true|      false|           false|    micro_influencer| News_Journalist|\n",
       "|1530319692539338752|Fri May 27 22:47:...|Live updates: Tex...|live updates: tex...|             null|         null|              null|                 null|            null|   false|   false|            |          0|          0|            0|             0|          null|      null|            null|          null|          null|            null|  24963237|     Matt_Kiser|bespoke political...|     Seattle, WA|       true|          13348|         1637|         428|            9813|         788|Tue Mar 17 21:52:...|   https://pbs.twimg...|      []|original_tweet|        true|               true|                  false|      false|           false|    micro_influencer|      Government|\n",
       "+-------------------+--------------------+--------------------+--------------------+-----------------+-------------+------------------+---------------------+----------------+--------+--------+------------+-----------+-----------+-------------+--------------+--------------+----------+----------------+--------------+--------------+----------------+----------+---------------+--------------------+----------------+-----------+---------------+-------------+------------+----------------+------------+--------------------+-----------------------+--------+--------------+------------+-------------------+-----------------------+-----------+----------------+--------------------+----------------+"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_df7_ver_orig=twitter_df7_ver.filter(col('tweet_type')=='original_tweet')\n",
    "twitter_df7_ver_orig.limit(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d79324f1-2a71-4cda-a029-301f45b7f938",
   "metadata": {},
   "source": [
    "### Segregating Tweet Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e96ab94f-befd-467b-86c9-051efa7e0638",
   "metadata": {},
   "outputs": [],
   "source": [
    "tweet_text = twitter_df7_ver_orig.rdd.map(lambda x : x['tweet_text']).filter(lambda x: x is not None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf652aad-a46c-4d7b-9e29-2714b085c1e4",
   "metadata": {},
   "source": [
    "### Taking a sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "397332fa-1826-4546-8f14-d0e9c65bf3d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PythonRDD[36] at RDD at PythonRDD.scala:53"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweet_text_sample = tweet_text.sample(withReplacement=False, fraction=0.05, seed=12345)\n",
    "tweet_text_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "07494fcf-bc01-493c-8d95-68ae4be8b3e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "tokens = tweet_text_sample\\\n",
    "    .map( lambda document: document.strip().lower())\\\n",
    "    .map( lambda document: re.split(\" \", document))\\\n",
    "    .map( lambda word: [x for x in word if x.isalnum()])\\\n",
    "    .map( lambda word: [x for x in word if len(x) > 3] )\\\n",
    "    .map( lambda word: [x for x in word if x not in StopWords])\\\n",
    "    .zipWithIndex()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c450681b-54c9-4721-aadb-f6234eff4bbc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 16:>                                                         (0 + 1) / 1]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+---+\n",
      "|          tweet_text| id|\n",
      "+--------------------+---+\n",
      "|{playing pretend?...|  0|\n",
      "|{it's national vo...|  1|\n",
      "|{a school distric...|  2|\n",
      "|{a student presen...|  3|\n",
      "|{pennsylvania hig...|  4|\n",
      "+--------------------+---+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "row = Row('tweet_text_sample')\n",
    "df_tweet_text_sample=tweet_text_sample.map(row).zipWithIndex().toDF(['tweet_text','id'])\n",
    "df_tweet_text_sample.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7eaab7ae-2504-4cd7-8b7a-60785857e893",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "13594"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tweet_text_sample.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2bbeed6c-56f0-4905-bcce-d0fc41e32244",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 21:>                                                         (0 + 1) / 1]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---+\n",
      "|list_of_words                                                                                                                                                                        |id |\n",
      "+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---+\n",
      "|[playing, jersey, woman, facing, charges, claims, used, false, government, documents, ploy, pose, teenager, high, according]                                                         |0  |\n",
      "|[national, voter, education, going, madison, college, week]                                                                                                                          |1  |\n",
      "|[school, district, texas, pulling, books, libraries, classrooms, previously, challenged, including, bible, illustrated, version, anne]                                               |2  |\n",
      "|[student, presentation, english, class, angered, parents, chino, hills, high, even, heading, school, pull, kids]                                                                     |3  |\n",
      "|[pennsylvania, high, school, football, scores]                                                                                                                                       |4  |\n",
      "|[united, states, witnesses, another, mass, shooting, seen, deadliest, school, shootings, years, gunman, killed, nearly, students, elementary, school]                                |5  |\n",
      "|[group, egyptian, officials, stopped, sollers, point, technical, high, school, last, week, learn, best, practices, technical]                                                        |6  |\n",
      "|[talking, truth, stranger, fiction, whole, class, back, along, alan, cinemas]                                                                                                        |7  |\n",
      "|[delighted, speaking, talking, power, reading, seen, transform, many, schools]                                                                                                       |8  |\n",
      "|[teacher, mireles, spoke, police, officer, husband, final, side, school, wall]                                                                                                       |9  |\n",
      "|[separate, auckland, incidents, police, officer, crash, house, inside, stolen, schools, lockdown, standoff, empty]                                                                   |10 |\n",
      "|[following, pandemic, rise, school, college, campuses, around, country, made, upgrades, ensure]                                                                                      |11 |\n",
      "|[dreamed, going, harvard, university, study, political, science, barred, school, final, studied, much, could, also, took, courses]                                                   |12 |\n",
      "|[black, student, told, back, racist, texts, california, high, school]                                                                                                                |13 |\n",
      "|[rotary, club, donates, dictionaries, grade, students, kids, excited]                                                                                                                |14 |\n",
      "|[accused, killing, high, school, students, orange, county, appeared, court, tuesday]                                                                                                 |15 |\n",
      "|[university, california, michael, announced, launch, native, american, opportunity, program, expand, diversity, make, schools]                                                       |16 |\n",
      "|[latest, twist, saga, surrounding, city, schools, prompted, fears, children, left, state]                                                                                            |17 |\n",
      "|[free, read, national, assessments, sats, work, white, paper, missed, opportunity, make, says]                                                                                       |18 |\n",
      "|[borders, college, department, enterprise, business, innovation, recently, held, celebration, event, skills, boost, care, skills, boost, electronics, engineering, read, full, story]|19 |\n",
      "+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "df_tweet_text_sample_tokens = spark.createDataFrame(tokens, [\"list_of_words\",'id'])\n",
    "df_tweet_text_sample_tokens = df_tweet_text_sample_tokens.where(col('list_of_words').getItem(0).isNotNull())\n",
    "df_tweet_text_sample_tokens.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "bd290128-40d2-4e10-99e5-01d654c41d12",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "13594"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tweet_text_sample_tokens.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb2b32a1-4d95-4c04-847c-a7ecfac865fe",
   "metadata": {},
   "source": [
    "#### Step 2. Fit countvectorizer to create word features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "780306ec-de51-43ff-8cb7-cafdae4fccfa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "vectorize = CountVectorizer(inputCol=\"list_of_words\", outputCol=\"features\", minDF=1.0)\n",
    "df_tweet_text_sample_vectorize = vectorize.fit(df_tweet_text_sample_tokens).transform(df_tweet_text_sample_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "64d2c9e4-f692-496d-a212-abe3be27833e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>list_of_words</th>\n",
       "      <th>id</th>\n",
       "      <th>features</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[playing, jersey, woman, facing, charges, claims, used, false, government, documents, ploy, pose, teenager, high, according]</td>\n",
       "      <td>0</td>\n",
       "      <td>(0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[national, voter, education, going, madison, college, week]</td>\n",
       "      <td>1</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[school, district, texas, pulling, books, libraries, classrooms, previously, challenged, including, bible, illustrated, version, anne]</td>\n",
       "      <td>2</td>\n",
       "      <td>(1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[student, presentation, english, class, angered, parents, chino, hills, high, even, heading, school, pull, kids]</td>\n",
       "      <td>3</td>\n",
       "      <td>(1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[pennsylvania, high, school, football, scores]</td>\n",
       "      <td>4</td>\n",
       "      <td>(1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                            list_of_words  \\\n",
       "0            [playing, jersey, woman, facing, charges, claims, used, false, government, documents, ploy, pose, teenager, high, according]   \n",
       "1                                                                             [national, voter, education, going, madison, college, week]   \n",
       "2  [school, district, texas, pulling, books, libraries, classrooms, previously, challenged, including, bible, illustrated, version, anne]   \n",
       "3                        [student, presentation, english, class, angered, parents, chino, hills, high, even, heading, school, pull, kids]   \n",
       "4                                                                                          [pennsylvania, high, school, football, scores]   \n",
       "\n",
       "   id  \\\n",
       "0   0   \n",
       "1   1   \n",
       "2   2   \n",
       "3   3   \n",
       "4   4   \n",
       "\n",
       "                                                                                                                                                                                                  features  \n",
       "0  (0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "1  (0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "2  (1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "3  (1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
       "4  (1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tweet_text_sample_vectorize.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21b5fbb6-61a9-4d2c-a2f0-316a30c3f83a",
   "metadata": {},
   "source": [
    "#### Step 3. Fit MinHashLSH to create hash table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e2b90b7b-292b-4498-a237-2c9a08e0bbb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "mh = MinHashLSH(inputCol=\"features\", outputCol=\"hashes\", numHashTables=5)\n",
    "model = mh.fit(df_tweet_text_sample_vectorize)\n",
    "df_hashed = mh.fit(df_tweet_text_sample_vectorize).transform(df_tweet_text_sample_vectorize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e42f961f-3647-4c20-b286-98bfef85d554",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>list_of_words</th>\n",
       "      <th>id</th>\n",
       "      <th>features</th>\n",
       "      <th>hashes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[playing, jersey, woman, facing, charges, claims, used, false, government, documents, ploy, pose, teenager, high, according]</td>\n",
       "      <td>0</td>\n",
       "      <td>(0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[[172079437.0], [93365763.0], [32892867.0], [71238812.0], [115748452.0]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[national, voter, education, going, madison, college, week]</td>\n",
       "      <td>1</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[[183183012.0], [175596978.0], [140540522.0], [192418556.0], [264512045.0]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[school, district, texas, pulling, books, libraries, classrooms, previously, challenged, including, bible, illustrated, version, anne]</td>\n",
       "      <td>2</td>\n",
       "      <td>(1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[[47945584.0], [296448403.0], [224252489.0], [61081684.0], [181052214.0]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[student, presentation, english, class, angered, parents, chino, hills, high, even, heading, school, pull, kids]</td>\n",
       "      <td>3</td>\n",
       "      <td>(1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[[47945584.0], [469804387.0], [48148855.0], [3119318.0], [277829823.0]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[pennsylvania, high, school, football, scores]</td>\n",
       "      <td>4</td>\n",
       "      <td>(1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[[47945584.0], [796077476.0], [4682056.0], [229017585.0], [23594946.0]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[united, states, witnesses, another, mass, shooting, seen, deadliest, school, shootings, years, gunman, killed, nearly, students, elementary, school]</td>\n",
       "      <td>5</td>\n",
       "      <td>(2.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...</td>\n",
       "      <td>[[47945584.0], [17063584.0], [179404991.0], [580036.0], [82346913.0]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[group, egyptian, officials, stopped, sollers, point, technical, high, school, last, week, learn, best, practices, technical]</td>\n",
       "      <td>6</td>\n",
       "      <td>(1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, ...</td>\n",
       "      <td>[[47945584.0], [73783649.0], [140540522.0], [424620916.0], [76525715.0]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>[talking, truth, stranger, fiction, whole, class, back, along, alan, cinemas]</td>\n",
       "      <td>7</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[[170069183.0], [340372155.0], [15993700.0], [8109658.0], [58155458.0]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>[delighted, speaking, talking, power, reading, seen, transform, many, schools]</td>\n",
       "      <td>8</td>\n",
       "      <td>(0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[[71157861.0], [73470902.0], [30854182.0], [415689317.0], [30360929.0]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>[teacher, mireles, spoke, police, officer, husband, final, side, school, wall]</td>\n",
       "      <td>9</td>\n",
       "      <td>(1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[[47945584.0], [126913702.0], [774985354.0], [112916405.0], [43464519.0]]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                           list_of_words  \\\n",
       "0                           [playing, jersey, woman, facing, charges, claims, used, false, government, documents, ploy, pose, teenager, high, according]   \n",
       "1                                                                                            [national, voter, education, going, madison, college, week]   \n",
       "2                 [school, district, texas, pulling, books, libraries, classrooms, previously, challenged, including, bible, illustrated, version, anne]   \n",
       "3                                       [student, presentation, english, class, angered, parents, chino, hills, high, even, heading, school, pull, kids]   \n",
       "4                                                                                                         [pennsylvania, high, school, football, scores]   \n",
       "5  [united, states, witnesses, another, mass, shooting, seen, deadliest, school, shootings, years, gunman, killed, nearly, students, elementary, school]   \n",
       "6                          [group, egyptian, officials, stopped, sollers, point, technical, high, school, last, week, learn, best, practices, technical]   \n",
       "7                                                                          [talking, truth, stranger, fiction, whole, class, back, along, alan, cinemas]   \n",
       "8                                                                         [delighted, speaking, talking, power, reading, seen, transform, many, schools]   \n",
       "9                                                                         [teacher, mireles, spoke, police, officer, husband, final, side, school, wall]   \n",
       "\n",
       "   id  \\\n",
       "0   0   \n",
       "1   1   \n",
       "2   2   \n",
       "3   3   \n",
       "4   4   \n",
       "5   5   \n",
       "6   6   \n",
       "7   7   \n",
       "8   8   \n",
       "9   9   \n",
       "\n",
       "                                                                                                                                                                                                  features  \\\n",
       "0  (0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1  (0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2  (1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3  (1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "4  (1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "5  (2.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, ...   \n",
       "6  (1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, ...   \n",
       "7  (0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "8  (0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "9  (1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                                                        hashes  \n",
       "0     [[172079437.0], [93365763.0], [32892867.0], [71238812.0], [115748452.0]]  \n",
       "1  [[183183012.0], [175596978.0], [140540522.0], [192418556.0], [264512045.0]]  \n",
       "2    [[47945584.0], [296448403.0], [224252489.0], [61081684.0], [181052214.0]]  \n",
       "3      [[47945584.0], [469804387.0], [48148855.0], [3119318.0], [277829823.0]]  \n",
       "4      [[47945584.0], [796077476.0], [4682056.0], [229017585.0], [23594946.0]]  \n",
       "5        [[47945584.0], [17063584.0], [179404991.0], [580036.0], [82346913.0]]  \n",
       "6     [[47945584.0], [73783649.0], [140540522.0], [424620916.0], [76525715.0]]  \n",
       "7      [[170069183.0], [340372155.0], [15993700.0], [8109658.0], [58155458.0]]  \n",
       "8      [[71157861.0], [73470902.0], [30854182.0], [415689317.0], [30360929.0]]  \n",
       "9    [[47945584.0], [126913702.0], [774985354.0], [112916405.0], [43464519.0]]  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hashed.limit(10).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0598ae0c-3921-4217-9bdf-0a8d3a1d8d17",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "13594"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hashed.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "9417cabe-d483-43e4-a37d-cb11d81e406e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tweet_text</th>\n",
       "      <th>list_of_words</th>\n",
       "      <th>features</th>\n",
       "      <th>hashes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>26</td>\n",
       "      <td>(a new washington state law supports access to free, high-quality language access services in schools and launches a training program for interpreters. https://t.co/7gl4peyiyb,)</td>\n",
       "      <td>[washington, state, supports, access, language, access, services, schools, launches, training, program]</td>\n",
       "      <td>(0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[[18678956.0], [21823108.0], [27434798.0], [256773239.0], [199422471.0]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>29</td>\n",
       "      <td>(several san antonio independent school district schools are using new 3d printers to create exciting stem opportunities for their students. https://t.co/1cobf1kyb9,)</td>\n",
       "      <td>[several, antonio, independent, school, district, schools, using, printers, create, exciting, stem, opportunities]</td>\n",
       "      <td>(1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[[47945584.0], [44682487.0], [30854182.0], [62395437.0], [407542473.0]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>474</td>\n",
       "      <td>(a male university of southern indiana student must comply with a suspension stemming from a sexual assault, even as his gender discrimination claim plays out, the seventh circuit ruled.\\n\\nhttps:...</td>\n",
       "      <td>[male, university, southern, indiana, student, must, comply, suspension, stemming, sexual, even, gender, discrimination, claim, plays, seventh, circuit]</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[[119094037.0], [7000518.0], [100032874.0], [3119318.0], [18114091.0]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>964</td>\n",
       "      <td>(school officials say it's been a tough time navigating a school year with covid-19, but they're better because of it. specifically in tuscaloosa city schools, the superintendent says this year wi...</td>\n",
       "      <td>[school, officials, tough, time, navigating, school, year, better, specifically, tuscaloosa, city, superintendent, says, year, restart]</td>\n",
       "      <td>(2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...</td>\n",
       "      <td>[[47945584.0], [34932168.0], [250226862.0], [26845489.0], [62691528.0]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1677</td>\n",
       "      <td>(free tickets to the st. paul cup at allianz field! the inaugural event on oct. 7 will feature harding senior high and washington tech, and will rotate through st. paul high schools in future year...</td>\n",
       "      <td>[free, tickets, paul, allianz, inaugural, event, feature, harding, senior, high, washington, rotate, paul, high, schools, future, event, free, please, request, ticket]</td>\n",
       "      <td>(0.0, 2.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...</td>\n",
       "      <td>[[116070793.0], [98125287.0], [30854182.0], [303960909.0], [277615635.0]]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id  \\\n",
       "0    26   \n",
       "1    29   \n",
       "2   474   \n",
       "3   964   \n",
       "4  1677   \n",
       "\n",
       "                                                                                                                                                                                                tweet_text  \\\n",
       "0                        (a new washington state law supports access to free, high-quality language access services in schools and launches a training program for interpreters. https://t.co/7gl4peyiyb,)   \n",
       "1                                   (several san antonio independent school district schools are using new 3d printers to create exciting stem opportunities for their students. https://t.co/1cobf1kyb9,)   \n",
       "2  (a male university of southern indiana student must comply with a suspension stemming from a sexual assault, even as his gender discrimination claim plays out, the seventh circuit ruled.\\n\\nhttps:...   \n",
       "3  (school officials say it's been a tough time navigating a school year with covid-19, but they're better because of it. specifically in tuscaloosa city schools, the superintendent says this year wi...   \n",
       "4  (free tickets to the st. paul cup at allianz field! the inaugural event on oct. 7 will feature harding senior high and washington tech, and will rotate through st. paul high schools in future year...   \n",
       "\n",
       "                                                                                                                                                             list_of_words  \\\n",
       "0                                                                  [washington, state, supports, access, language, access, services, schools, launches, training, program]   \n",
       "1                                                       [several, antonio, independent, school, district, schools, using, printers, create, exciting, stem, opportunities]   \n",
       "2                 [male, university, southern, indiana, student, must, comply, suspension, stemming, sexual, even, gender, discrimination, claim, plays, seventh, circuit]   \n",
       "3                                  [school, officials, tough, time, navigating, school, year, better, specifically, tuscaloosa, city, superintendent, says, year, restart]   \n",
       "4  [free, tickets, paul, allianz, inaugural, event, feature, harding, senior, high, washington, rotate, paul, high, schools, future, event, free, please, request, ticket]   \n",
       "\n",
       "                                                                                                                                                                                                  features  \\\n",
       "0  (0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "1  (1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "2  (0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "3  (2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, ...   \n",
       "4  (0.0, 2.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...   \n",
       "\n",
       "                                                                      hashes  \n",
       "0   [[18678956.0], [21823108.0], [27434798.0], [256773239.0], [199422471.0]]  \n",
       "1    [[47945584.0], [44682487.0], [30854182.0], [62395437.0], [407542473.0]]  \n",
       "2     [[119094037.0], [7000518.0], [100032874.0], [3119318.0], [18114091.0]]  \n",
       "3    [[47945584.0], [34932168.0], [250226862.0], [26845489.0], [62691528.0]]  \n",
       "4  [[116070793.0], [98125287.0], [30854182.0], [303960909.0], [277615635.0]]  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_hashed_tweet_text_samp = df_tweet_text_sample.join(df_hashed, \"id\", how = 'left').cache()\n",
    "df_hashed_tweet_text_samp.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d16c905f-9709-4c21-858e-316d8e4a64a0",
   "metadata": {},
   "source": [
    "#### Step 4. Establish similarity threshold and return near-duplicate records\n",
    "**Note:** we are joining dataframe to itself to get near-duplicate pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4ad20882-ee10-4fbe-a96c-17900b205d79",
   "metadata": {},
   "outputs": [],
   "source": [
    "jaccard_distance = 0.3\n",
    "\n",
    "df_dups_tweet_text = model.approxSimilarityJoin(df_hashed_tweet_text_samp, df_hashed_tweet_text_samp, jaccard_distance).filter(\"datasetA.id < datasetB.id\").select(\n",
    "            col(\"distCol\"),\n",
    "            col(\"datasetA.id\").alias(\"id_A\"),\n",
    "            col(\"datasetB.id\").alias(\"id_B\"),\n",
    "            col('datasetA.tweet_text').alias('tweet_text_A'),\n",
    "            col('datasetB.tweet_text').alias('tweet_text_B'),\n",
    "#             col('datasetA.list_of_words').alias('words_A'),\n",
    "#             col('datasetB.list_of_words').alias('words_B')\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "36459905-7b7f-4fcf-8057-610cc7579b13",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>distCol</th>\n",
       "      <th>id_A</th>\n",
       "      <th>id_B</th>\n",
       "      <th>tweet_text_A</th>\n",
       "      <th>tweet_text_B</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1742</td>\n",
       "      <td>10127</td>\n",
       "      <td>(florida school shooter nikolas cruz will be sentenced to life without parole for the 2018 murder of 17 people at parkland’s marjory stoneman douglas high school. \\n\\nhttps://t.co/bjy2sonuly,)</td>\n",
       "      <td>(florida school shooter nikolas cruz will be sentenced to life without parole for the 2018 murder of 17 people at parkland’s marjory stoneman douglas high school. https://t.co/hybgtgkma6,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>636</td>\n",
       "      <td>13088</td>\n",
       "      <td>(opponents of school choice regularly laud traditional public schools for being \"open to all.\" but they often aren't, with geographic boundaries that require hefty mortgages and property taxes or ...</td>\n",
       "      <td>(opponents of school choice regularly laud traditional public schools for being \"open to all.\" but they often aren't, with geographic boundaries that require hefty mortgages and property taxes or ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>4369</td>\n",
       "      <td>12486</td>\n",
       "      <td>(acting uvalde police chief during school shooting steps down https://t.co/811li3ruwr,)</td>\n",
       "      <td>(acting uvalde police chief during school shooting steps down https://t.co/qqqhy94nkx,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>4787</td>\n",
       "      <td>9818</td>\n",
       "      <td>(do you know any junior high school student who can step up to the literacy challenge?\\n\\nif yes, then tell them #theliteracychallenge is on!!! https://t.co/o1burl82ca,)</td>\n",
       "      <td>(do you know any junior high school student who can step up to the literacy challenge?\\n\\nif yes, then tell them #theliteracychallenge is on!!! https://t.co/rwi1jeg1kr,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.142857</td>\n",
       "      <td>155</td>\n",
       "      <td>4755</td>\n",
       "      <td>(icymi: a 6-year-old student shot and wounded a teacher at his school in virginia during an altercation inside a first-grade classroom friday, police and school officials in the city of newport ne...</td>\n",
       "      <td>(police in the virginia city of newport news say a 6-year-old student shot and wounded a teacher at his school during an altercation inside a first-grade classroom. https://t.co/fzwfrb7yen,)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    distCol  id_A   id_B  \\\n",
       "0  0.000000  1742  10127   \n",
       "1  0.000000   636  13088   \n",
       "2  0.000000  4369  12486   \n",
       "3  0.000000  4787   9818   \n",
       "4  0.142857   155   4755   \n",
       "\n",
       "                                                                                                                                                                                              tweet_text_A  \\\n",
       "0         (florida school shooter nikolas cruz will be sentenced to life without parole for the 2018 murder of 17 people at parkland’s marjory stoneman douglas high school. \\n\\nhttps://t.co/bjy2sonuly,)   \n",
       "1  (opponents of school choice regularly laud traditional public schools for being \"open to all.\" but they often aren't, with geographic boundaries that require hefty mortgages and property taxes or ...   \n",
       "2                                                                                                                  (acting uvalde police chief during school shooting steps down https://t.co/811li3ruwr,)   \n",
       "3                                (do you know any junior high school student who can step up to the literacy challenge?\\n\\nif yes, then tell them #theliteracychallenge is on!!! https://t.co/o1burl82ca,)   \n",
       "4  (icymi: a 6-year-old student shot and wounded a teacher at his school in virginia during an altercation inside a first-grade classroom friday, police and school officials in the city of newport ne...   \n",
       "\n",
       "                                                                                                                                                                                              tweet_text_B  \n",
       "0             (florida school shooter nikolas cruz will be sentenced to life without parole for the 2018 murder of 17 people at parkland’s marjory stoneman douglas high school. https://t.co/hybgtgkma6,)  \n",
       "1  (opponents of school choice regularly laud traditional public schools for being \"open to all.\" but they often aren't, with geographic boundaries that require hefty mortgages and property taxes or ...  \n",
       "2                                                                                                                  (acting uvalde police chief during school shooting steps down https://t.co/qqqhy94nkx,)  \n",
       "3                                (do you know any junior high school student who can step up to the literacy challenge?\\n\\nif yes, then tell them #theliteracychallenge is on!!! https://t.co/rwi1jeg1kr,)  \n",
       "4           (police in the virginia city of newport news say a 6-year-old student shot and wounded a teacher at his school during an altercation inside a first-grade classroom. https://t.co/fzwfrb7yen,)  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dups_30_tweet_text = df_dups_tweet_text\n",
    "df_dups_30_tweet_text.cache()\n",
    "df_dups_30_tweet_text.limit(5).toPandas()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "15fe9861-765e-4314-988c-5f17c7dd8ff9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "302"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dups_30_tweet_text.select('id_A').distinct().count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "3d841858-e73a-44a5-8598-3b3cae55cda4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 137:===============================================>     (181 + 4) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records:  13594\n",
      "Duplicate titles based on { 0.3 } jaccard distance:  302\n",
      "Unique titles based on { 0.3 } jaccard distance:  :  13292\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "records = df_hashed_tweet_text_samp.count()\n",
    "dups_30 = df_dups_30_tweet_text.select('id_A').distinct().count()\n",
    "uniques_30 = records - dups_30\n",
    "\n",
    "print ('Total records: ', records)\n",
    "print ('Duplicate titles based on {', jaccard_distance, '} jaccard distance: ', dups_30)\n",
    "print ('Unique titles based on {', jaccard_distance, '} jaccard distance: ', ': ', uniques_30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "fda80e32-27e1-44b3-b563-944501b5959c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       "  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<svg height=\"324.950281pt\" version=\"1.1\" viewBox=\"0 0 411.415078 324.950281\" width=\"411.415078pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       " <metadata>\n",
       "  <rdf:RDF xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n",
       "   <cc:Work>\n",
       "    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\n",
       "    <dc:date>2023-03-09T00:41:55.067341</dc:date>\n",
       "    <dc:format>image/svg+xml</dc:format>\n",
       "    <dc:creator>\n",
       "     <cc:Agent>\n",
       "      <dc:title>Matplotlib v3.4.3, https://matplotlib.org/</dc:title>\n",
       "     </cc:Agent>\n",
       "    </dc:creator>\n",
       "   </cc:Work>\n",
       "  </rdf:RDF>\n",
       " </metadata>\n",
       " <defs>\n",
       "  <style type=\"text/css\">*{stroke-linecap:butt;stroke-linejoin:round;}</style>\n",
       " </defs>\n",
       " <g id=\"figure_1\">\n",
       "  <g id=\"patch_1\">\n",
       "   <path d=\"M 0 324.950281 \n",
       "L 411.415078 324.950281 \n",
       "L 411.415078 0 \n",
       "L 0 0 \n",
       "z\n",
       "\" style=\"fill:#ffffff;\"/>\n",
       "  </g>\n",
       "  <g id=\"axes_1\">\n",
       "   <g id=\"patch_2\">\n",
       "    <path d=\"M 46.0125 290.709656 \n",
       "L 403.1325 290.709656 \n",
       "L 403.1325 24.597656 \n",
       "L 46.0125 24.597656 \n",
       "z\n",
       "\" style=\"fill:#ffffff;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_3\">\n",
       "    <path clip-path=\"url(#pabeae22aee)\" d=\"M 114.689423 290.709656 \n",
       "L 224.5725 290.709656 \n",
       "L 224.5725 284.95139 \n",
       "L 114.689423 284.95139 \n",
       "z\n",
       "\" style=\"fill:#d62728;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_4\">\n",
       "    <path clip-path=\"url(#pabeae22aee)\" d=\"M 224.5725 290.709656 \n",
       "L 334.455577 290.709656 \n",
       "L 334.455577 37.269656 \n",
       "L 224.5725 37.269656 \n",
       "z\n",
       "\" style=\"fill:#2ca02c;\"/>\n",
       "   </g>\n",
       "   <g id=\"matplotlib.axis_1\">\n",
       "    <g id=\"xtick_1\">\n",
       "     <g id=\"line2d_1\">\n",
       "      <defs>\n",
       "       <path d=\"M 0 0 \n",
       "L 0 3.5 \n",
       "\" id=\"m27e616c909\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"224.5725\" xlink:href=\"#m27e616c909\" y=\"290.709656\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_1\">\n",
       "      <!-- 0 -->\n",
       "      <g transform=\"translate(227.331875 304.072156)rotate(-90)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 2034 4250 \n",
       "Q 1547 4250 1301 3770 \n",
       "Q 1056 3291 1056 2328 \n",
       "Q 1056 1369 1301 889 \n",
       "Q 1547 409 2034 409 \n",
       "Q 2525 409 2770 889 \n",
       "Q 3016 1369 3016 2328 \n",
       "Q 3016 3291 2770 3770 \n",
       "Q 2525 4250 2034 4250 \n",
       "z\n",
       "M 2034 4750 \n",
       "Q 2819 4750 3233 4129 \n",
       "Q 3647 3509 3647 2328 \n",
       "Q 3647 1150 3233 529 \n",
       "Q 2819 -91 2034 -91 \n",
       "Q 1250 -91 836 529 \n",
       "Q 422 1150 422 2328 \n",
       "Q 422 3509 836 4129 \n",
       "Q 1250 4750 2034 4750 \n",
       "z\n",
       "\" id=\"DejaVuSans-30\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"text_2\">\n",
       "     <!-- Duplicates vs. Unique -->\n",
       "     <g transform=\"translate(170.17875 315.670594)scale(0.1 -0.1)\">\n",
       "      <defs>\n",
       "       <path d=\"M 1259 4147 \n",
       "L 1259 519 \n",
       "L 2022 519 \n",
       "Q 2988 519 3436 956 \n",
       "Q 3884 1394 3884 2338 \n",
       "Q 3884 3275 3436 3711 \n",
       "Q 2988 4147 2022 4147 \n",
       "L 1259 4147 \n",
       "z\n",
       "M 628 4666 \n",
       "L 1925 4666 \n",
       "Q 3281 4666 3915 4102 \n",
       "Q 4550 3538 4550 2338 \n",
       "Q 4550 1131 3912 565 \n",
       "Q 3275 0 1925 0 \n",
       "L 628 0 \n",
       "L 628 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-44\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 544 1381 \n",
       "L 544 3500 \n",
       "L 1119 3500 \n",
       "L 1119 1403 \n",
       "Q 1119 906 1312 657 \n",
       "Q 1506 409 1894 409 \n",
       "Q 2359 409 2629 706 \n",
       "Q 2900 1003 2900 1516 \n",
       "L 2900 3500 \n",
       "L 3475 3500 \n",
       "L 3475 0 \n",
       "L 2900 0 \n",
       "L 2900 538 \n",
       "Q 2691 219 2414 64 \n",
       "Q 2138 -91 1772 -91 \n",
       "Q 1169 -91 856 284 \n",
       "Q 544 659 544 1381 \n",
       "z\n",
       "M 1991 3584 \n",
       "L 1991 3584 \n",
       "z\n",
       "\" id=\"DejaVuSans-75\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 1159 525 \n",
       "L 1159 -1331 \n",
       "L 581 -1331 \n",
       "L 581 3500 \n",
       "L 1159 3500 \n",
       "L 1159 2969 \n",
       "Q 1341 3281 1617 3432 \n",
       "Q 1894 3584 2278 3584 \n",
       "Q 2916 3584 3314 3078 \n",
       "Q 3713 2572 3713 1747 \n",
       "Q 3713 922 3314 415 \n",
       "Q 2916 -91 2278 -91 \n",
       "Q 1894 -91 1617 61 \n",
       "Q 1341 213 1159 525 \n",
       "z\n",
       "M 3116 1747 \n",
       "Q 3116 2381 2855 2742 \n",
       "Q 2594 3103 2138 3103 \n",
       "Q 1681 3103 1420 2742 \n",
       "Q 1159 2381 1159 1747 \n",
       "Q 1159 1113 1420 752 \n",
       "Q 1681 391 2138 391 \n",
       "Q 2594 391 2855 752 \n",
       "Q 3116 1113 3116 1747 \n",
       "z\n",
       "\" id=\"DejaVuSans-70\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 603 4863 \n",
       "L 1178 4863 \n",
       "L 1178 0 \n",
       "L 603 0 \n",
       "L 603 4863 \n",
       "z\n",
       "\" id=\"DejaVuSans-6c\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 603 3500 \n",
       "L 1178 3500 \n",
       "L 1178 0 \n",
       "L 603 0 \n",
       "L 603 3500 \n",
       "z\n",
       "M 603 4863 \n",
       "L 1178 4863 \n",
       "L 1178 4134 \n",
       "L 603 4134 \n",
       "L 603 4863 \n",
       "z\n",
       "\" id=\"DejaVuSans-69\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 3122 3366 \n",
       "L 3122 2828 \n",
       "Q 2878 2963 2633 3030 \n",
       "Q 2388 3097 2138 3097 \n",
       "Q 1578 3097 1268 2742 \n",
       "Q 959 2388 959 1747 \n",
       "Q 959 1106 1268 751 \n",
       "Q 1578 397 2138 397 \n",
       "Q 2388 397 2633 464 \n",
       "Q 2878 531 3122 666 \n",
       "L 3122 134 \n",
       "Q 2881 22 2623 -34 \n",
       "Q 2366 -91 2075 -91 \n",
       "Q 1284 -91 818 406 \n",
       "Q 353 903 353 1747 \n",
       "Q 353 2603 823 3093 \n",
       "Q 1294 3584 2113 3584 \n",
       "Q 2378 3584 2631 3529 \n",
       "Q 2884 3475 3122 3366 \n",
       "z\n",
       "\" id=\"DejaVuSans-63\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2194 1759 \n",
       "Q 1497 1759 1228 1600 \n",
       "Q 959 1441 959 1056 \n",
       "Q 959 750 1161 570 \n",
       "Q 1363 391 1709 391 \n",
       "Q 2188 391 2477 730 \n",
       "Q 2766 1069 2766 1631 \n",
       "L 2766 1759 \n",
       "L 2194 1759 \n",
       "z\n",
       "M 3341 1997 \n",
       "L 3341 0 \n",
       "L 2766 0 \n",
       "L 2766 531 \n",
       "Q 2569 213 2275 61 \n",
       "Q 1981 -91 1556 -91 \n",
       "Q 1019 -91 701 211 \n",
       "Q 384 513 384 1019 \n",
       "Q 384 1609 779 1909 \n",
       "Q 1175 2209 1959 2209 \n",
       "L 2766 2209 \n",
       "L 2766 2266 \n",
       "Q 2766 2663 2505 2880 \n",
       "Q 2244 3097 1772 3097 \n",
       "Q 1472 3097 1187 3025 \n",
       "Q 903 2953 641 2809 \n",
       "L 641 3341 \n",
       "Q 956 3463 1253 3523 \n",
       "Q 1550 3584 1831 3584 \n",
       "Q 2591 3584 2966 3190 \n",
       "Q 3341 2797 3341 1997 \n",
       "z\n",
       "\" id=\"DejaVuSans-61\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 1172 4494 \n",
       "L 1172 3500 \n",
       "L 2356 3500 \n",
       "L 2356 3053 \n",
       "L 1172 3053 \n",
       "L 1172 1153 \n",
       "Q 1172 725 1289 603 \n",
       "Q 1406 481 1766 481 \n",
       "L 2356 481 \n",
       "L 2356 0 \n",
       "L 1766 0 \n",
       "Q 1100 0 847 248 \n",
       "Q 594 497 594 1153 \n",
       "L 594 3053 \n",
       "L 172 3053 \n",
       "L 172 3500 \n",
       "L 594 3500 \n",
       "L 594 4494 \n",
       "L 1172 4494 \n",
       "z\n",
       "\" id=\"DejaVuSans-74\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 3597 1894 \n",
       "L 3597 1613 \n",
       "L 953 1613 \n",
       "Q 991 1019 1311 708 \n",
       "Q 1631 397 2203 397 \n",
       "Q 2534 397 2845 478 \n",
       "Q 3156 559 3463 722 \n",
       "L 3463 178 \n",
       "Q 3153 47 2828 -22 \n",
       "Q 2503 -91 2169 -91 \n",
       "Q 1331 -91 842 396 \n",
       "Q 353 884 353 1716 \n",
       "Q 353 2575 817 3079 \n",
       "Q 1281 3584 2069 3584 \n",
       "Q 2775 3584 3186 3129 \n",
       "Q 3597 2675 3597 1894 \n",
       "z\n",
       "M 3022 2063 \n",
       "Q 3016 2534 2758 2815 \n",
       "Q 2500 3097 2075 3097 \n",
       "Q 1594 3097 1305 2825 \n",
       "Q 1016 2553 972 2059 \n",
       "L 3022 2063 \n",
       "z\n",
       "\" id=\"DejaVuSans-65\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2834 3397 \n",
       "L 2834 2853 \n",
       "Q 2591 2978 2328 3040 \n",
       "Q 2066 3103 1784 3103 \n",
       "Q 1356 3103 1142 2972 \n",
       "Q 928 2841 928 2578 \n",
       "Q 928 2378 1081 2264 \n",
       "Q 1234 2150 1697 2047 \n",
       "L 1894 2003 \n",
       "Q 2506 1872 2764 1633 \n",
       "Q 3022 1394 3022 966 \n",
       "Q 3022 478 2636 193 \n",
       "Q 2250 -91 1575 -91 \n",
       "Q 1294 -91 989 -36 \n",
       "Q 684 19 347 128 \n",
       "L 347 722 \n",
       "Q 666 556 975 473 \n",
       "Q 1284 391 1588 391 \n",
       "Q 1994 391 2212 530 \n",
       "Q 2431 669 2431 922 \n",
       "Q 2431 1156 2273 1281 \n",
       "Q 2116 1406 1581 1522 \n",
       "L 1381 1569 \n",
       "Q 847 1681 609 1914 \n",
       "Q 372 2147 372 2553 \n",
       "Q 372 3047 722 3315 \n",
       "Q 1072 3584 1716 3584 \n",
       "Q 2034 3584 2315 3537 \n",
       "Q 2597 3491 2834 3397 \n",
       "z\n",
       "\" id=\"DejaVuSans-73\" transform=\"scale(0.015625)\"/>\n",
       "       <path id=\"DejaVuSans-20\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 191 3500 \n",
       "L 800 3500 \n",
       "L 1894 563 \n",
       "L 2988 3500 \n",
       "L 3597 3500 \n",
       "L 2284 0 \n",
       "L 1503 0 \n",
       "L 191 3500 \n",
       "z\n",
       "\" id=\"DejaVuSans-76\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 684 794 \n",
       "L 1344 794 \n",
       "L 1344 0 \n",
       "L 684 0 \n",
       "L 684 794 \n",
       "z\n",
       "\" id=\"DejaVuSans-2e\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 556 4666 \n",
       "L 1191 4666 \n",
       "L 1191 1831 \n",
       "Q 1191 1081 1462 751 \n",
       "Q 1734 422 2344 422 \n",
       "Q 2950 422 3222 751 \n",
       "Q 3494 1081 3494 1831 \n",
       "L 3494 4666 \n",
       "L 4128 4666 \n",
       "L 4128 1753 \n",
       "Q 4128 841 3676 375 \n",
       "Q 3225 -91 2344 -91 \n",
       "Q 1459 -91 1007 375 \n",
       "Q 556 841 556 1753 \n",
       "L 556 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-55\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 3513 2113 \n",
       "L 3513 0 \n",
       "L 2938 0 \n",
       "L 2938 2094 \n",
       "Q 2938 2591 2744 2837 \n",
       "Q 2550 3084 2163 3084 \n",
       "Q 1697 3084 1428 2787 \n",
       "Q 1159 2491 1159 1978 \n",
       "L 1159 0 \n",
       "L 581 0 \n",
       "L 581 3500 \n",
       "L 1159 3500 \n",
       "L 1159 2956 \n",
       "Q 1366 3272 1645 3428 \n",
       "Q 1925 3584 2291 3584 \n",
       "Q 2894 3584 3203 3211 \n",
       "Q 3513 2838 3513 2113 \n",
       "z\n",
       "\" id=\"DejaVuSans-6e\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 947 1747 \n",
       "Q 947 1113 1208 752 \n",
       "Q 1469 391 1925 391 \n",
       "Q 2381 391 2643 752 \n",
       "Q 2906 1113 2906 1747 \n",
       "Q 2906 2381 2643 2742 \n",
       "Q 2381 3103 1925 3103 \n",
       "Q 1469 3103 1208 2742 \n",
       "Q 947 2381 947 1747 \n",
       "z\n",
       "M 2906 525 \n",
       "Q 2725 213 2448 61 \n",
       "Q 2172 -91 1784 -91 \n",
       "Q 1150 -91 751 415 \n",
       "Q 353 922 353 1747 \n",
       "Q 353 2572 751 3078 \n",
       "Q 1150 3584 1784 3584 \n",
       "Q 2172 3584 2448 3432 \n",
       "Q 2725 3281 2906 2969 \n",
       "L 2906 3500 \n",
       "L 3481 3500 \n",
       "L 3481 -1331 \n",
       "L 2906 -1331 \n",
       "L 2906 525 \n",
       "z\n",
       "\" id=\"DejaVuSans-71\" transform=\"scale(0.015625)\"/>\n",
       "      </defs>\n",
       "      <use xlink:href=\"#DejaVuSans-44\"/>\n",
       "      <use x=\"77.001953\" xlink:href=\"#DejaVuSans-75\"/>\n",
       "      <use x=\"140.380859\" xlink:href=\"#DejaVuSans-70\"/>\n",
       "      <use x=\"203.857422\" xlink:href=\"#DejaVuSans-6c\"/>\n",
       "      <use x=\"231.640625\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "      <use x=\"259.423828\" xlink:href=\"#DejaVuSans-63\"/>\n",
       "      <use x=\"314.404297\" xlink:href=\"#DejaVuSans-61\"/>\n",
       "      <use x=\"375.683594\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "      <use x=\"414.892578\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "      <use x=\"476.416016\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "      <use x=\"528.515625\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"560.302734\" xlink:href=\"#DejaVuSans-76\"/>\n",
       "      <use x=\"619.482422\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "      <use x=\"671.582031\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "      <use x=\"703.369141\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"735.15625\" xlink:href=\"#DejaVuSans-55\"/>\n",
       "      <use x=\"808.349609\" xlink:href=\"#DejaVuSans-6e\"/>\n",
       "      <use x=\"871.728516\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "      <use x=\"899.511719\" xlink:href=\"#DejaVuSans-71\"/>\n",
       "      <use x=\"962.988281\" xlink:href=\"#DejaVuSans-75\"/>\n",
       "      <use x=\"1026.367188\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"matplotlib.axis_2\">\n",
       "    <g id=\"ytick_1\">\n",
       "     <g id=\"line2d_2\">\n",
       "      <defs>\n",
       "       <path d=\"M 0 0 \n",
       "L -3.5 0 \n",
       "\" id=\"mad1b41aa64\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#mad1b41aa64\" y=\"290.709656\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_3\">\n",
       "      <!-- 0 -->\n",
       "      <g transform=\"translate(32.65 294.508875)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_2\">\n",
       "     <g id=\"line2d_3\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#mad1b41aa64\" y=\"252.57544\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_4\">\n",
       "      <!-- 2000 -->\n",
       "      <g transform=\"translate(13.5625 256.374659)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 1228 531 \n",
       "L 3431 531 \n",
       "L 3431 0 \n",
       "L 469 0 \n",
       "L 469 531 \n",
       "Q 828 903 1448 1529 \n",
       "Q 2069 2156 2228 2338 \n",
       "Q 2531 2678 2651 2914 \n",
       "Q 2772 3150 2772 3378 \n",
       "Q 2772 3750 2511 3984 \n",
       "Q 2250 4219 1831 4219 \n",
       "Q 1534 4219 1204 4116 \n",
       "Q 875 4013 500 3803 \n",
       "L 500 4441 \n",
       "Q 881 4594 1212 4672 \n",
       "Q 1544 4750 1819 4750 \n",
       "Q 2544 4750 2975 4387 \n",
       "Q 3406 4025 3406 3419 \n",
       "Q 3406 3131 3298 2873 \n",
       "Q 3191 2616 2906 2266 \n",
       "Q 2828 2175 2409 1742 \n",
       "Q 1991 1309 1228 531 \n",
       "z\n",
       "\" id=\"DejaVuSans-32\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-32\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_3\">\n",
       "     <g id=\"line2d_4\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#mad1b41aa64\" y=\"214.441224\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_5\">\n",
       "      <!-- 4000 -->\n",
       "      <g transform=\"translate(13.5625 218.240443)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 2419 4116 \n",
       "L 825 1625 \n",
       "L 2419 1625 \n",
       "L 2419 4116 \n",
       "z\n",
       "M 2253 4666 \n",
       "L 3047 4666 \n",
       "L 3047 1625 \n",
       "L 3713 1625 \n",
       "L 3713 1100 \n",
       "L 3047 1100 \n",
       "L 3047 0 \n",
       "L 2419 0 \n",
       "L 2419 1100 \n",
       "L 313 1100 \n",
       "L 313 1709 \n",
       "L 2253 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-34\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-34\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_4\">\n",
       "     <g id=\"line2d_5\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#mad1b41aa64\" y=\"176.307008\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_6\">\n",
       "      <!-- 6000 -->\n",
       "      <g transform=\"translate(13.5625 180.106227)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 2113 2584 \n",
       "Q 1688 2584 1439 2293 \n",
       "Q 1191 2003 1191 1497 \n",
       "Q 1191 994 1439 701 \n",
       "Q 1688 409 2113 409 \n",
       "Q 2538 409 2786 701 \n",
       "Q 3034 994 3034 1497 \n",
       "Q 3034 2003 2786 2293 \n",
       "Q 2538 2584 2113 2584 \n",
       "z\n",
       "M 3366 4563 \n",
       "L 3366 3988 \n",
       "Q 3128 4100 2886 4159 \n",
       "Q 2644 4219 2406 4219 \n",
       "Q 1781 4219 1451 3797 \n",
       "Q 1122 3375 1075 2522 \n",
       "Q 1259 2794 1537 2939 \n",
       "Q 1816 3084 2150 3084 \n",
       "Q 2853 3084 3261 2657 \n",
       "Q 3669 2231 3669 1497 \n",
       "Q 3669 778 3244 343 \n",
       "Q 2819 -91 2113 -91 \n",
       "Q 1303 -91 875 529 \n",
       "Q 447 1150 447 2328 \n",
       "Q 447 3434 972 4092 \n",
       "Q 1497 4750 2381 4750 \n",
       "Q 2619 4750 2861 4703 \n",
       "Q 3103 4656 3366 4563 \n",
       "z\n",
       "\" id=\"DejaVuSans-36\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-36\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_5\">\n",
       "     <g id=\"line2d_6\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#mad1b41aa64\" y=\"138.172792\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_7\">\n",
       "      <!-- 8000 -->\n",
       "      <g transform=\"translate(13.5625 141.972011)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 2034 2216 \n",
       "Q 1584 2216 1326 1975 \n",
       "Q 1069 1734 1069 1313 \n",
       "Q 1069 891 1326 650 \n",
       "Q 1584 409 2034 409 \n",
       "Q 2484 409 2743 651 \n",
       "Q 3003 894 3003 1313 \n",
       "Q 3003 1734 2745 1975 \n",
       "Q 2488 2216 2034 2216 \n",
       "z\n",
       "M 1403 2484 \n",
       "Q 997 2584 770 2862 \n",
       "Q 544 3141 544 3541 \n",
       "Q 544 4100 942 4425 \n",
       "Q 1341 4750 2034 4750 \n",
       "Q 2731 4750 3128 4425 \n",
       "Q 3525 4100 3525 3541 \n",
       "Q 3525 3141 3298 2862 \n",
       "Q 3072 2584 2669 2484 \n",
       "Q 3125 2378 3379 2068 \n",
       "Q 3634 1759 3634 1313 \n",
       "Q 3634 634 3220 271 \n",
       "Q 2806 -91 2034 -91 \n",
       "Q 1263 -91 848 271 \n",
       "Q 434 634 434 1313 \n",
       "Q 434 1759 690 2068 \n",
       "Q 947 2378 1403 2484 \n",
       "z\n",
       "M 1172 3481 \n",
       "Q 1172 3119 1398 2916 \n",
       "Q 1625 2713 2034 2713 \n",
       "Q 2441 2713 2670 2916 \n",
       "Q 2900 3119 2900 3481 \n",
       "Q 2900 3844 2670 4047 \n",
       "Q 2441 4250 2034 4250 \n",
       "Q 1625 4250 1398 4047 \n",
       "Q 1172 3844 1172 3481 \n",
       "z\n",
       "\" id=\"DejaVuSans-38\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-38\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_6\">\n",
       "     <g id=\"line2d_7\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#mad1b41aa64\" y=\"100.038576\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_8\">\n",
       "      <!-- 10000 -->\n",
       "      <g transform=\"translate(7.2 103.837795)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 794 531 \n",
       "L 1825 531 \n",
       "L 1825 4091 \n",
       "L 703 3866 \n",
       "L 703 4441 \n",
       "L 1819 4666 \n",
       "L 2450 4666 \n",
       "L 2450 531 \n",
       "L 3481 531 \n",
       "L 3481 0 \n",
       "L 794 0 \n",
       "L 794 531 \n",
       "z\n",
       "\" id=\"DejaVuSans-31\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-31\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"254.492188\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_7\">\n",
       "     <g id=\"line2d_8\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#mad1b41aa64\" y=\"61.90436\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_9\">\n",
       "      <!-- 12000 -->\n",
       "      <g transform=\"translate(7.2 65.703579)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#DejaVuSans-31\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-32\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"254.492188\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"patch_5\">\n",
       "    <path d=\"M 46.0125 290.709656 \n",
       "L 46.0125 24.597656 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_6\">\n",
       "    <path d=\"M 403.1325 290.709656 \n",
       "L 403.1325 24.597656 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_7\">\n",
       "    <path d=\"M 46.0125 290.709656 \n",
       "L 403.1325 290.709656 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_8\">\n",
       "    <path d=\"M 46.0125 24.597656 \n",
       "L 403.1325 24.597656 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"text_10\">\n",
       "    <!-- 302.0 -->\n",
       "    <g transform=\"translate(155.316899 281.589898)scale(0.1 -0.1)\">\n",
       "     <defs>\n",
       "      <path d=\"M 2597 2516 \n",
       "Q 3050 2419 3304 2112 \n",
       "Q 3559 1806 3559 1356 \n",
       "Q 3559 666 3084 287 \n",
       "Q 2609 -91 1734 -91 \n",
       "Q 1441 -91 1130 -33 \n",
       "Q 819 25 488 141 \n",
       "L 488 750 \n",
       "Q 750 597 1062 519 \n",
       "Q 1375 441 1716 441 \n",
       "Q 2309 441 2620 675 \n",
       "Q 2931 909 2931 1356 \n",
       "Q 2931 1769 2642 2001 \n",
       "Q 2353 2234 1838 2234 \n",
       "L 1294 2234 \n",
       "L 1294 2753 \n",
       "L 1863 2753 \n",
       "Q 2328 2753 2575 2939 \n",
       "Q 2822 3125 2822 3475 \n",
       "Q 2822 3834 2567 4026 \n",
       "Q 2313 4219 1838 4219 \n",
       "Q 1578 4219 1281 4162 \n",
       "Q 984 4106 628 3988 \n",
       "L 628 4550 \n",
       "Q 988 4650 1302 4700 \n",
       "Q 1616 4750 1894 4750 \n",
       "Q 2613 4750 3031 4423 \n",
       "Q 3450 4097 3450 3541 \n",
       "Q 3450 3153 3228 2886 \n",
       "Q 3006 2619 2597 2516 \n",
       "z\n",
       "\" id=\"DejaVuSans-33\" transform=\"scale(0.015625)\"/>\n",
       "     </defs>\n",
       "     <use xlink:href=\"#DejaVuSans-33\"/>\n",
       "     <use x=\"63.623047\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "     <use x=\"127.246094\" xlink:href=\"#DejaVuSans-32\"/>\n",
       "     <use x=\"190.869141\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "     <use x=\"222.65625\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"text_11\">\n",
       "    <!-- 13292.0 -->\n",
       "    <g transform=\"translate(258.837476 157.749031)scale(0.1 -0.1)\">\n",
       "     <defs>\n",
       "      <path d=\"M 703 97 \n",
       "L 703 672 \n",
       "Q 941 559 1184 500 \n",
       "Q 1428 441 1663 441 \n",
       "Q 2288 441 2617 861 \n",
       "Q 2947 1281 2994 2138 \n",
       "Q 2813 1869 2534 1725 \n",
       "Q 2256 1581 1919 1581 \n",
       "Q 1219 1581 811 2004 \n",
       "Q 403 2428 403 3163 \n",
       "Q 403 3881 828 4315 \n",
       "Q 1253 4750 1959 4750 \n",
       "Q 2769 4750 3195 4129 \n",
       "Q 3622 3509 3622 2328 \n",
       "Q 3622 1225 3098 567 \n",
       "Q 2575 -91 1691 -91 \n",
       "Q 1453 -91 1209 -44 \n",
       "Q 966 3 703 97 \n",
       "z\n",
       "M 1959 2075 \n",
       "Q 2384 2075 2632 2365 \n",
       "Q 2881 2656 2881 3163 \n",
       "Q 2881 3666 2632 3958 \n",
       "Q 2384 4250 1959 4250 \n",
       "Q 1534 4250 1286 3958 \n",
       "Q 1038 3666 1038 3163 \n",
       "Q 1038 2656 1286 2365 \n",
       "Q 1534 2075 1959 2075 \n",
       "z\n",
       "\" id=\"DejaVuSans-39\" transform=\"scale(0.015625)\"/>\n",
       "     </defs>\n",
       "     <use xlink:href=\"#DejaVuSans-31\"/>\n",
       "     <use x=\"63.623047\" xlink:href=\"#DejaVuSans-33\"/>\n",
       "     <use x=\"127.246094\" xlink:href=\"#DejaVuSans-32\"/>\n",
       "     <use x=\"190.869141\" xlink:href=\"#DejaVuSans-39\"/>\n",
       "     <use x=\"254.492188\" xlink:href=\"#DejaVuSans-32\"/>\n",
       "     <use x=\"318.115234\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "     <use x=\"349.902344\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"text_12\">\n",
       "    <!-- Twitter Text Duplication Analysis - 0.3 Threshold -->\n",
       "    <g transform=\"translate(44.929922 18.597656)scale(0.15 -0.15)\">\n",
       "     <defs>\n",
       "      <path d=\"M -19 4666 \n",
       "L 3928 4666 \n",
       "L 3928 4134 \n",
       "L 2272 4134 \n",
       "L 2272 0 \n",
       "L 1638 0 \n",
       "L 1638 4134 \n",
       "L -19 4134 \n",
       "L -19 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-54\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 269 3500 \n",
       "L 844 3500 \n",
       "L 1563 769 \n",
       "L 2278 3500 \n",
       "L 2956 3500 \n",
       "L 3675 769 \n",
       "L 4391 3500 \n",
       "L 4966 3500 \n",
       "L 4050 0 \n",
       "L 3372 0 \n",
       "L 2619 2869 \n",
       "L 1863 0 \n",
       "L 1184 0 \n",
       "L 269 3500 \n",
       "z\n",
       "\" id=\"DejaVuSans-77\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 2631 2963 \n",
       "Q 2534 3019 2420 3045 \n",
       "Q 2306 3072 2169 3072 \n",
       "Q 1681 3072 1420 2755 \n",
       "Q 1159 2438 1159 1844 \n",
       "L 1159 0 \n",
       "L 581 0 \n",
       "L 581 3500 \n",
       "L 1159 3500 \n",
       "L 1159 2956 \n",
       "Q 1341 3275 1631 3429 \n",
       "Q 1922 3584 2338 3584 \n",
       "Q 2397 3584 2469 3576 \n",
       "Q 2541 3569 2628 3553 \n",
       "L 2631 2963 \n",
       "z\n",
       "\" id=\"DejaVuSans-72\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 3513 3500 \n",
       "L 2247 1797 \n",
       "L 3578 0 \n",
       "L 2900 0 \n",
       "L 1881 1375 \n",
       "L 863 0 \n",
       "L 184 0 \n",
       "L 1544 1831 \n",
       "L 300 3500 \n",
       "L 978 3500 \n",
       "L 1906 2253 \n",
       "L 2834 3500 \n",
       "L 3513 3500 \n",
       "z\n",
       "\" id=\"DejaVuSans-78\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 1959 3097 \n",
       "Q 1497 3097 1228 2736 \n",
       "Q 959 2375 959 1747 \n",
       "Q 959 1119 1226 758 \n",
       "Q 1494 397 1959 397 \n",
       "Q 2419 397 2687 759 \n",
       "Q 2956 1122 2956 1747 \n",
       "Q 2956 2369 2687 2733 \n",
       "Q 2419 3097 1959 3097 \n",
       "z\n",
       "M 1959 3584 \n",
       "Q 2709 3584 3137 3096 \n",
       "Q 3566 2609 3566 1747 \n",
       "Q 3566 888 3137 398 \n",
       "Q 2709 -91 1959 -91 \n",
       "Q 1206 -91 779 398 \n",
       "Q 353 888 353 1747 \n",
       "Q 353 2609 779 3096 \n",
       "Q 1206 3584 1959 3584 \n",
       "z\n",
       "\" id=\"DejaVuSans-6f\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 2188 4044 \n",
       "L 1331 1722 \n",
       "L 3047 1722 \n",
       "L 2188 4044 \n",
       "z\n",
       "M 1831 4666 \n",
       "L 2547 4666 \n",
       "L 4325 0 \n",
       "L 3669 0 \n",
       "L 3244 1197 \n",
       "L 1141 1197 \n",
       "L 716 0 \n",
       "L 50 0 \n",
       "L 1831 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-41\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 2059 -325 \n",
       "Q 1816 -950 1584 -1140 \n",
       "Q 1353 -1331 966 -1331 \n",
       "L 506 -1331 \n",
       "L 506 -850 \n",
       "L 844 -850 \n",
       "Q 1081 -850 1212 -737 \n",
       "Q 1344 -625 1503 -206 \n",
       "L 1606 56 \n",
       "L 191 3500 \n",
       "L 800 3500 \n",
       "L 1894 763 \n",
       "L 2988 3500 \n",
       "L 3597 3500 \n",
       "L 2059 -325 \n",
       "z\n",
       "\" id=\"DejaVuSans-79\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 313 2009 \n",
       "L 1997 2009 \n",
       "L 1997 1497 \n",
       "L 313 1497 \n",
       "L 313 2009 \n",
       "z\n",
       "\" id=\"DejaVuSans-2d\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 3513 2113 \n",
       "L 3513 0 \n",
       "L 2938 0 \n",
       "L 2938 2094 \n",
       "Q 2938 2591 2744 2837 \n",
       "Q 2550 3084 2163 3084 \n",
       "Q 1697 3084 1428 2787 \n",
       "Q 1159 2491 1159 1978 \n",
       "L 1159 0 \n",
       "L 581 0 \n",
       "L 581 4863 \n",
       "L 1159 4863 \n",
       "L 1159 2956 \n",
       "Q 1366 3272 1645 3428 \n",
       "Q 1925 3584 2291 3584 \n",
       "Q 2894 3584 3203 3211 \n",
       "Q 3513 2838 3513 2113 \n",
       "z\n",
       "\" id=\"DejaVuSans-68\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 2906 2969 \n",
       "L 2906 4863 \n",
       "L 3481 4863 \n",
       "L 3481 0 \n",
       "L 2906 0 \n",
       "L 2906 525 \n",
       "Q 2725 213 2448 61 \n",
       "Q 2172 -91 1784 -91 \n",
       "Q 1150 -91 751 415 \n",
       "Q 353 922 353 1747 \n",
       "Q 353 2572 751 3078 \n",
       "Q 1150 3584 1784 3584 \n",
       "Q 2172 3584 2448 3432 \n",
       "Q 2725 3281 2906 2969 \n",
       "z\n",
       "M 947 1747 \n",
       "Q 947 1113 1208 752 \n",
       "Q 1469 391 1925 391 \n",
       "Q 2381 391 2643 752 \n",
       "Q 2906 1113 2906 1747 \n",
       "Q 2906 2381 2643 2742 \n",
       "Q 2381 3103 1925 3103 \n",
       "Q 1469 3103 1208 2742 \n",
       "Q 947 2381 947 1747 \n",
       "z\n",
       "\" id=\"DejaVuSans-64\" transform=\"scale(0.015625)\"/>\n",
       "     </defs>\n",
       "     <use xlink:href=\"#DejaVuSans-54\"/>\n",
       "     <use x=\"44.583984\" xlink:href=\"#DejaVuSans-77\"/>\n",
       "     <use x=\"126.371094\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "     <use x=\"154.154297\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "     <use x=\"193.363281\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "     <use x=\"232.572266\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "     <use x=\"294.095703\" xlink:href=\"#DejaVuSans-72\"/>\n",
       "     <use x=\"335.208984\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"366.996094\" xlink:href=\"#DejaVuSans-54\"/>\n",
       "     <use x=\"411.080078\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "     <use x=\"470.853516\" xlink:href=\"#DejaVuSans-78\"/>\n",
       "     <use x=\"530.033203\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "     <use x=\"569.242188\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"601.029297\" xlink:href=\"#DejaVuSans-44\"/>\n",
       "     <use x=\"678.03125\" xlink:href=\"#DejaVuSans-75\"/>\n",
       "     <use x=\"741.410156\" xlink:href=\"#DejaVuSans-70\"/>\n",
       "     <use x=\"804.886719\" xlink:href=\"#DejaVuSans-6c\"/>\n",
       "     <use x=\"832.669922\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "     <use x=\"860.453125\" xlink:href=\"#DejaVuSans-63\"/>\n",
       "     <use x=\"915.433594\" xlink:href=\"#DejaVuSans-61\"/>\n",
       "     <use x=\"976.712891\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "     <use x=\"1015.921875\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "     <use x=\"1043.705078\" xlink:href=\"#DejaVuSans-6f\"/>\n",
       "     <use x=\"1104.886719\" xlink:href=\"#DejaVuSans-6e\"/>\n",
       "     <use x=\"1168.265625\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"1200.052734\" xlink:href=\"#DejaVuSans-41\"/>\n",
       "     <use x=\"1268.460938\" xlink:href=\"#DejaVuSans-6e\"/>\n",
       "     <use x=\"1331.839844\" xlink:href=\"#DejaVuSans-61\"/>\n",
       "     <use x=\"1393.119141\" xlink:href=\"#DejaVuSans-6c\"/>\n",
       "     <use x=\"1420.902344\" xlink:href=\"#DejaVuSans-79\"/>\n",
       "     <use x=\"1480.082031\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "     <use x=\"1532.181641\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "     <use x=\"1559.964844\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "     <use x=\"1612.064453\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"1643.851562\" xlink:href=\"#DejaVuSans-2d\"/>\n",
       "     <use x=\"1679.935547\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"1711.722656\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "     <use x=\"1775.345703\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "     <use x=\"1807.132812\" xlink:href=\"#DejaVuSans-33\"/>\n",
       "     <use x=\"1870.755859\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"1902.542969\" xlink:href=\"#DejaVuSans-54\"/>\n",
       "     <use x=\"1963.626953\" xlink:href=\"#DejaVuSans-68\"/>\n",
       "     <use x=\"2027.005859\" xlink:href=\"#DejaVuSans-72\"/>\n",
       "     <use x=\"2065.869141\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "     <use x=\"2127.392578\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "     <use x=\"2179.492188\" xlink:href=\"#DejaVuSans-68\"/>\n",
       "     <use x=\"2242.871094\" xlink:href=\"#DejaVuSans-6f\"/>\n",
       "     <use x=\"2304.052734\" xlink:href=\"#DejaVuSans-6c\"/>\n",
       "     <use x=\"2331.835938\" xlink:href=\"#DejaVuSans-64\"/>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"legend_1\">\n",
       "    <g id=\"patch_9\">\n",
       "     <path d=\"M 53.0125 61.953906 \n",
       "L 190.782812 61.953906 \n",
       "Q 192.782812 61.953906 192.782812 59.953906 \n",
       "L 192.782812 31.597656 \n",
       "Q 192.782812 29.597656 190.782812 29.597656 \n",
       "L 53.0125 29.597656 \n",
       "Q 51.0125 29.597656 51.0125 31.597656 \n",
       "L 51.0125 59.953906 \n",
       "Q 51.0125 61.953906 53.0125 61.953906 \n",
       "z\n",
       "\" style=\"fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;\"/>\n",
       "    </g>\n",
       "    <g id=\"patch_10\">\n",
       "     <path d=\"M 55.0125 41.196094 \n",
       "L 75.0125 41.196094 \n",
       "L 75.0125 34.196094 \n",
       "L 55.0125 34.196094 \n",
       "z\n",
       "\" style=\"fill:#d62728;\"/>\n",
       "    </g>\n",
       "    <g id=\"text_13\">\n",
       "     <!-- Near Duplicates - .30 -->\n",
       "     <g transform=\"translate(83.0125 41.196094)scale(0.1 -0.1)\">\n",
       "      <defs>\n",
       "       <path d=\"M 628 4666 \n",
       "L 1478 4666 \n",
       "L 3547 763 \n",
       "L 3547 4666 \n",
       "L 4159 4666 \n",
       "L 4159 0 \n",
       "L 3309 0 \n",
       "L 1241 3903 \n",
       "L 1241 0 \n",
       "L 628 0 \n",
       "L 628 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-4e\" transform=\"scale(0.015625)\"/>\n",
       "      </defs>\n",
       "      <use xlink:href=\"#DejaVuSans-4e\"/>\n",
       "      <use x=\"74.804688\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "      <use x=\"136.328125\" xlink:href=\"#DejaVuSans-61\"/>\n",
       "      <use x=\"197.607422\" xlink:href=\"#DejaVuSans-72\"/>\n",
       "      <use x=\"238.720703\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"270.507812\" xlink:href=\"#DejaVuSans-44\"/>\n",
       "      <use x=\"347.509766\" xlink:href=\"#DejaVuSans-75\"/>\n",
       "      <use x=\"410.888672\" xlink:href=\"#DejaVuSans-70\"/>\n",
       "      <use x=\"474.365234\" xlink:href=\"#DejaVuSans-6c\"/>\n",
       "      <use x=\"502.148438\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "      <use x=\"529.931641\" xlink:href=\"#DejaVuSans-63\"/>\n",
       "      <use x=\"584.912109\" xlink:href=\"#DejaVuSans-61\"/>\n",
       "      <use x=\"646.191406\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "      <use x=\"685.400391\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "      <use x=\"746.923828\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "      <use x=\"799.023438\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"830.810547\" xlink:href=\"#DejaVuSans-2d\"/>\n",
       "      <use x=\"866.894531\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"898.681641\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "      <use x=\"930.46875\" xlink:href=\"#DejaVuSans-33\"/>\n",
       "      <use x=\"994.091797\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"patch_11\">\n",
       "     <path d=\"M 55.0125 55.874219 \n",
       "L 75.0125 55.874219 \n",
       "L 75.0125 48.874219 \n",
       "L 55.0125 48.874219 \n",
       "z\n",
       "\" style=\"fill:#2ca02c;\"/>\n",
       "    </g>\n",
       "    <g id=\"text_14\">\n",
       "     <!-- Uniques - .30 -->\n",
       "     <g transform=\"translate(83.0125 55.874219)scale(0.1 -0.1)\">\n",
       "      <use xlink:href=\"#DejaVuSans-55\"/>\n",
       "      <use x=\"73.193359\" xlink:href=\"#DejaVuSans-6e\"/>\n",
       "      <use x=\"136.572266\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "      <use x=\"164.355469\" xlink:href=\"#DejaVuSans-71\"/>\n",
       "      <use x=\"227.832031\" xlink:href=\"#DejaVuSans-75\"/>\n",
       "      <use x=\"291.210938\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "      <use x=\"352.734375\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "      <use x=\"404.833984\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"436.621094\" xlink:href=\"#DejaVuSans-2d\"/>\n",
       "      <use x=\"472.705078\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"504.492188\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "      <use x=\"536.279297\" xlink:href=\"#DejaVuSans-33\"/>\n",
       "      <use x=\"599.902344\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "  </g>\n",
       " </g>\n",
       " <defs>\n",
       "  <clipPath id=\"pabeae22aee\">\n",
       "   <rect height=\"266.112\" width=\"357.12\" x=\"46.0125\" y=\"24.597656\"/>\n",
       "  </clipPath>\n",
       " </defs>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dups_df = pd.DataFrame.from_dict({'Near Duplicates - .30': [dups_30], 'Uniques - .30': [uniques_30]})\n",
    "\n",
    "ax=dups_df.plot(kind = 'bar',y=['Near Duplicates - .30', 'Uniques - .30'], fontsize=10, color=['C3', 'C2'], align='center', width=0.8, xlabel=\"Duplicates vs. Unique\")\n",
    "ax.set_title('Twitter Text Duplication Analysis - 0.3 Threshold', fontsize=15)\n",
    "for p in ax.patches:\n",
    "       ax.annotate(format(p.get_height(), '.1f'), \n",
    "                   (p.get_x() + p.get_width() / 2., p.get_height()/2), \n",
    "                   ha = 'center', va = 'center', \n",
    "                   xytext = (0, 9), \n",
    "                   textcoords = 'offset points') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "63cb3eae-2d59-4a14-8e0f-f5500757272a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>distCol</th><th>id_A</th><th>id_B</th><th>tweet_text_A</th><th>tweet_text_B</th></tr>\n",
       "<tr><td>0.0</td><td>7077</td><td>8679</td><td>{the university o...</td><td>{the university o...</td></tr>\n",
       "<tr><td>0.19999999999999996</td><td>6363</td><td>12841</td><td>{afghanistan’s ta...</td><td>{#afghanistan&#x27;s #...</td></tr>\n",
       "<tr><td>0.125</td><td>1782</td><td>3007</td><td>{❔🔬  do you know...</td><td>{❔🔢  do you know...</td></tr>\n",
       "<tr><td>0.1428571428571429</td><td>155</td><td>4755</td><td>{icymi: a 6-year-...</td><td>{police in the vi...</td></tr>\n",
       "<tr><td>0.2857142857142857</td><td>9034</td><td>9850</td><td>{clearview at wel...</td><td>{firelands at cle...</td></tr>\n",
       "<tr><td>0.0</td><td>2162</td><td>5419</td><td>{friday night spo...</td><td>{friday night spo...</td></tr>\n",
       "<tr><td>0.2857142857142857</td><td>110</td><td>6936</td><td>{high school foot...</td><td>{high school foot...</td></tr>\n",
       "<tr><td>0.0</td><td>7059</td><td>11736</td><td>{bay area news gr...</td><td>{bay area news gr...</td></tr>\n",
       "<tr><td>0.0</td><td>3036</td><td>6272</td><td>{five takeaways f...</td><td>{five takeaways f...</td></tr>\n",
       "<tr><td>0.2857142857142857</td><td>1021</td><td>9006</td><td>{virginia school ...</td><td>{school officials...</td></tr>\n",
       "<tr><td>0.25</td><td>852</td><td>4869</td><td>{here is this wee...</td><td>{week 5 high scho...</td></tr>\n",
       "<tr><td>0.19999999999999996</td><td>1721</td><td>2848</td><td>{suspect dead, sh...</td><td>{suspect shot dea...</td></tr>\n",
       "<tr><td>0.25</td><td>8126</td><td>8774</td><td>{daily breeze wee...</td><td>{daily breeze hig...</td></tr>\n",
       "<tr><td>0.0</td><td>5613</td><td>8041</td><td>{high school scor...</td><td>{high school scor...</td></tr>\n",
       "<tr><td>0.25</td><td>6781</td><td>11672</td><td>{high school scor...</td><td>{high school foot...</td></tr>\n",
       "<tr><td>0.0</td><td>52</td><td>4452</td><td>{teacher shortfal...</td><td>{teacher shortfal...</td></tr>\n",
       "<tr><td>0.0</td><td>6846</td><td>10147</td><td>{high school scor...</td><td>{high school scor...</td></tr>\n",
       "<tr><td>0.0</td><td>2496</td><td>4060</td><td>{if you&#x27;re strugg...</td><td>{if you&#x27;re strugg...</td></tr>\n",
       "<tr><td>0.0</td><td>3274</td><td>4486</td><td>{texas massacre s...</td><td>{texas massacre s...</td></tr>\n",
       "<tr><td>0.0</td><td>3118</td><td>12132</td><td>{investigators in...</td><td>{investigators in...</td></tr>\n",
       "</table>\n",
       "only showing top 20 rows\n"
      ],
      "text/plain": [
       "+-------------------+-----+-----+--------------------+--------------------+\n",
       "|            distCol| id_A| id_B|        tweet_text_A|        tweet_text_B|\n",
       "+-------------------+-----+-----+--------------------+--------------------+\n",
       "|                0.0| 6123| 8475|{week 12 connecti...|{week 8 connectic...|\n",
       "|                0.0| 6595| 7657|{the best destina...|{the best destina...|\n",
       "|                0.0| 8751| 8752|{female high scho...|{female high scho...|\n",
       "|                0.0| 4746| 6781|{high school scor...|{high school scor...|\n",
       "|                0.0|10839|12486|{acting uvalde po...|{acting uvalde po...|\n",
       "|                0.0| 9545|12347|{poll: which high...|{poll: which high...|\n",
       "|               0.25|11418|11672|{high school scor...|{high school foot...|\n",
       "|               0.25| 2610| 9292|{vote for the hig...|{vote for the hig...|\n",
       "|                0.0|  120| 4680|{what did police ...|{what did police ...|\n",
       "|0.19999999999999996| 6050| 6518|{meet the nominee...|{here are the nom...|\n",
       "|0.19999999999999996| 6050|11562|{meet the nominee...|{here are the nom...|\n",
       "| 0.2857142857142857| 5574| 7994|{see the woodstow...|{see the irvingto...|\n",
       "|                0.0| 5324|12781|{high school http...|{high school http...|\n",
       "|0.10526315789473684| 3785| 9588|{the robb element...|{state police say...|\n",
       "|0.09999999999999998| 2746|12406|{❔🔢 can you solv...|{❓🔢 can you solv...|\n",
       "| 0.2727272727272727| 1201|11860|{for subscribers:...|{for subscribers:...|\n",
       "| 0.2857142857142857| 2990| 7994|{see the madison ...|{see the irvingto...|\n",
       "|0.16666666666666663| 6096|10294|{poll: which high...|{poll: which dade...|\n",
       "|                0.0| 6096| 9545|{poll: which high...|{poll: which high...|\n",
       "|               0.25| 4740| 9462|{high school scor...|{high school scor...|\n",
       "+-------------------+-----+-----+--------------------+--------------------+\n",
       "only showing top 20 rows"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dups_30_tweet_text.unpersist()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "403de005-5263-441c-8f6f-5153c1605ab5",
   "metadata": {},
   "source": [
    "#### 0.5 Distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "bc7ffa28-68d3-4f32-a127-382851726a91",
   "metadata": {},
   "outputs": [],
   "source": [
    "jaccard_distance = 0.5\n",
    "\n",
    "df_dups_tweet_text = model.approxSimilarityJoin(df_hashed_tweet_text_samp, df_hashed_tweet_text_samp, jaccard_distance).filter(\"datasetA.id < datasetB.id\").select(\n",
    "            col(\"distCol\"),\n",
    "            col(\"datasetA.id\").alias(\"id_A\"),\n",
    "            col(\"datasetB.id\").alias(\"id_B\"),\n",
    "            col('datasetA.tweet_text').alias('tweet_text_A'),\n",
    "            col('datasetB.tweet_text').alias('tweet_text_B'),\n",
    "#             col('datasetA.list_of_words').alias('words_A'),\n",
    "#             col('datasetB.list_of_words').alias('words_B')\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "0dc87d00-ba2e-48f7-9c1b-90b880c42b7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>distCol</th>\n",
       "      <th>id_A</th>\n",
       "      <th>id_B</th>\n",
       "      <th>tweet_text_A</th>\n",
       "      <th>tweet_text_B</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>636</td>\n",
       "      <td>13088</td>\n",
       "      <td>(opponents of school choice regularly laud traditional public schools for being \"open to all.\" but they often aren't, with geographic boundaries that require hefty mortgages and property taxes or ...</td>\n",
       "      <td>(opponents of school choice regularly laud traditional public schools for being \"open to all.\" but they often aren't, with geographic boundaries that require hefty mortgages and property taxes or ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.400000</td>\n",
       "      <td>11672</td>\n",
       "      <td>12843</td>\n",
       "      <td>(high school football: week 1 scoreboard https://t.co/eqh0ptjoay,)</td>\n",
       "      <td>(1st &amp;amp; 10: week 10 of acadiana’s high school football [video] https://t.co/olsvjzukuq,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1742</td>\n",
       "      <td>10127</td>\n",
       "      <td>(florida school shooter nikolas cruz will be sentenced to life without parole for the 2018 murder of 17 people at parkland’s marjory stoneman douglas high school. \\n\\nhttps://t.co/bjy2sonuly,)</td>\n",
       "      <td>(florida school shooter nikolas cruz will be sentenced to life without parole for the 2018 murder of 17 people at parkland’s marjory stoneman douglas high school. https://t.co/hybgtgkma6,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.333333</td>\n",
       "      <td>7962</td>\n",
       "      <td>8839</td>\n",
       "      <td>(high school https://t.co/a2a0tbwnqc,)</td>\n",
       "      <td>(and that’s a sunset of high school football. https://t.co/kltovfgapo,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.333333</td>\n",
       "      <td>7962</td>\n",
       "      <td>8618</td>\n",
       "      <td>(high school https://t.co/a2a0tbwnqc,)</td>\n",
       "      <td>(nj high school may be site of 'cancer cluster' https://t.co/lery6xrhao,)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    distCol   id_A   id_B  \\\n",
       "0  0.000000    636  13088   \n",
       "1  0.400000  11672  12843   \n",
       "2  0.000000   1742  10127   \n",
       "3  0.333333   7962   8839   \n",
       "4  0.333333   7962   8618   \n",
       "\n",
       "                                                                                                                                                                                              tweet_text_A  \\\n",
       "0  (opponents of school choice regularly laud traditional public schools for being \"open to all.\" but they often aren't, with geographic boundaries that require hefty mortgages and property taxes or ...   \n",
       "1                                                                                                                                       (high school football: week 1 scoreboard https://t.co/eqh0ptjoay,)   \n",
       "2         (florida school shooter nikolas cruz will be sentenced to life without parole for the 2018 murder of 17 people at parkland’s marjory stoneman douglas high school. \\n\\nhttps://t.co/bjy2sonuly,)   \n",
       "3                                                                                                                                                                   (high school https://t.co/a2a0tbwnqc,)   \n",
       "4                                                                                                                                                                   (high school https://t.co/a2a0tbwnqc,)   \n",
       "\n",
       "                                                                                                                                                                                              tweet_text_B  \n",
       "0  (opponents of school choice regularly laud traditional public schools for being \"open to all.\" but they often aren't, with geographic boundaries that require hefty mortgages and property taxes or ...  \n",
       "1                                                                                                              (1st &amp; 10: week 10 of acadiana’s high school football [video] https://t.co/olsvjzukuq,)  \n",
       "2             (florida school shooter nikolas cruz will be sentenced to life without parole for the 2018 murder of 17 people at parkland’s marjory stoneman douglas high school. https://t.co/hybgtgkma6,)  \n",
       "3                                                                                                                                  (and that’s a sunset of high school football. https://t.co/kltovfgapo,)  \n",
       "4                                                                                                                                (nj high school may be site of 'cancer cluster' https://t.co/lery6xrhao,)  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dups_50_tweet_text = df_dups_tweet_text\n",
    "df_dups_50_tweet_text.cache()\n",
    "df_dups_50_tweet_text.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "1efa4647-830f-4985-aac8-53d318c328ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 190:================================================>    (184 + 4) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records:  13594\n",
      "Duplicate titles based on { 0.5 } jaccard distance:  487\n",
      "Unique titles based on { 0.5 } jaccard distance:  :  13107\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "records = df_hashed_tweet_text_samp.count()\n",
    "dups_50 = df_dups_50_tweet_text.select('id_A').distinct().count()\n",
    "uniques_50 = records - dups_50\n",
    "\n",
    "print ('Total records: ', records)\n",
    "print ('Duplicate titles based on {', jaccard_distance, '} jaccard distance: ', dups_50)\n",
    "print ('Unique titles based on {', jaccard_distance, '} jaccard distance: ', ': ', uniques_50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "d7b456dd-e33f-4dd9-a8b7-39b0d05e8f33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       "  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<svg height=\"324.950281pt\" version=\"1.1\" viewBox=\"0 0 411.415078 324.950281\" width=\"411.415078pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       " <metadata>\n",
       "  <rdf:RDF xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n",
       "   <cc:Work>\n",
       "    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\n",
       "    <dc:date>2023-03-09T00:51:48.762043</dc:date>\n",
       "    <dc:format>image/svg+xml</dc:format>\n",
       "    <dc:creator>\n",
       "     <cc:Agent>\n",
       "      <dc:title>Matplotlib v3.4.3, https://matplotlib.org/</dc:title>\n",
       "     </cc:Agent>\n",
       "    </dc:creator>\n",
       "   </cc:Work>\n",
       "  </rdf:RDF>\n",
       " </metadata>\n",
       " <defs>\n",
       "  <style type=\"text/css\">*{stroke-linecap:butt;stroke-linejoin:round;}</style>\n",
       " </defs>\n",
       " <g id=\"figure_1\">\n",
       "  <g id=\"patch_1\">\n",
       "   <path d=\"M 0 324.950281 \n",
       "L 411.415078 324.950281 \n",
       "L 411.415078 0 \n",
       "L 0 0 \n",
       "z\n",
       "\" style=\"fill:#ffffff;\"/>\n",
       "  </g>\n",
       "  <g id=\"axes_1\">\n",
       "   <g id=\"patch_2\">\n",
       "    <path d=\"M 46.0125 290.709656 \n",
       "L 403.1325 290.709656 \n",
       "L 403.1325 24.597656 \n",
       "L 46.0125 24.597656 \n",
       "z\n",
       "\" style=\"fill:#ffffff;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_3\">\n",
       "    <path clip-path=\"url(#p53320dc66b)\" d=\"M 114.689423 290.709656 \n",
       "L 224.5725 290.709656 \n",
       "L 224.5725 281.292911 \n",
       "L 114.689423 281.292911 \n",
       "z\n",
       "\" style=\"fill:#d62728;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_4\">\n",
       "    <path clip-path=\"url(#p53320dc66b)\" d=\"M 224.5725 290.709656 \n",
       "L 334.455577 290.709656 \n",
       "L 334.455577 37.269656 \n",
       "L 224.5725 37.269656 \n",
       "z\n",
       "\" style=\"fill:#2ca02c;\"/>\n",
       "   </g>\n",
       "   <g id=\"matplotlib.axis_1\">\n",
       "    <g id=\"xtick_1\">\n",
       "     <g id=\"line2d_1\">\n",
       "      <defs>\n",
       "       <path d=\"M 0 0 \n",
       "L 0 3.5 \n",
       "\" id=\"ma61c38aa84\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"224.5725\" xlink:href=\"#ma61c38aa84\" y=\"290.709656\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_1\">\n",
       "      <!-- 0 -->\n",
       "      <g transform=\"translate(227.331875 304.072156)rotate(-90)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 2034 4250 \n",
       "Q 1547 4250 1301 3770 \n",
       "Q 1056 3291 1056 2328 \n",
       "Q 1056 1369 1301 889 \n",
       "Q 1547 409 2034 409 \n",
       "Q 2525 409 2770 889 \n",
       "Q 3016 1369 3016 2328 \n",
       "Q 3016 3291 2770 3770 \n",
       "Q 2525 4250 2034 4250 \n",
       "z\n",
       "M 2034 4750 \n",
       "Q 2819 4750 3233 4129 \n",
       "Q 3647 3509 3647 2328 \n",
       "Q 3647 1150 3233 529 \n",
       "Q 2819 -91 2034 -91 \n",
       "Q 1250 -91 836 529 \n",
       "Q 422 1150 422 2328 \n",
       "Q 422 3509 836 4129 \n",
       "Q 1250 4750 2034 4750 \n",
       "z\n",
       "\" id=\"DejaVuSans-30\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"text_2\">\n",
       "     <!-- Duplicates vs. Unique -->\n",
       "     <g transform=\"translate(170.17875 315.670594)scale(0.1 -0.1)\">\n",
       "      <defs>\n",
       "       <path d=\"M 1259 4147 \n",
       "L 1259 519 \n",
       "L 2022 519 \n",
       "Q 2988 519 3436 956 \n",
       "Q 3884 1394 3884 2338 \n",
       "Q 3884 3275 3436 3711 \n",
       "Q 2988 4147 2022 4147 \n",
       "L 1259 4147 \n",
       "z\n",
       "M 628 4666 \n",
       "L 1925 4666 \n",
       "Q 3281 4666 3915 4102 \n",
       "Q 4550 3538 4550 2338 \n",
       "Q 4550 1131 3912 565 \n",
       "Q 3275 0 1925 0 \n",
       "L 628 0 \n",
       "L 628 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-44\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 544 1381 \n",
       "L 544 3500 \n",
       "L 1119 3500 \n",
       "L 1119 1403 \n",
       "Q 1119 906 1312 657 \n",
       "Q 1506 409 1894 409 \n",
       "Q 2359 409 2629 706 \n",
       "Q 2900 1003 2900 1516 \n",
       "L 2900 3500 \n",
       "L 3475 3500 \n",
       "L 3475 0 \n",
       "L 2900 0 \n",
       "L 2900 538 \n",
       "Q 2691 219 2414 64 \n",
       "Q 2138 -91 1772 -91 \n",
       "Q 1169 -91 856 284 \n",
       "Q 544 659 544 1381 \n",
       "z\n",
       "M 1991 3584 \n",
       "L 1991 3584 \n",
       "z\n",
       "\" id=\"DejaVuSans-75\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 1159 525 \n",
       "L 1159 -1331 \n",
       "L 581 -1331 \n",
       "L 581 3500 \n",
       "L 1159 3500 \n",
       "L 1159 2969 \n",
       "Q 1341 3281 1617 3432 \n",
       "Q 1894 3584 2278 3584 \n",
       "Q 2916 3584 3314 3078 \n",
       "Q 3713 2572 3713 1747 \n",
       "Q 3713 922 3314 415 \n",
       "Q 2916 -91 2278 -91 \n",
       "Q 1894 -91 1617 61 \n",
       "Q 1341 213 1159 525 \n",
       "z\n",
       "M 3116 1747 \n",
       "Q 3116 2381 2855 2742 \n",
       "Q 2594 3103 2138 3103 \n",
       "Q 1681 3103 1420 2742 \n",
       "Q 1159 2381 1159 1747 \n",
       "Q 1159 1113 1420 752 \n",
       "Q 1681 391 2138 391 \n",
       "Q 2594 391 2855 752 \n",
       "Q 3116 1113 3116 1747 \n",
       "z\n",
       "\" id=\"DejaVuSans-70\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 603 4863 \n",
       "L 1178 4863 \n",
       "L 1178 0 \n",
       "L 603 0 \n",
       "L 603 4863 \n",
       "z\n",
       "\" id=\"DejaVuSans-6c\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 603 3500 \n",
       "L 1178 3500 \n",
       "L 1178 0 \n",
       "L 603 0 \n",
       "L 603 3500 \n",
       "z\n",
       "M 603 4863 \n",
       "L 1178 4863 \n",
       "L 1178 4134 \n",
       "L 603 4134 \n",
       "L 603 4863 \n",
       "z\n",
       "\" id=\"DejaVuSans-69\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 3122 3366 \n",
       "L 3122 2828 \n",
       "Q 2878 2963 2633 3030 \n",
       "Q 2388 3097 2138 3097 \n",
       "Q 1578 3097 1268 2742 \n",
       "Q 959 2388 959 1747 \n",
       "Q 959 1106 1268 751 \n",
       "Q 1578 397 2138 397 \n",
       "Q 2388 397 2633 464 \n",
       "Q 2878 531 3122 666 \n",
       "L 3122 134 \n",
       "Q 2881 22 2623 -34 \n",
       "Q 2366 -91 2075 -91 \n",
       "Q 1284 -91 818 406 \n",
       "Q 353 903 353 1747 \n",
       "Q 353 2603 823 3093 \n",
       "Q 1294 3584 2113 3584 \n",
       "Q 2378 3584 2631 3529 \n",
       "Q 2884 3475 3122 3366 \n",
       "z\n",
       "\" id=\"DejaVuSans-63\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2194 1759 \n",
       "Q 1497 1759 1228 1600 \n",
       "Q 959 1441 959 1056 \n",
       "Q 959 750 1161 570 \n",
       "Q 1363 391 1709 391 \n",
       "Q 2188 391 2477 730 \n",
       "Q 2766 1069 2766 1631 \n",
       "L 2766 1759 \n",
       "L 2194 1759 \n",
       "z\n",
       "M 3341 1997 \n",
       "L 3341 0 \n",
       "L 2766 0 \n",
       "L 2766 531 \n",
       "Q 2569 213 2275 61 \n",
       "Q 1981 -91 1556 -91 \n",
       "Q 1019 -91 701 211 \n",
       "Q 384 513 384 1019 \n",
       "Q 384 1609 779 1909 \n",
       "Q 1175 2209 1959 2209 \n",
       "L 2766 2209 \n",
       "L 2766 2266 \n",
       "Q 2766 2663 2505 2880 \n",
       "Q 2244 3097 1772 3097 \n",
       "Q 1472 3097 1187 3025 \n",
       "Q 903 2953 641 2809 \n",
       "L 641 3341 \n",
       "Q 956 3463 1253 3523 \n",
       "Q 1550 3584 1831 3584 \n",
       "Q 2591 3584 2966 3190 \n",
       "Q 3341 2797 3341 1997 \n",
       "z\n",
       "\" id=\"DejaVuSans-61\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 1172 4494 \n",
       "L 1172 3500 \n",
       "L 2356 3500 \n",
       "L 2356 3053 \n",
       "L 1172 3053 \n",
       "L 1172 1153 \n",
       "Q 1172 725 1289 603 \n",
       "Q 1406 481 1766 481 \n",
       "L 2356 481 \n",
       "L 2356 0 \n",
       "L 1766 0 \n",
       "Q 1100 0 847 248 \n",
       "Q 594 497 594 1153 \n",
       "L 594 3053 \n",
       "L 172 3053 \n",
       "L 172 3500 \n",
       "L 594 3500 \n",
       "L 594 4494 \n",
       "L 1172 4494 \n",
       "z\n",
       "\" id=\"DejaVuSans-74\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 3597 1894 \n",
       "L 3597 1613 \n",
       "L 953 1613 \n",
       "Q 991 1019 1311 708 \n",
       "Q 1631 397 2203 397 \n",
       "Q 2534 397 2845 478 \n",
       "Q 3156 559 3463 722 \n",
       "L 3463 178 \n",
       "Q 3153 47 2828 -22 \n",
       "Q 2503 -91 2169 -91 \n",
       "Q 1331 -91 842 396 \n",
       "Q 353 884 353 1716 \n",
       "Q 353 2575 817 3079 \n",
       "Q 1281 3584 2069 3584 \n",
       "Q 2775 3584 3186 3129 \n",
       "Q 3597 2675 3597 1894 \n",
       "z\n",
       "M 3022 2063 \n",
       "Q 3016 2534 2758 2815 \n",
       "Q 2500 3097 2075 3097 \n",
       "Q 1594 3097 1305 2825 \n",
       "Q 1016 2553 972 2059 \n",
       "L 3022 2063 \n",
       "z\n",
       "\" id=\"DejaVuSans-65\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2834 3397 \n",
       "L 2834 2853 \n",
       "Q 2591 2978 2328 3040 \n",
       "Q 2066 3103 1784 3103 \n",
       "Q 1356 3103 1142 2972 \n",
       "Q 928 2841 928 2578 \n",
       "Q 928 2378 1081 2264 \n",
       "Q 1234 2150 1697 2047 \n",
       "L 1894 2003 \n",
       "Q 2506 1872 2764 1633 \n",
       "Q 3022 1394 3022 966 \n",
       "Q 3022 478 2636 193 \n",
       "Q 2250 -91 1575 -91 \n",
       "Q 1294 -91 989 -36 \n",
       "Q 684 19 347 128 \n",
       "L 347 722 \n",
       "Q 666 556 975 473 \n",
       "Q 1284 391 1588 391 \n",
       "Q 1994 391 2212 530 \n",
       "Q 2431 669 2431 922 \n",
       "Q 2431 1156 2273 1281 \n",
       "Q 2116 1406 1581 1522 \n",
       "L 1381 1569 \n",
       "Q 847 1681 609 1914 \n",
       "Q 372 2147 372 2553 \n",
       "Q 372 3047 722 3315 \n",
       "Q 1072 3584 1716 3584 \n",
       "Q 2034 3584 2315 3537 \n",
       "Q 2597 3491 2834 3397 \n",
       "z\n",
       "\" id=\"DejaVuSans-73\" transform=\"scale(0.015625)\"/>\n",
       "       <path id=\"DejaVuSans-20\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 191 3500 \n",
       "L 800 3500 \n",
       "L 1894 563 \n",
       "L 2988 3500 \n",
       "L 3597 3500 \n",
       "L 2284 0 \n",
       "L 1503 0 \n",
       "L 191 3500 \n",
       "z\n",
       "\" id=\"DejaVuSans-76\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 684 794 \n",
       "L 1344 794 \n",
       "L 1344 0 \n",
       "L 684 0 \n",
       "L 684 794 \n",
       "z\n",
       "\" id=\"DejaVuSans-2e\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 556 4666 \n",
       "L 1191 4666 \n",
       "L 1191 1831 \n",
       "Q 1191 1081 1462 751 \n",
       "Q 1734 422 2344 422 \n",
       "Q 2950 422 3222 751 \n",
       "Q 3494 1081 3494 1831 \n",
       "L 3494 4666 \n",
       "L 4128 4666 \n",
       "L 4128 1753 \n",
       "Q 4128 841 3676 375 \n",
       "Q 3225 -91 2344 -91 \n",
       "Q 1459 -91 1007 375 \n",
       "Q 556 841 556 1753 \n",
       "L 556 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-55\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 3513 2113 \n",
       "L 3513 0 \n",
       "L 2938 0 \n",
       "L 2938 2094 \n",
       "Q 2938 2591 2744 2837 \n",
       "Q 2550 3084 2163 3084 \n",
       "Q 1697 3084 1428 2787 \n",
       "Q 1159 2491 1159 1978 \n",
       "L 1159 0 \n",
       "L 581 0 \n",
       "L 581 3500 \n",
       "L 1159 3500 \n",
       "L 1159 2956 \n",
       "Q 1366 3272 1645 3428 \n",
       "Q 1925 3584 2291 3584 \n",
       "Q 2894 3584 3203 3211 \n",
       "Q 3513 2838 3513 2113 \n",
       "z\n",
       "\" id=\"DejaVuSans-6e\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 947 1747 \n",
       "Q 947 1113 1208 752 \n",
       "Q 1469 391 1925 391 \n",
       "Q 2381 391 2643 752 \n",
       "Q 2906 1113 2906 1747 \n",
       "Q 2906 2381 2643 2742 \n",
       "Q 2381 3103 1925 3103 \n",
       "Q 1469 3103 1208 2742 \n",
       "Q 947 2381 947 1747 \n",
       "z\n",
       "M 2906 525 \n",
       "Q 2725 213 2448 61 \n",
       "Q 2172 -91 1784 -91 \n",
       "Q 1150 -91 751 415 \n",
       "Q 353 922 353 1747 \n",
       "Q 353 2572 751 3078 \n",
       "Q 1150 3584 1784 3584 \n",
       "Q 2172 3584 2448 3432 \n",
       "Q 2725 3281 2906 2969 \n",
       "L 2906 3500 \n",
       "L 3481 3500 \n",
       "L 3481 -1331 \n",
       "L 2906 -1331 \n",
       "L 2906 525 \n",
       "z\n",
       "\" id=\"DejaVuSans-71\" transform=\"scale(0.015625)\"/>\n",
       "      </defs>\n",
       "      <use xlink:href=\"#DejaVuSans-44\"/>\n",
       "      <use x=\"77.001953\" xlink:href=\"#DejaVuSans-75\"/>\n",
       "      <use x=\"140.380859\" xlink:href=\"#DejaVuSans-70\"/>\n",
       "      <use x=\"203.857422\" xlink:href=\"#DejaVuSans-6c\"/>\n",
       "      <use x=\"231.640625\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "      <use x=\"259.423828\" xlink:href=\"#DejaVuSans-63\"/>\n",
       "      <use x=\"314.404297\" xlink:href=\"#DejaVuSans-61\"/>\n",
       "      <use x=\"375.683594\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "      <use x=\"414.892578\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "      <use x=\"476.416016\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "      <use x=\"528.515625\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"560.302734\" xlink:href=\"#DejaVuSans-76\"/>\n",
       "      <use x=\"619.482422\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "      <use x=\"671.582031\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "      <use x=\"703.369141\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"735.15625\" xlink:href=\"#DejaVuSans-55\"/>\n",
       "      <use x=\"808.349609\" xlink:href=\"#DejaVuSans-6e\"/>\n",
       "      <use x=\"871.728516\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "      <use x=\"899.511719\" xlink:href=\"#DejaVuSans-71\"/>\n",
       "      <use x=\"962.988281\" xlink:href=\"#DejaVuSans-75\"/>\n",
       "      <use x=\"1026.367188\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"matplotlib.axis_2\">\n",
       "    <g id=\"ytick_1\">\n",
       "     <g id=\"line2d_2\">\n",
       "      <defs>\n",
       "       <path d=\"M 0 0 \n",
       "L -3.5 0 \n",
       "\" id=\"m75fa04a3ce\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#m75fa04a3ce\" y=\"290.709656\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_3\">\n",
       "      <!-- 0 -->\n",
       "      <g transform=\"translate(32.65 294.508875)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_2\">\n",
       "     <g id=\"line2d_3\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#m75fa04a3ce\" y=\"252.037191\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_4\">\n",
       "      <!-- 2000 -->\n",
       "      <g transform=\"translate(13.5625 255.83641)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 1228 531 \n",
       "L 3431 531 \n",
       "L 3431 0 \n",
       "L 469 0 \n",
       "L 469 531 \n",
       "Q 828 903 1448 1529 \n",
       "Q 2069 2156 2228 2338 \n",
       "Q 2531 2678 2651 2914 \n",
       "Q 2772 3150 2772 3378 \n",
       "Q 2772 3750 2511 3984 \n",
       "Q 2250 4219 1831 4219 \n",
       "Q 1534 4219 1204 4116 \n",
       "Q 875 4013 500 3803 \n",
       "L 500 4441 \n",
       "Q 881 4594 1212 4672 \n",
       "Q 1544 4750 1819 4750 \n",
       "Q 2544 4750 2975 4387 \n",
       "Q 3406 4025 3406 3419 \n",
       "Q 3406 3131 3298 2873 \n",
       "Q 3191 2616 2906 2266 \n",
       "Q 2828 2175 2409 1742 \n",
       "Q 1991 1309 1228 531 \n",
       "z\n",
       "\" id=\"DejaVuSans-32\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-32\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_3\">\n",
       "     <g id=\"line2d_4\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#m75fa04a3ce\" y=\"213.364726\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_5\">\n",
       "      <!-- 4000 -->\n",
       "      <g transform=\"translate(13.5625 217.163945)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 2419 4116 \n",
       "L 825 1625 \n",
       "L 2419 1625 \n",
       "L 2419 4116 \n",
       "z\n",
       "M 2253 4666 \n",
       "L 3047 4666 \n",
       "L 3047 1625 \n",
       "L 3713 1625 \n",
       "L 3713 1100 \n",
       "L 3047 1100 \n",
       "L 3047 0 \n",
       "L 2419 0 \n",
       "L 2419 1100 \n",
       "L 313 1100 \n",
       "L 313 1709 \n",
       "L 2253 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-34\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-34\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_4\">\n",
       "     <g id=\"line2d_5\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#m75fa04a3ce\" y=\"174.692261\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_6\">\n",
       "      <!-- 6000 -->\n",
       "      <g transform=\"translate(13.5625 178.49148)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 2113 2584 \n",
       "Q 1688 2584 1439 2293 \n",
       "Q 1191 2003 1191 1497 \n",
       "Q 1191 994 1439 701 \n",
       "Q 1688 409 2113 409 \n",
       "Q 2538 409 2786 701 \n",
       "Q 3034 994 3034 1497 \n",
       "Q 3034 2003 2786 2293 \n",
       "Q 2538 2584 2113 2584 \n",
       "z\n",
       "M 3366 4563 \n",
       "L 3366 3988 \n",
       "Q 3128 4100 2886 4159 \n",
       "Q 2644 4219 2406 4219 \n",
       "Q 1781 4219 1451 3797 \n",
       "Q 1122 3375 1075 2522 \n",
       "Q 1259 2794 1537 2939 \n",
       "Q 1816 3084 2150 3084 \n",
       "Q 2853 3084 3261 2657 \n",
       "Q 3669 2231 3669 1497 \n",
       "Q 3669 778 3244 343 \n",
       "Q 2819 -91 2113 -91 \n",
       "Q 1303 -91 875 529 \n",
       "Q 447 1150 447 2328 \n",
       "Q 447 3434 972 4092 \n",
       "Q 1497 4750 2381 4750 \n",
       "Q 2619 4750 2861 4703 \n",
       "Q 3103 4656 3366 4563 \n",
       "z\n",
       "\" id=\"DejaVuSans-36\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-36\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_5\">\n",
       "     <g id=\"line2d_6\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#m75fa04a3ce\" y=\"136.019796\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_7\">\n",
       "      <!-- 8000 -->\n",
       "      <g transform=\"translate(13.5625 139.819015)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 2034 2216 \n",
       "Q 1584 2216 1326 1975 \n",
       "Q 1069 1734 1069 1313 \n",
       "Q 1069 891 1326 650 \n",
       "Q 1584 409 2034 409 \n",
       "Q 2484 409 2743 651 \n",
       "Q 3003 894 3003 1313 \n",
       "Q 3003 1734 2745 1975 \n",
       "Q 2488 2216 2034 2216 \n",
       "z\n",
       "M 1403 2484 \n",
       "Q 997 2584 770 2862 \n",
       "Q 544 3141 544 3541 \n",
       "Q 544 4100 942 4425 \n",
       "Q 1341 4750 2034 4750 \n",
       "Q 2731 4750 3128 4425 \n",
       "Q 3525 4100 3525 3541 \n",
       "Q 3525 3141 3298 2862 \n",
       "Q 3072 2584 2669 2484 \n",
       "Q 3125 2378 3379 2068 \n",
       "Q 3634 1759 3634 1313 \n",
       "Q 3634 634 3220 271 \n",
       "Q 2806 -91 2034 -91 \n",
       "Q 1263 -91 848 271 \n",
       "Q 434 634 434 1313 \n",
       "Q 434 1759 690 2068 \n",
       "Q 947 2378 1403 2484 \n",
       "z\n",
       "M 1172 3481 \n",
       "Q 1172 3119 1398 2916 \n",
       "Q 1625 2713 2034 2713 \n",
       "Q 2441 2713 2670 2916 \n",
       "Q 2900 3119 2900 3481 \n",
       "Q 2900 3844 2670 4047 \n",
       "Q 2441 4250 2034 4250 \n",
       "Q 1625 4250 1398 4047 \n",
       "Q 1172 3844 1172 3481 \n",
       "z\n",
       "\" id=\"DejaVuSans-38\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-38\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_6\">\n",
       "     <g id=\"line2d_7\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#m75fa04a3ce\" y=\"97.347331\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_8\">\n",
       "      <!-- 10000 -->\n",
       "      <g transform=\"translate(7.2 101.14655)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 794 531 \n",
       "L 1825 531 \n",
       "L 1825 4091 \n",
       "L 703 3866 \n",
       "L 703 4441 \n",
       "L 1819 4666 \n",
       "L 2450 4666 \n",
       "L 2450 531 \n",
       "L 3481 531 \n",
       "L 3481 0 \n",
       "L 794 0 \n",
       "L 794 531 \n",
       "z\n",
       "\" id=\"DejaVuSans-31\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-31\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"254.492188\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_7\">\n",
       "     <g id=\"line2d_8\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#m75fa04a3ce\" y=\"58.674866\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_9\">\n",
       "      <!-- 12000 -->\n",
       "      <g transform=\"translate(7.2 62.474084)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#DejaVuSans-31\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-32\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"254.492188\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"patch_5\">\n",
       "    <path d=\"M 46.0125 290.709656 \n",
       "L 46.0125 24.597656 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_6\">\n",
       "    <path d=\"M 403.1325 290.709656 \n",
       "L 403.1325 24.597656 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_7\">\n",
       "    <path d=\"M 46.0125 290.709656 \n",
       "L 403.1325 290.709656 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_8\">\n",
       "    <path d=\"M 46.0125 24.597656 \n",
       "L 403.1325 24.597656 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"text_10\">\n",
       "    <!-- 487.0 -->\n",
       "    <g transform=\"translate(155.316899 279.760659)scale(0.1 -0.1)\">\n",
       "     <defs>\n",
       "      <path d=\"M 525 4666 \n",
       "L 3525 4666 \n",
       "L 3525 4397 \n",
       "L 1831 0 \n",
       "L 1172 0 \n",
       "L 2766 4134 \n",
       "L 525 4134 \n",
       "L 525 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-37\" transform=\"scale(0.015625)\"/>\n",
       "     </defs>\n",
       "     <use xlink:href=\"#DejaVuSans-34\"/>\n",
       "     <use x=\"63.623047\" xlink:href=\"#DejaVuSans-38\"/>\n",
       "     <use x=\"127.246094\" xlink:href=\"#DejaVuSans-37\"/>\n",
       "     <use x=\"190.869141\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "     <use x=\"222.65625\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"text_11\">\n",
       "    <!-- 13107.0 -->\n",
       "    <g transform=\"translate(258.837476 157.749031)scale(0.1 -0.1)\">\n",
       "     <defs>\n",
       "      <path d=\"M 2597 2516 \n",
       "Q 3050 2419 3304 2112 \n",
       "Q 3559 1806 3559 1356 \n",
       "Q 3559 666 3084 287 \n",
       "Q 2609 -91 1734 -91 \n",
       "Q 1441 -91 1130 -33 \n",
       "Q 819 25 488 141 \n",
       "L 488 750 \n",
       "Q 750 597 1062 519 \n",
       "Q 1375 441 1716 441 \n",
       "Q 2309 441 2620 675 \n",
       "Q 2931 909 2931 1356 \n",
       "Q 2931 1769 2642 2001 \n",
       "Q 2353 2234 1838 2234 \n",
       "L 1294 2234 \n",
       "L 1294 2753 \n",
       "L 1863 2753 \n",
       "Q 2328 2753 2575 2939 \n",
       "Q 2822 3125 2822 3475 \n",
       "Q 2822 3834 2567 4026 \n",
       "Q 2313 4219 1838 4219 \n",
       "Q 1578 4219 1281 4162 \n",
       "Q 984 4106 628 3988 \n",
       "L 628 4550 \n",
       "Q 988 4650 1302 4700 \n",
       "Q 1616 4750 1894 4750 \n",
       "Q 2613 4750 3031 4423 \n",
       "Q 3450 4097 3450 3541 \n",
       "Q 3450 3153 3228 2886 \n",
       "Q 3006 2619 2597 2516 \n",
       "z\n",
       "\" id=\"DejaVuSans-33\" transform=\"scale(0.015625)\"/>\n",
       "     </defs>\n",
       "     <use xlink:href=\"#DejaVuSans-31\"/>\n",
       "     <use x=\"63.623047\" xlink:href=\"#DejaVuSans-33\"/>\n",
       "     <use x=\"127.246094\" xlink:href=\"#DejaVuSans-31\"/>\n",
       "     <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "     <use x=\"254.492188\" xlink:href=\"#DejaVuSans-37\"/>\n",
       "     <use x=\"318.115234\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "     <use x=\"349.902344\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"text_12\">\n",
       "    <!-- Twitter Text Duplication Analysis - 0.5 Threshold -->\n",
       "    <g transform=\"translate(44.929922 18.597656)scale(0.15 -0.15)\">\n",
       "     <defs>\n",
       "      <path d=\"M -19 4666 \n",
       "L 3928 4666 \n",
       "L 3928 4134 \n",
       "L 2272 4134 \n",
       "L 2272 0 \n",
       "L 1638 0 \n",
       "L 1638 4134 \n",
       "L -19 4134 \n",
       "L -19 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-54\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 269 3500 \n",
       "L 844 3500 \n",
       "L 1563 769 \n",
       "L 2278 3500 \n",
       "L 2956 3500 \n",
       "L 3675 769 \n",
       "L 4391 3500 \n",
       "L 4966 3500 \n",
       "L 4050 0 \n",
       "L 3372 0 \n",
       "L 2619 2869 \n",
       "L 1863 0 \n",
       "L 1184 0 \n",
       "L 269 3500 \n",
       "z\n",
       "\" id=\"DejaVuSans-77\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 2631 2963 \n",
       "Q 2534 3019 2420 3045 \n",
       "Q 2306 3072 2169 3072 \n",
       "Q 1681 3072 1420 2755 \n",
       "Q 1159 2438 1159 1844 \n",
       "L 1159 0 \n",
       "L 581 0 \n",
       "L 581 3500 \n",
       "L 1159 3500 \n",
       "L 1159 2956 \n",
       "Q 1341 3275 1631 3429 \n",
       "Q 1922 3584 2338 3584 \n",
       "Q 2397 3584 2469 3576 \n",
       "Q 2541 3569 2628 3553 \n",
       "L 2631 2963 \n",
       "z\n",
       "\" id=\"DejaVuSans-72\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 3513 3500 \n",
       "L 2247 1797 \n",
       "L 3578 0 \n",
       "L 2900 0 \n",
       "L 1881 1375 \n",
       "L 863 0 \n",
       "L 184 0 \n",
       "L 1544 1831 \n",
       "L 300 3500 \n",
       "L 978 3500 \n",
       "L 1906 2253 \n",
       "L 2834 3500 \n",
       "L 3513 3500 \n",
       "z\n",
       "\" id=\"DejaVuSans-78\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 1959 3097 \n",
       "Q 1497 3097 1228 2736 \n",
       "Q 959 2375 959 1747 \n",
       "Q 959 1119 1226 758 \n",
       "Q 1494 397 1959 397 \n",
       "Q 2419 397 2687 759 \n",
       "Q 2956 1122 2956 1747 \n",
       "Q 2956 2369 2687 2733 \n",
       "Q 2419 3097 1959 3097 \n",
       "z\n",
       "M 1959 3584 \n",
       "Q 2709 3584 3137 3096 \n",
       "Q 3566 2609 3566 1747 \n",
       "Q 3566 888 3137 398 \n",
       "Q 2709 -91 1959 -91 \n",
       "Q 1206 -91 779 398 \n",
       "Q 353 888 353 1747 \n",
       "Q 353 2609 779 3096 \n",
       "Q 1206 3584 1959 3584 \n",
       "z\n",
       "\" id=\"DejaVuSans-6f\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 2188 4044 \n",
       "L 1331 1722 \n",
       "L 3047 1722 \n",
       "L 2188 4044 \n",
       "z\n",
       "M 1831 4666 \n",
       "L 2547 4666 \n",
       "L 4325 0 \n",
       "L 3669 0 \n",
       "L 3244 1197 \n",
       "L 1141 1197 \n",
       "L 716 0 \n",
       "L 50 0 \n",
       "L 1831 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-41\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 2059 -325 \n",
       "Q 1816 -950 1584 -1140 \n",
       "Q 1353 -1331 966 -1331 \n",
       "L 506 -1331 \n",
       "L 506 -850 \n",
       "L 844 -850 \n",
       "Q 1081 -850 1212 -737 \n",
       "Q 1344 -625 1503 -206 \n",
       "L 1606 56 \n",
       "L 191 3500 \n",
       "L 800 3500 \n",
       "L 1894 763 \n",
       "L 2988 3500 \n",
       "L 3597 3500 \n",
       "L 2059 -325 \n",
       "z\n",
       "\" id=\"DejaVuSans-79\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 313 2009 \n",
       "L 1997 2009 \n",
       "L 1997 1497 \n",
       "L 313 1497 \n",
       "L 313 2009 \n",
       "z\n",
       "\" id=\"DejaVuSans-2d\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 691 4666 \n",
       "L 3169 4666 \n",
       "L 3169 4134 \n",
       "L 1269 4134 \n",
       "L 1269 2991 \n",
       "Q 1406 3038 1543 3061 \n",
       "Q 1681 3084 1819 3084 \n",
       "Q 2600 3084 3056 2656 \n",
       "Q 3513 2228 3513 1497 \n",
       "Q 3513 744 3044 326 \n",
       "Q 2575 -91 1722 -91 \n",
       "Q 1428 -91 1123 -41 \n",
       "Q 819 9 494 109 \n",
       "L 494 744 \n",
       "Q 775 591 1075 516 \n",
       "Q 1375 441 1709 441 \n",
       "Q 2250 441 2565 725 \n",
       "Q 2881 1009 2881 1497 \n",
       "Q 2881 1984 2565 2268 \n",
       "Q 2250 2553 1709 2553 \n",
       "Q 1456 2553 1204 2497 \n",
       "Q 953 2441 691 2322 \n",
       "L 691 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-35\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 3513 2113 \n",
       "L 3513 0 \n",
       "L 2938 0 \n",
       "L 2938 2094 \n",
       "Q 2938 2591 2744 2837 \n",
       "Q 2550 3084 2163 3084 \n",
       "Q 1697 3084 1428 2787 \n",
       "Q 1159 2491 1159 1978 \n",
       "L 1159 0 \n",
       "L 581 0 \n",
       "L 581 4863 \n",
       "L 1159 4863 \n",
       "L 1159 2956 \n",
       "Q 1366 3272 1645 3428 \n",
       "Q 1925 3584 2291 3584 \n",
       "Q 2894 3584 3203 3211 \n",
       "Q 3513 2838 3513 2113 \n",
       "z\n",
       "\" id=\"DejaVuSans-68\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 2906 2969 \n",
       "L 2906 4863 \n",
       "L 3481 4863 \n",
       "L 3481 0 \n",
       "L 2906 0 \n",
       "L 2906 525 \n",
       "Q 2725 213 2448 61 \n",
       "Q 2172 -91 1784 -91 \n",
       "Q 1150 -91 751 415 \n",
       "Q 353 922 353 1747 \n",
       "Q 353 2572 751 3078 \n",
       "Q 1150 3584 1784 3584 \n",
       "Q 2172 3584 2448 3432 \n",
       "Q 2725 3281 2906 2969 \n",
       "z\n",
       "M 947 1747 \n",
       "Q 947 1113 1208 752 \n",
       "Q 1469 391 1925 391 \n",
       "Q 2381 391 2643 752 \n",
       "Q 2906 1113 2906 1747 \n",
       "Q 2906 2381 2643 2742 \n",
       "Q 2381 3103 1925 3103 \n",
       "Q 1469 3103 1208 2742 \n",
       "Q 947 2381 947 1747 \n",
       "z\n",
       "\" id=\"DejaVuSans-64\" transform=\"scale(0.015625)\"/>\n",
       "     </defs>\n",
       "     <use xlink:href=\"#DejaVuSans-54\"/>\n",
       "     <use x=\"44.583984\" xlink:href=\"#DejaVuSans-77\"/>\n",
       "     <use x=\"126.371094\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "     <use x=\"154.154297\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "     <use x=\"193.363281\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "     <use x=\"232.572266\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "     <use x=\"294.095703\" xlink:href=\"#DejaVuSans-72\"/>\n",
       "     <use x=\"335.208984\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"366.996094\" xlink:href=\"#DejaVuSans-54\"/>\n",
       "     <use x=\"411.080078\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "     <use x=\"470.853516\" xlink:href=\"#DejaVuSans-78\"/>\n",
       "     <use x=\"530.033203\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "     <use x=\"569.242188\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"601.029297\" xlink:href=\"#DejaVuSans-44\"/>\n",
       "     <use x=\"678.03125\" xlink:href=\"#DejaVuSans-75\"/>\n",
       "     <use x=\"741.410156\" xlink:href=\"#DejaVuSans-70\"/>\n",
       "     <use x=\"804.886719\" xlink:href=\"#DejaVuSans-6c\"/>\n",
       "     <use x=\"832.669922\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "     <use x=\"860.453125\" xlink:href=\"#DejaVuSans-63\"/>\n",
       "     <use x=\"915.433594\" xlink:href=\"#DejaVuSans-61\"/>\n",
       "     <use x=\"976.712891\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "     <use x=\"1015.921875\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "     <use x=\"1043.705078\" xlink:href=\"#DejaVuSans-6f\"/>\n",
       "     <use x=\"1104.886719\" xlink:href=\"#DejaVuSans-6e\"/>\n",
       "     <use x=\"1168.265625\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"1200.052734\" xlink:href=\"#DejaVuSans-41\"/>\n",
       "     <use x=\"1268.460938\" xlink:href=\"#DejaVuSans-6e\"/>\n",
       "     <use x=\"1331.839844\" xlink:href=\"#DejaVuSans-61\"/>\n",
       "     <use x=\"1393.119141\" xlink:href=\"#DejaVuSans-6c\"/>\n",
       "     <use x=\"1420.902344\" xlink:href=\"#DejaVuSans-79\"/>\n",
       "     <use x=\"1480.082031\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "     <use x=\"1532.181641\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "     <use x=\"1559.964844\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "     <use x=\"1612.064453\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"1643.851562\" xlink:href=\"#DejaVuSans-2d\"/>\n",
       "     <use x=\"1679.935547\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"1711.722656\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "     <use x=\"1775.345703\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "     <use x=\"1807.132812\" xlink:href=\"#DejaVuSans-35\"/>\n",
       "     <use x=\"1870.755859\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"1902.542969\" xlink:href=\"#DejaVuSans-54\"/>\n",
       "     <use x=\"1963.626953\" xlink:href=\"#DejaVuSans-68\"/>\n",
       "     <use x=\"2027.005859\" xlink:href=\"#DejaVuSans-72\"/>\n",
       "     <use x=\"2065.869141\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "     <use x=\"2127.392578\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "     <use x=\"2179.492188\" xlink:href=\"#DejaVuSans-68\"/>\n",
       "     <use x=\"2242.871094\" xlink:href=\"#DejaVuSans-6f\"/>\n",
       "     <use x=\"2304.052734\" xlink:href=\"#DejaVuSans-6c\"/>\n",
       "     <use x=\"2331.835938\" xlink:href=\"#DejaVuSans-64\"/>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"legend_1\">\n",
       "    <g id=\"patch_9\">\n",
       "     <path d=\"M 53.0125 61.953906 \n",
       "L 190.782812 61.953906 \n",
       "Q 192.782812 61.953906 192.782812 59.953906 \n",
       "L 192.782812 31.597656 \n",
       "Q 192.782812 29.597656 190.782812 29.597656 \n",
       "L 53.0125 29.597656 \n",
       "Q 51.0125 29.597656 51.0125 31.597656 \n",
       "L 51.0125 59.953906 \n",
       "Q 51.0125 61.953906 53.0125 61.953906 \n",
       "z\n",
       "\" style=\"fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;\"/>\n",
       "    </g>\n",
       "    <g id=\"patch_10\">\n",
       "     <path d=\"M 55.0125 41.196094 \n",
       "L 75.0125 41.196094 \n",
       "L 75.0125 34.196094 \n",
       "L 55.0125 34.196094 \n",
       "z\n",
       "\" style=\"fill:#d62728;\"/>\n",
       "    </g>\n",
       "    <g id=\"text_13\">\n",
       "     <!-- Near Duplicates - .50 -->\n",
       "     <g transform=\"translate(83.0125 41.196094)scale(0.1 -0.1)\">\n",
       "      <defs>\n",
       "       <path d=\"M 628 4666 \n",
       "L 1478 4666 \n",
       "L 3547 763 \n",
       "L 3547 4666 \n",
       "L 4159 4666 \n",
       "L 4159 0 \n",
       "L 3309 0 \n",
       "L 1241 3903 \n",
       "L 1241 0 \n",
       "L 628 0 \n",
       "L 628 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-4e\" transform=\"scale(0.015625)\"/>\n",
       "      </defs>\n",
       "      <use xlink:href=\"#DejaVuSans-4e\"/>\n",
       "      <use x=\"74.804688\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "      <use x=\"136.328125\" xlink:href=\"#DejaVuSans-61\"/>\n",
       "      <use x=\"197.607422\" xlink:href=\"#DejaVuSans-72\"/>\n",
       "      <use x=\"238.720703\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"270.507812\" xlink:href=\"#DejaVuSans-44\"/>\n",
       "      <use x=\"347.509766\" xlink:href=\"#DejaVuSans-75\"/>\n",
       "      <use x=\"410.888672\" xlink:href=\"#DejaVuSans-70\"/>\n",
       "      <use x=\"474.365234\" xlink:href=\"#DejaVuSans-6c\"/>\n",
       "      <use x=\"502.148438\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "      <use x=\"529.931641\" xlink:href=\"#DejaVuSans-63\"/>\n",
       "      <use x=\"584.912109\" xlink:href=\"#DejaVuSans-61\"/>\n",
       "      <use x=\"646.191406\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "      <use x=\"685.400391\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "      <use x=\"746.923828\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "      <use x=\"799.023438\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"830.810547\" xlink:href=\"#DejaVuSans-2d\"/>\n",
       "      <use x=\"866.894531\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"898.681641\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "      <use x=\"930.46875\" xlink:href=\"#DejaVuSans-35\"/>\n",
       "      <use x=\"994.091797\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"patch_11\">\n",
       "     <path d=\"M 55.0125 55.874219 \n",
       "L 75.0125 55.874219 \n",
       "L 75.0125 48.874219 \n",
       "L 55.0125 48.874219 \n",
       "z\n",
       "\" style=\"fill:#2ca02c;\"/>\n",
       "    </g>\n",
       "    <g id=\"text_14\">\n",
       "     <!-- Uniques - .50 -->\n",
       "     <g transform=\"translate(83.0125 55.874219)scale(0.1 -0.1)\">\n",
       "      <use xlink:href=\"#DejaVuSans-55\"/>\n",
       "      <use x=\"73.193359\" xlink:href=\"#DejaVuSans-6e\"/>\n",
       "      <use x=\"136.572266\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "      <use x=\"164.355469\" xlink:href=\"#DejaVuSans-71\"/>\n",
       "      <use x=\"227.832031\" xlink:href=\"#DejaVuSans-75\"/>\n",
       "      <use x=\"291.210938\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "      <use x=\"352.734375\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "      <use x=\"404.833984\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"436.621094\" xlink:href=\"#DejaVuSans-2d\"/>\n",
       "      <use x=\"472.705078\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"504.492188\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "      <use x=\"536.279297\" xlink:href=\"#DejaVuSans-35\"/>\n",
       "      <use x=\"599.902344\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "  </g>\n",
       " </g>\n",
       " <defs>\n",
       "  <clipPath id=\"p53320dc66b\">\n",
       "   <rect height=\"266.112\" width=\"357.12\" x=\"46.0125\" y=\"24.597656\"/>\n",
       "  </clipPath>\n",
       " </defs>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dups_df = pd.DataFrame.from_dict({'Near Duplicates - .50': [dups_50], 'Uniques - .50': [uniques_50]})\n",
    "\n",
    "ax=dups_df.plot(kind = 'bar',y=['Near Duplicates - .50', 'Uniques - .50'], fontsize=10, color=['C3', 'C2'], align='center', width=0.8, xlabel=\"Duplicates vs. Unique\")\n",
    "ax.set_title('Twitter Text Duplication Analysis - 0.5 Threshold', fontsize=15)\n",
    "for p in ax.patches:\n",
    "       ax.annotate(format(p.get_height(), '.1f'), \n",
    "                   (p.get_x() + p.get_width() / 2., p.get_height()/2), \n",
    "                   ha = 'center', va = 'center', \n",
    "                   xytext = (0, 9), \n",
    "                   textcoords = 'offset points') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "7679c255-dae4-49eb-808d-b4190f9c09f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>distCol</th><th>id_A</th><th>id_B</th><th>tweet_text_A</th><th>tweet_text_B</th></tr>\n",
       "<tr><td>0.4545454545454546</td><td>8209</td><td>12487</td><td>{acting uvalde po...</td><td>{the acting uvald...</td></tr>\n",
       "<tr><td>0.4</td><td>3120</td><td>4869</td><td>{jon hamm on his ...</td><td>{week 5 high scho...</td></tr>\n",
       "<tr><td>0.4545454545454546</td><td>7595</td><td>10414</td><td>{preseason all-ba...</td><td>{bay area news gr...</td></tr>\n",
       "<tr><td>0.4</td><td>3120</td><td>6907</td><td>{jon hamm on his ...</td><td>{saturday’s houst...</td></tr>\n",
       "<tr><td>0.4285714285714286</td><td>6123</td><td>12935</td><td>{week 12 connecti...</td><td>{houston high sch...</td></tr>\n",
       "<tr><td>0.4</td><td>11672</td><td>12843</td><td>{high school foot...</td><td>{1st &amp;amp; 10: we...</td></tr>\n",
       "<tr><td>0.4285714285714286</td><td>3258</td><td>13184</td><td>{take a look at h...</td><td>{take a look at m...</td></tr>\n",
       "<tr><td>0.0</td><td>6595</td><td>7657</td><td>{the best destina...</td><td>{the best destina...</td></tr>\n",
       "<tr><td>0.4</td><td>4746</td><td>10038</td><td>{high school scor...</td><td>{week 11 high sch...</td></tr>\n",
       "<tr><td>0.0</td><td>10839</td><td>12486</td><td>{acting uvalde po...</td><td>{acting uvalde po...</td></tr>\n",
       "<tr><td>0.0</td><td>9545</td><td>12347</td><td>{poll: which high...</td><td>{poll: which high...</td></tr>\n",
       "<tr><td>0.33333333333333337</td><td>7962</td><td>8839</td><td>{high school http...</td><td>{and that’s a sun...</td></tr>\n",
       "<tr><td>0.33333333333333337</td><td>7962</td><td>8618</td><td>{high school http...</td><td>{nj high school m...</td></tr>\n",
       "<tr><td>0.33333333333333337</td><td>7962</td><td>10211</td><td>{high school http...</td><td>{high school foot...</td></tr>\n",
       "<tr><td>0.33333333333333337</td><td>7962</td><td>8917</td><td>{high school http...</td><td>{school closure: ...</td></tr>\n",
       "<tr><td>0.33333333333333337</td><td>2051</td><td>5324</td><td>{here are this we...</td><td>{high school http...</td></tr>\n",
       "<tr><td>0.4</td><td>8994</td><td>12843</td><td>{high school foot...</td><td>{1st &amp;amp; 10: we...</td></tr>\n",
       "<tr><td>0.4285714285714286</td><td>9292</td><td>12843</td><td>{vote for the hig...</td><td>{1st &amp;amp; 10: we...</td></tr>\n",
       "<tr><td>0.4444444444444444</td><td>2610</td><td>7435</td><td>{vote for the hig...</td><td>{how area- and st...</td></tr>\n",
       "<tr><td>0.25</td><td>2610</td><td>9292</td><td>{vote for the hig...</td><td>{vote for the hig...</td></tr>\n",
       "</table>\n",
       "only showing top 20 rows\n"
      ],
      "text/plain": [
       "+-------------------+-----+-----+--------------------+--------------------+\n",
       "|            distCol| id_A| id_B|        tweet_text_A|        tweet_text_B|\n",
       "+-------------------+-----+-----+--------------------+--------------------+\n",
       "| 0.4545454545454546| 8209|12487|{acting uvalde po...|{the acting uvald...|\n",
       "|                0.4| 3120| 4869|{jon hamm on his ...|{week 5 high scho...|\n",
       "| 0.4545454545454546| 7595|10414|{preseason all-ba...|{bay area news gr...|\n",
       "|                0.4| 3120| 6907|{jon hamm on his ...|{saturday’s houst...|\n",
       "| 0.4285714285714286| 6123|12935|{week 12 connecti...|{houston high sch...|\n",
       "|                0.4|11672|12843|{high school foot...|{1st &amp; 10: we...|\n",
       "| 0.4285714285714286| 3258|13184|{take a look at h...|{take a look at m...|\n",
       "|                0.0| 6595| 7657|{the best destina...|{the best destina...|\n",
       "|                0.4| 4746|10038|{high school scor...|{week 11 high sch...|\n",
       "|                0.0|10839|12486|{acting uvalde po...|{acting uvalde po...|\n",
       "|                0.0| 9545|12347|{poll: which high...|{poll: which high...|\n",
       "|0.33333333333333337| 7962| 8839|{high school http...|{and that’s a sun...|\n",
       "|0.33333333333333337| 7962| 8618|{high school http...|{nj high school m...|\n",
       "|0.33333333333333337| 7962|10211|{high school http...|{high school foot...|\n",
       "|0.33333333333333337| 7962| 8917|{high school http...|{school closure: ...|\n",
       "|0.33333333333333337| 2051| 5324|{here are this we...|{high school http...|\n",
       "|                0.4| 8994|12843|{high school foot...|{1st &amp; 10: we...|\n",
       "| 0.4285714285714286| 9292|12843|{vote for the hig...|{1st &amp; 10: we...|\n",
       "| 0.4444444444444444| 2610| 7435|{vote for the hig...|{how area- and st...|\n",
       "|               0.25| 2610| 9292|{vote for the hig...|{vote for the hig...|\n",
       "+-------------------+-----+-----+--------------------+--------------------+\n",
       "only showing top 20 rows"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dups_50_tweet_text.unpersist()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fb180d5-002d-4bcf-85cb-5920b168b055",
   "metadata": {},
   "source": [
    "### 0.7 Distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "153cd110-204a-4ef8-bdf8-58a1d61b7525",
   "metadata": {},
   "outputs": [],
   "source": [
    "jaccard_distance = 0.7\n",
    "\n",
    "df_dups_tweet_text = model.approxSimilarityJoin(df_hashed_tweet_text_samp, df_hashed_tweet_text_samp, jaccard_distance).filter(\"datasetA.id < datasetB.id\").select(\n",
    "            col(\"distCol\"),\n",
    "            col(\"datasetA.id\").alias(\"id_A\"),\n",
    "            col(\"datasetB.id\").alias(\"id_B\"),\n",
    "            col('datasetA.tweet_text').alias('tweet_text_A'),\n",
    "            col('datasetB.tweet_text').alias('tweet_text_B'),\n",
    "#             col('datasetA.list_of_words').alias('words_A'),\n",
    "#             col('datasetB.list_of_words').alias('words_B')\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "e54c8c49-acce-428e-be0f-6343bfa9e9c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>distCol</th>\n",
       "      <th>id_A</th>\n",
       "      <th>id_B</th>\n",
       "      <th>tweet_text_A</th>\n",
       "      <th>tweet_text_B</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>965</td>\n",
       "      <td>8839</td>\n",
       "      <td>(just one more day of texas high school football…😕\\n\\n#txhsfb #uilstate,)</td>\n",
       "      <td>(and that’s a sunset of high school football. https://t.co/kltovfgapo,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.666667</td>\n",
       "      <td>7540</td>\n",
       "      <td>13155</td>\n",
       "      <td>(middle school survey asked if students identify as 'transgender' or 'non-binary'\\n\\nhttps://t.co/wlhrut68ub,)</td>\n",
       "      <td>(#iranrevolution\\ntehran, #iran's capital—#iranprotests against #mahsaamini's murder, day 19.\\nhigh school students protest, chanting:\\n\"no fear! we're all together!\"https://t.co/fkrcrnpxjk,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.636364</td>\n",
       "      <td>6136</td>\n",
       "      <td>7435</td>\n",
       "      <td>(icymi: here's what we learned from week 1 of the high school football season in the south sound 🔒\\n\\nhttps://t.co/zkmfryil5f,)</td>\n",
       "      <td>(how area- and state-ranked teams fared in texas high school football this week 🏈\\n\\n#txhsfb https://t.co/u3lglvy1uk,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.666667</td>\n",
       "      <td>6136</td>\n",
       "      <td>7059</td>\n",
       "      <td>(icymi: here's what we learned from week 1 of the high school football season in the south sound 🔒\\n\\nhttps://t.co/zkmfryil5f,)</td>\n",
       "      <td>(bay area news group high school football predictions: week 8, 2022 https://t.co/27o5blgpbo,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.583333</td>\n",
       "      <td>6184</td>\n",
       "      <td>7110</td>\n",
       "      <td>(great first friday night of stl high school football coverage on @ksdknews #5onyoursidelines with @ahmadhickstv and @corey_miller5 #welldone #fridaynightlights 🏈#mustseetv https://t.co/deeawn9unt,)</td>\n",
       "      <td>(high school football live updates: cif socal regional finals on friday night https://t.co/m3fv2kjpdi,)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    distCol  id_A   id_B  \\\n",
       "0  0.500000   965   8839   \n",
       "1  0.666667  7540  13155   \n",
       "2  0.636364  6136   7435   \n",
       "3  0.666667  6136   7059   \n",
       "4  0.583333  6184   7110   \n",
       "\n",
       "                                                                                                                                                                                             tweet_text_A  \\\n",
       "0                                                                                                                               (just one more day of texas high school football…😕\\n\\n#txhsfb #uilstate,)   \n",
       "1                                                                                          (middle school survey asked if students identify as 'transgender' or 'non-binary'\\n\\nhttps://t.co/wlhrut68ub,)   \n",
       "2                                                                         (icymi: here's what we learned from week 1 of the high school football season in the south sound 🔒\\n\\nhttps://t.co/zkmfryil5f,)   \n",
       "3                                                                         (icymi: here's what we learned from week 1 of the high school football season in the south sound 🔒\\n\\nhttps://t.co/zkmfryil5f,)   \n",
       "4  (great first friday night of stl high school football coverage on @ksdknews #5onyoursidelines with @ahmadhickstv and @corey_miller5 #welldone #fridaynightlights 🏈#mustseetv https://t.co/deeawn9unt,)   \n",
       "\n",
       "                                                                                                                                                                                      tweet_text_B  \n",
       "0                                                                                                                          (and that’s a sunset of high school football. https://t.co/kltovfgapo,)  \n",
       "1  (#iranrevolution\\ntehran, #iran's capital—#iranprotests against #mahsaamini's murder, day 19.\\nhigh school students protest, chanting:\\n\"no fear! we're all together!\"https://t.co/fkrcrnpxjk,)  \n",
       "2                                                                           (how area- and state-ranked teams fared in texas high school football this week 🏈\\n\\n#txhsfb https://t.co/u3lglvy1uk,)  \n",
       "3                                                                                                    (bay area news group high school football predictions: week 8, 2022 https://t.co/27o5blgpbo,)  \n",
       "4                                                                                          (high school football live updates: cif socal regional finals on friday night https://t.co/m3fv2kjpdi,)  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dups_70_tweet_text = df_dups_tweet_text\n",
    "df_dups_70_tweet_text.cache()\n",
    "df_dups_70_tweet_text.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "e180ea44-cacb-40b5-81e1-49b560913d9c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 239:==================================================>  (191 + 4) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total records:  13594\n",
      "Duplicate titles based on { 0.7 } jaccard distance:  1787\n",
      "Unique titles based on { 0.7 } jaccard distance:  :  11807\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "records = df_hashed_tweet_text_samp.count()\n",
    "dups_70 = df_dups_70_tweet_text.select('id_A').distinct().count()\n",
    "uniques_70 = records - dups_70\n",
    "\n",
    "print ('Total records: ', records)\n",
    "print ('Duplicate titles based on {', jaccard_distance, '} jaccard distance: ', dups_70)\n",
    "print ('Unique titles based on {', jaccard_distance, '} jaccard distance: ', ': ', uniques_70)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "242b4a6e-d230-4dc7-b2b9-8d8ac49c9d6a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       "  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<svg height=\"324.950281pt\" version=\"1.1\" viewBox=\"0 0 411.415078 324.950281\" width=\"411.415078pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       " <metadata>\n",
       "  <rdf:RDF xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n",
       "   <cc:Work>\n",
       "    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\n",
       "    <dc:date>2023-03-09T01:02:10.404872</dc:date>\n",
       "    <dc:format>image/svg+xml</dc:format>\n",
       "    <dc:creator>\n",
       "     <cc:Agent>\n",
       "      <dc:title>Matplotlib v3.4.3, https://matplotlib.org/</dc:title>\n",
       "     </cc:Agent>\n",
       "    </dc:creator>\n",
       "   </cc:Work>\n",
       "  </rdf:RDF>\n",
       " </metadata>\n",
       " <defs>\n",
       "  <style type=\"text/css\">*{stroke-linecap:butt;stroke-linejoin:round;}</style>\n",
       " </defs>\n",
       " <g id=\"figure_1\">\n",
       "  <g id=\"patch_1\">\n",
       "   <path d=\"M 0 324.950281 \n",
       "L 411.415078 324.950281 \n",
       "L 411.415078 0 \n",
       "L 0 0 \n",
       "z\n",
       "\" style=\"fill:#ffffff;\"/>\n",
       "  </g>\n",
       "  <g id=\"axes_1\">\n",
       "   <g id=\"patch_2\">\n",
       "    <path d=\"M 46.0125 290.709656 \n",
       "L 403.1325 290.709656 \n",
       "L 403.1325 24.597656 \n",
       "L 46.0125 24.597656 \n",
       "z\n",
       "\" style=\"fill:#ffffff;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_3\">\n",
       "    <path clip-path=\"url(#pe2a3fdeeb7)\" d=\"M 114.689423 290.709656 \n",
       "L 224.5725 290.709656 \n",
       "L 224.5725 252.351286 \n",
       "L 114.689423 252.351286 \n",
       "z\n",
       "\" style=\"fill:#d62728;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_4\">\n",
       "    <path clip-path=\"url(#pe2a3fdeeb7)\" d=\"M 224.5725 290.709656 \n",
       "L 334.455577 290.709656 \n",
       "L 334.455577 37.269656 \n",
       "L 224.5725 37.269656 \n",
       "z\n",
       "\" style=\"fill:#2ca02c;\"/>\n",
       "   </g>\n",
       "   <g id=\"matplotlib.axis_1\">\n",
       "    <g id=\"xtick_1\">\n",
       "     <g id=\"line2d_1\">\n",
       "      <defs>\n",
       "       <path d=\"M 0 0 \n",
       "L 0 3.5 \n",
       "\" id=\"m48f264a248\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"224.5725\" xlink:href=\"#m48f264a248\" y=\"290.709656\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_1\">\n",
       "      <!-- 0 -->\n",
       "      <g transform=\"translate(227.331875 304.072156)rotate(-90)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 2034 4250 \n",
       "Q 1547 4250 1301 3770 \n",
       "Q 1056 3291 1056 2328 \n",
       "Q 1056 1369 1301 889 \n",
       "Q 1547 409 2034 409 \n",
       "Q 2525 409 2770 889 \n",
       "Q 3016 1369 3016 2328 \n",
       "Q 3016 3291 2770 3770 \n",
       "Q 2525 4250 2034 4250 \n",
       "z\n",
       "M 2034 4750 \n",
       "Q 2819 4750 3233 4129 \n",
       "Q 3647 3509 3647 2328 \n",
       "Q 3647 1150 3233 529 \n",
       "Q 2819 -91 2034 -91 \n",
       "Q 1250 -91 836 529 \n",
       "Q 422 1150 422 2328 \n",
       "Q 422 3509 836 4129 \n",
       "Q 1250 4750 2034 4750 \n",
       "z\n",
       "\" id=\"DejaVuSans-30\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"text_2\">\n",
       "     <!-- Duplicates vs. Unique -->\n",
       "     <g transform=\"translate(170.17875 315.670594)scale(0.1 -0.1)\">\n",
       "      <defs>\n",
       "       <path d=\"M 1259 4147 \n",
       "L 1259 519 \n",
       "L 2022 519 \n",
       "Q 2988 519 3436 956 \n",
       "Q 3884 1394 3884 2338 \n",
       "Q 3884 3275 3436 3711 \n",
       "Q 2988 4147 2022 4147 \n",
       "L 1259 4147 \n",
       "z\n",
       "M 628 4666 \n",
       "L 1925 4666 \n",
       "Q 3281 4666 3915 4102 \n",
       "Q 4550 3538 4550 2338 \n",
       "Q 4550 1131 3912 565 \n",
       "Q 3275 0 1925 0 \n",
       "L 628 0 \n",
       "L 628 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-44\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 544 1381 \n",
       "L 544 3500 \n",
       "L 1119 3500 \n",
       "L 1119 1403 \n",
       "Q 1119 906 1312 657 \n",
       "Q 1506 409 1894 409 \n",
       "Q 2359 409 2629 706 \n",
       "Q 2900 1003 2900 1516 \n",
       "L 2900 3500 \n",
       "L 3475 3500 \n",
       "L 3475 0 \n",
       "L 2900 0 \n",
       "L 2900 538 \n",
       "Q 2691 219 2414 64 \n",
       "Q 2138 -91 1772 -91 \n",
       "Q 1169 -91 856 284 \n",
       "Q 544 659 544 1381 \n",
       "z\n",
       "M 1991 3584 \n",
       "L 1991 3584 \n",
       "z\n",
       "\" id=\"DejaVuSans-75\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 1159 525 \n",
       "L 1159 -1331 \n",
       "L 581 -1331 \n",
       "L 581 3500 \n",
       "L 1159 3500 \n",
       "L 1159 2969 \n",
       "Q 1341 3281 1617 3432 \n",
       "Q 1894 3584 2278 3584 \n",
       "Q 2916 3584 3314 3078 \n",
       "Q 3713 2572 3713 1747 \n",
       "Q 3713 922 3314 415 \n",
       "Q 2916 -91 2278 -91 \n",
       "Q 1894 -91 1617 61 \n",
       "Q 1341 213 1159 525 \n",
       "z\n",
       "M 3116 1747 \n",
       "Q 3116 2381 2855 2742 \n",
       "Q 2594 3103 2138 3103 \n",
       "Q 1681 3103 1420 2742 \n",
       "Q 1159 2381 1159 1747 \n",
       "Q 1159 1113 1420 752 \n",
       "Q 1681 391 2138 391 \n",
       "Q 2594 391 2855 752 \n",
       "Q 3116 1113 3116 1747 \n",
       "z\n",
       "\" id=\"DejaVuSans-70\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 603 4863 \n",
       "L 1178 4863 \n",
       "L 1178 0 \n",
       "L 603 0 \n",
       "L 603 4863 \n",
       "z\n",
       "\" id=\"DejaVuSans-6c\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 603 3500 \n",
       "L 1178 3500 \n",
       "L 1178 0 \n",
       "L 603 0 \n",
       "L 603 3500 \n",
       "z\n",
       "M 603 4863 \n",
       "L 1178 4863 \n",
       "L 1178 4134 \n",
       "L 603 4134 \n",
       "L 603 4863 \n",
       "z\n",
       "\" id=\"DejaVuSans-69\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 3122 3366 \n",
       "L 3122 2828 \n",
       "Q 2878 2963 2633 3030 \n",
       "Q 2388 3097 2138 3097 \n",
       "Q 1578 3097 1268 2742 \n",
       "Q 959 2388 959 1747 \n",
       "Q 959 1106 1268 751 \n",
       "Q 1578 397 2138 397 \n",
       "Q 2388 397 2633 464 \n",
       "Q 2878 531 3122 666 \n",
       "L 3122 134 \n",
       "Q 2881 22 2623 -34 \n",
       "Q 2366 -91 2075 -91 \n",
       "Q 1284 -91 818 406 \n",
       "Q 353 903 353 1747 \n",
       "Q 353 2603 823 3093 \n",
       "Q 1294 3584 2113 3584 \n",
       "Q 2378 3584 2631 3529 \n",
       "Q 2884 3475 3122 3366 \n",
       "z\n",
       "\" id=\"DejaVuSans-63\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2194 1759 \n",
       "Q 1497 1759 1228 1600 \n",
       "Q 959 1441 959 1056 \n",
       "Q 959 750 1161 570 \n",
       "Q 1363 391 1709 391 \n",
       "Q 2188 391 2477 730 \n",
       "Q 2766 1069 2766 1631 \n",
       "L 2766 1759 \n",
       "L 2194 1759 \n",
       "z\n",
       "M 3341 1997 \n",
       "L 3341 0 \n",
       "L 2766 0 \n",
       "L 2766 531 \n",
       "Q 2569 213 2275 61 \n",
       "Q 1981 -91 1556 -91 \n",
       "Q 1019 -91 701 211 \n",
       "Q 384 513 384 1019 \n",
       "Q 384 1609 779 1909 \n",
       "Q 1175 2209 1959 2209 \n",
       "L 2766 2209 \n",
       "L 2766 2266 \n",
       "Q 2766 2663 2505 2880 \n",
       "Q 2244 3097 1772 3097 \n",
       "Q 1472 3097 1187 3025 \n",
       "Q 903 2953 641 2809 \n",
       "L 641 3341 \n",
       "Q 956 3463 1253 3523 \n",
       "Q 1550 3584 1831 3584 \n",
       "Q 2591 3584 2966 3190 \n",
       "Q 3341 2797 3341 1997 \n",
       "z\n",
       "\" id=\"DejaVuSans-61\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 1172 4494 \n",
       "L 1172 3500 \n",
       "L 2356 3500 \n",
       "L 2356 3053 \n",
       "L 1172 3053 \n",
       "L 1172 1153 \n",
       "Q 1172 725 1289 603 \n",
       "Q 1406 481 1766 481 \n",
       "L 2356 481 \n",
       "L 2356 0 \n",
       "L 1766 0 \n",
       "Q 1100 0 847 248 \n",
       "Q 594 497 594 1153 \n",
       "L 594 3053 \n",
       "L 172 3053 \n",
       "L 172 3500 \n",
       "L 594 3500 \n",
       "L 594 4494 \n",
       "L 1172 4494 \n",
       "z\n",
       "\" id=\"DejaVuSans-74\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 3597 1894 \n",
       "L 3597 1613 \n",
       "L 953 1613 \n",
       "Q 991 1019 1311 708 \n",
       "Q 1631 397 2203 397 \n",
       "Q 2534 397 2845 478 \n",
       "Q 3156 559 3463 722 \n",
       "L 3463 178 \n",
       "Q 3153 47 2828 -22 \n",
       "Q 2503 -91 2169 -91 \n",
       "Q 1331 -91 842 396 \n",
       "Q 353 884 353 1716 \n",
       "Q 353 2575 817 3079 \n",
       "Q 1281 3584 2069 3584 \n",
       "Q 2775 3584 3186 3129 \n",
       "Q 3597 2675 3597 1894 \n",
       "z\n",
       "M 3022 2063 \n",
       "Q 3016 2534 2758 2815 \n",
       "Q 2500 3097 2075 3097 \n",
       "Q 1594 3097 1305 2825 \n",
       "Q 1016 2553 972 2059 \n",
       "L 3022 2063 \n",
       "z\n",
       "\" id=\"DejaVuSans-65\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 2834 3397 \n",
       "L 2834 2853 \n",
       "Q 2591 2978 2328 3040 \n",
       "Q 2066 3103 1784 3103 \n",
       "Q 1356 3103 1142 2972 \n",
       "Q 928 2841 928 2578 \n",
       "Q 928 2378 1081 2264 \n",
       "Q 1234 2150 1697 2047 \n",
       "L 1894 2003 \n",
       "Q 2506 1872 2764 1633 \n",
       "Q 3022 1394 3022 966 \n",
       "Q 3022 478 2636 193 \n",
       "Q 2250 -91 1575 -91 \n",
       "Q 1294 -91 989 -36 \n",
       "Q 684 19 347 128 \n",
       "L 347 722 \n",
       "Q 666 556 975 473 \n",
       "Q 1284 391 1588 391 \n",
       "Q 1994 391 2212 530 \n",
       "Q 2431 669 2431 922 \n",
       "Q 2431 1156 2273 1281 \n",
       "Q 2116 1406 1581 1522 \n",
       "L 1381 1569 \n",
       "Q 847 1681 609 1914 \n",
       "Q 372 2147 372 2553 \n",
       "Q 372 3047 722 3315 \n",
       "Q 1072 3584 1716 3584 \n",
       "Q 2034 3584 2315 3537 \n",
       "Q 2597 3491 2834 3397 \n",
       "z\n",
       "\" id=\"DejaVuSans-73\" transform=\"scale(0.015625)\"/>\n",
       "       <path id=\"DejaVuSans-20\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 191 3500 \n",
       "L 800 3500 \n",
       "L 1894 563 \n",
       "L 2988 3500 \n",
       "L 3597 3500 \n",
       "L 2284 0 \n",
       "L 1503 0 \n",
       "L 191 3500 \n",
       "z\n",
       "\" id=\"DejaVuSans-76\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 684 794 \n",
       "L 1344 794 \n",
       "L 1344 0 \n",
       "L 684 0 \n",
       "L 684 794 \n",
       "z\n",
       "\" id=\"DejaVuSans-2e\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 556 4666 \n",
       "L 1191 4666 \n",
       "L 1191 1831 \n",
       "Q 1191 1081 1462 751 \n",
       "Q 1734 422 2344 422 \n",
       "Q 2950 422 3222 751 \n",
       "Q 3494 1081 3494 1831 \n",
       "L 3494 4666 \n",
       "L 4128 4666 \n",
       "L 4128 1753 \n",
       "Q 4128 841 3676 375 \n",
       "Q 3225 -91 2344 -91 \n",
       "Q 1459 -91 1007 375 \n",
       "Q 556 841 556 1753 \n",
       "L 556 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-55\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 3513 2113 \n",
       "L 3513 0 \n",
       "L 2938 0 \n",
       "L 2938 2094 \n",
       "Q 2938 2591 2744 2837 \n",
       "Q 2550 3084 2163 3084 \n",
       "Q 1697 3084 1428 2787 \n",
       "Q 1159 2491 1159 1978 \n",
       "L 1159 0 \n",
       "L 581 0 \n",
       "L 581 3500 \n",
       "L 1159 3500 \n",
       "L 1159 2956 \n",
       "Q 1366 3272 1645 3428 \n",
       "Q 1925 3584 2291 3584 \n",
       "Q 2894 3584 3203 3211 \n",
       "Q 3513 2838 3513 2113 \n",
       "z\n",
       "\" id=\"DejaVuSans-6e\" transform=\"scale(0.015625)\"/>\n",
       "       <path d=\"M 947 1747 \n",
       "Q 947 1113 1208 752 \n",
       "Q 1469 391 1925 391 \n",
       "Q 2381 391 2643 752 \n",
       "Q 2906 1113 2906 1747 \n",
       "Q 2906 2381 2643 2742 \n",
       "Q 2381 3103 1925 3103 \n",
       "Q 1469 3103 1208 2742 \n",
       "Q 947 2381 947 1747 \n",
       "z\n",
       "M 2906 525 \n",
       "Q 2725 213 2448 61 \n",
       "Q 2172 -91 1784 -91 \n",
       "Q 1150 -91 751 415 \n",
       "Q 353 922 353 1747 \n",
       "Q 353 2572 751 3078 \n",
       "Q 1150 3584 1784 3584 \n",
       "Q 2172 3584 2448 3432 \n",
       "Q 2725 3281 2906 2969 \n",
       "L 2906 3500 \n",
       "L 3481 3500 \n",
       "L 3481 -1331 \n",
       "L 2906 -1331 \n",
       "L 2906 525 \n",
       "z\n",
       "\" id=\"DejaVuSans-71\" transform=\"scale(0.015625)\"/>\n",
       "      </defs>\n",
       "      <use xlink:href=\"#DejaVuSans-44\"/>\n",
       "      <use x=\"77.001953\" xlink:href=\"#DejaVuSans-75\"/>\n",
       "      <use x=\"140.380859\" xlink:href=\"#DejaVuSans-70\"/>\n",
       "      <use x=\"203.857422\" xlink:href=\"#DejaVuSans-6c\"/>\n",
       "      <use x=\"231.640625\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "      <use x=\"259.423828\" xlink:href=\"#DejaVuSans-63\"/>\n",
       "      <use x=\"314.404297\" xlink:href=\"#DejaVuSans-61\"/>\n",
       "      <use x=\"375.683594\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "      <use x=\"414.892578\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "      <use x=\"476.416016\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "      <use x=\"528.515625\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"560.302734\" xlink:href=\"#DejaVuSans-76\"/>\n",
       "      <use x=\"619.482422\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "      <use x=\"671.582031\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "      <use x=\"703.369141\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"735.15625\" xlink:href=\"#DejaVuSans-55\"/>\n",
       "      <use x=\"808.349609\" xlink:href=\"#DejaVuSans-6e\"/>\n",
       "      <use x=\"871.728516\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "      <use x=\"899.511719\" xlink:href=\"#DejaVuSans-71\"/>\n",
       "      <use x=\"962.988281\" xlink:href=\"#DejaVuSans-75\"/>\n",
       "      <use x=\"1026.367188\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"matplotlib.axis_2\">\n",
       "    <g id=\"ytick_1\">\n",
       "     <g id=\"line2d_2\">\n",
       "      <defs>\n",
       "       <path d=\"M 0 0 \n",
       "L -3.5 0 \n",
       "\" id=\"mdf57d45725\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n",
       "      </defs>\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#mdf57d45725\" y=\"290.709656\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_3\">\n",
       "      <!-- 0 -->\n",
       "      <g transform=\"translate(32.65 294.508875)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_2\">\n",
       "     <g id=\"line2d_3\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#mdf57d45725\" y=\"247.779191\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_4\">\n",
       "      <!-- 2000 -->\n",
       "      <g transform=\"translate(13.5625 251.57841)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 1228 531 \n",
       "L 3431 531 \n",
       "L 3431 0 \n",
       "L 469 0 \n",
       "L 469 531 \n",
       "Q 828 903 1448 1529 \n",
       "Q 2069 2156 2228 2338 \n",
       "Q 2531 2678 2651 2914 \n",
       "Q 2772 3150 2772 3378 \n",
       "Q 2772 3750 2511 3984 \n",
       "Q 2250 4219 1831 4219 \n",
       "Q 1534 4219 1204 4116 \n",
       "Q 875 4013 500 3803 \n",
       "L 500 4441 \n",
       "Q 881 4594 1212 4672 \n",
       "Q 1544 4750 1819 4750 \n",
       "Q 2544 4750 2975 4387 \n",
       "Q 3406 4025 3406 3419 \n",
       "Q 3406 3131 3298 2873 \n",
       "Q 3191 2616 2906 2266 \n",
       "Q 2828 2175 2409 1742 \n",
       "Q 1991 1309 1228 531 \n",
       "z\n",
       "\" id=\"DejaVuSans-32\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-32\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_3\">\n",
       "     <g id=\"line2d_4\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#mdf57d45725\" y=\"204.848726\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_5\">\n",
       "      <!-- 4000 -->\n",
       "      <g transform=\"translate(13.5625 208.647945)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 2419 4116 \n",
       "L 825 1625 \n",
       "L 2419 1625 \n",
       "L 2419 4116 \n",
       "z\n",
       "M 2253 4666 \n",
       "L 3047 4666 \n",
       "L 3047 1625 \n",
       "L 3713 1625 \n",
       "L 3713 1100 \n",
       "L 3047 1100 \n",
       "L 3047 0 \n",
       "L 2419 0 \n",
       "L 2419 1100 \n",
       "L 313 1100 \n",
       "L 313 1709 \n",
       "L 2253 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-34\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-34\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_4\">\n",
       "     <g id=\"line2d_5\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#mdf57d45725\" y=\"161.918261\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_6\">\n",
       "      <!-- 6000 -->\n",
       "      <g transform=\"translate(13.5625 165.71748)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 2113 2584 \n",
       "Q 1688 2584 1439 2293 \n",
       "Q 1191 2003 1191 1497 \n",
       "Q 1191 994 1439 701 \n",
       "Q 1688 409 2113 409 \n",
       "Q 2538 409 2786 701 \n",
       "Q 3034 994 3034 1497 \n",
       "Q 3034 2003 2786 2293 \n",
       "Q 2538 2584 2113 2584 \n",
       "z\n",
       "M 3366 4563 \n",
       "L 3366 3988 \n",
       "Q 3128 4100 2886 4159 \n",
       "Q 2644 4219 2406 4219 \n",
       "Q 1781 4219 1451 3797 \n",
       "Q 1122 3375 1075 2522 \n",
       "Q 1259 2794 1537 2939 \n",
       "Q 1816 3084 2150 3084 \n",
       "Q 2853 3084 3261 2657 \n",
       "Q 3669 2231 3669 1497 \n",
       "Q 3669 778 3244 343 \n",
       "Q 2819 -91 2113 -91 \n",
       "Q 1303 -91 875 529 \n",
       "Q 447 1150 447 2328 \n",
       "Q 447 3434 972 4092 \n",
       "Q 1497 4750 2381 4750 \n",
       "Q 2619 4750 2861 4703 \n",
       "Q 3103 4656 3366 4563 \n",
       "z\n",
       "\" id=\"DejaVuSans-36\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-36\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_5\">\n",
       "     <g id=\"line2d_6\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#mdf57d45725\" y=\"118.987796\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_7\">\n",
       "      <!-- 8000 -->\n",
       "      <g transform=\"translate(13.5625 122.787015)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 2034 2216 \n",
       "Q 1584 2216 1326 1975 \n",
       "Q 1069 1734 1069 1313 \n",
       "Q 1069 891 1326 650 \n",
       "Q 1584 409 2034 409 \n",
       "Q 2484 409 2743 651 \n",
       "Q 3003 894 3003 1313 \n",
       "Q 3003 1734 2745 1975 \n",
       "Q 2488 2216 2034 2216 \n",
       "z\n",
       "M 1403 2484 \n",
       "Q 997 2584 770 2862 \n",
       "Q 544 3141 544 3541 \n",
       "Q 544 4100 942 4425 \n",
       "Q 1341 4750 2034 4750 \n",
       "Q 2731 4750 3128 4425 \n",
       "Q 3525 4100 3525 3541 \n",
       "Q 3525 3141 3298 2862 \n",
       "Q 3072 2584 2669 2484 \n",
       "Q 3125 2378 3379 2068 \n",
       "Q 3634 1759 3634 1313 \n",
       "Q 3634 634 3220 271 \n",
       "Q 2806 -91 2034 -91 \n",
       "Q 1263 -91 848 271 \n",
       "Q 434 634 434 1313 \n",
       "Q 434 1759 690 2068 \n",
       "Q 947 2378 1403 2484 \n",
       "z\n",
       "M 1172 3481 \n",
       "Q 1172 3119 1398 2916 \n",
       "Q 1625 2713 2034 2713 \n",
       "Q 2441 2713 2670 2916 \n",
       "Q 2900 3119 2900 3481 \n",
       "Q 2900 3844 2670 4047 \n",
       "Q 2441 4250 2034 4250 \n",
       "Q 1625 4250 1398 4047 \n",
       "Q 1172 3844 1172 3481 \n",
       "z\n",
       "\" id=\"DejaVuSans-38\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-38\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_6\">\n",
       "     <g id=\"line2d_7\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#mdf57d45725\" y=\"76.057331\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_8\">\n",
       "      <!-- 10000 -->\n",
       "      <g transform=\"translate(7.2 79.85655)scale(0.1 -0.1)\">\n",
       "       <defs>\n",
       "        <path d=\"M 794 531 \n",
       "L 1825 531 \n",
       "L 1825 4091 \n",
       "L 703 3866 \n",
       "L 703 4441 \n",
       "L 1819 4666 \n",
       "L 2450 4666 \n",
       "L 2450 531 \n",
       "L 3481 531 \n",
       "L 3481 0 \n",
       "L 794 0 \n",
       "L 794 531 \n",
       "z\n",
       "\" id=\"DejaVuSans-31\" transform=\"scale(0.015625)\"/>\n",
       "       </defs>\n",
       "       <use xlink:href=\"#DejaVuSans-31\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"254.492188\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"ytick_7\">\n",
       "     <g id=\"line2d_8\">\n",
       "      <g>\n",
       "       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#mdf57d45725\" y=\"33.126866\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "     <g id=\"text_9\">\n",
       "      <!-- 12000 -->\n",
       "      <g transform=\"translate(7.2 36.926085)scale(0.1 -0.1)\">\n",
       "       <use xlink:href=\"#DejaVuSans-31\"/>\n",
       "       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-32\"/>\n",
       "       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "       <use x=\"254.492188\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "      </g>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"patch_5\">\n",
       "    <path d=\"M 46.0125 290.709656 \n",
       "L 46.0125 24.597656 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_6\">\n",
       "    <path d=\"M 403.1325 290.709656 \n",
       "L 403.1325 24.597656 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_7\">\n",
       "    <path d=\"M 46.0125 290.709656 \n",
       "L 403.1325 290.709656 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"patch_8\">\n",
       "    <path d=\"M 46.0125 24.597656 \n",
       "L 403.1325 24.597656 \n",
       "\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n",
       "   </g>\n",
       "   <g id=\"text_10\">\n",
       "    <!-- 1787.0 -->\n",
       "    <g transform=\"translate(152.135649 265.289846)scale(0.1 -0.1)\">\n",
       "     <defs>\n",
       "      <path d=\"M 525 4666 \n",
       "L 3525 4666 \n",
       "L 3525 4397 \n",
       "L 1831 0 \n",
       "L 1172 0 \n",
       "L 2766 4134 \n",
       "L 525 4134 \n",
       "L 525 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-37\" transform=\"scale(0.015625)\"/>\n",
       "     </defs>\n",
       "     <use xlink:href=\"#DejaVuSans-31\"/>\n",
       "     <use x=\"63.623047\" xlink:href=\"#DejaVuSans-37\"/>\n",
       "     <use x=\"127.246094\" xlink:href=\"#DejaVuSans-38\"/>\n",
       "     <use x=\"190.869141\" xlink:href=\"#DejaVuSans-37\"/>\n",
       "     <use x=\"254.492188\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "     <use x=\"286.279297\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"text_11\">\n",
       "    <!-- 11807.0 -->\n",
       "    <g transform=\"translate(258.837476 157.749031)scale(0.1 -0.1)\">\n",
       "     <use xlink:href=\"#DejaVuSans-31\"/>\n",
       "     <use x=\"63.623047\" xlink:href=\"#DejaVuSans-31\"/>\n",
       "     <use x=\"127.246094\" xlink:href=\"#DejaVuSans-38\"/>\n",
       "     <use x=\"190.869141\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "     <use x=\"254.492188\" xlink:href=\"#DejaVuSans-37\"/>\n",
       "     <use x=\"318.115234\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "     <use x=\"349.902344\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"text_12\">\n",
       "    <!-- Twitter Text Duplication Analysis - 0.7 Threshold -->\n",
       "    <g transform=\"translate(44.929922 18.597656)scale(0.15 -0.15)\">\n",
       "     <defs>\n",
       "      <path d=\"M -19 4666 \n",
       "L 3928 4666 \n",
       "L 3928 4134 \n",
       "L 2272 4134 \n",
       "L 2272 0 \n",
       "L 1638 0 \n",
       "L 1638 4134 \n",
       "L -19 4134 \n",
       "L -19 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-54\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 269 3500 \n",
       "L 844 3500 \n",
       "L 1563 769 \n",
       "L 2278 3500 \n",
       "L 2956 3500 \n",
       "L 3675 769 \n",
       "L 4391 3500 \n",
       "L 4966 3500 \n",
       "L 4050 0 \n",
       "L 3372 0 \n",
       "L 2619 2869 \n",
       "L 1863 0 \n",
       "L 1184 0 \n",
       "L 269 3500 \n",
       "z\n",
       "\" id=\"DejaVuSans-77\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 2631 2963 \n",
       "Q 2534 3019 2420 3045 \n",
       "Q 2306 3072 2169 3072 \n",
       "Q 1681 3072 1420 2755 \n",
       "Q 1159 2438 1159 1844 \n",
       "L 1159 0 \n",
       "L 581 0 \n",
       "L 581 3500 \n",
       "L 1159 3500 \n",
       "L 1159 2956 \n",
       "Q 1341 3275 1631 3429 \n",
       "Q 1922 3584 2338 3584 \n",
       "Q 2397 3584 2469 3576 \n",
       "Q 2541 3569 2628 3553 \n",
       "L 2631 2963 \n",
       "z\n",
       "\" id=\"DejaVuSans-72\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 3513 3500 \n",
       "L 2247 1797 \n",
       "L 3578 0 \n",
       "L 2900 0 \n",
       "L 1881 1375 \n",
       "L 863 0 \n",
       "L 184 0 \n",
       "L 1544 1831 \n",
       "L 300 3500 \n",
       "L 978 3500 \n",
       "L 1906 2253 \n",
       "L 2834 3500 \n",
       "L 3513 3500 \n",
       "z\n",
       "\" id=\"DejaVuSans-78\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 1959 3097 \n",
       "Q 1497 3097 1228 2736 \n",
       "Q 959 2375 959 1747 \n",
       "Q 959 1119 1226 758 \n",
       "Q 1494 397 1959 397 \n",
       "Q 2419 397 2687 759 \n",
       "Q 2956 1122 2956 1747 \n",
       "Q 2956 2369 2687 2733 \n",
       "Q 2419 3097 1959 3097 \n",
       "z\n",
       "M 1959 3584 \n",
       "Q 2709 3584 3137 3096 \n",
       "Q 3566 2609 3566 1747 \n",
       "Q 3566 888 3137 398 \n",
       "Q 2709 -91 1959 -91 \n",
       "Q 1206 -91 779 398 \n",
       "Q 353 888 353 1747 \n",
       "Q 353 2609 779 3096 \n",
       "Q 1206 3584 1959 3584 \n",
       "z\n",
       "\" id=\"DejaVuSans-6f\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 2188 4044 \n",
       "L 1331 1722 \n",
       "L 3047 1722 \n",
       "L 2188 4044 \n",
       "z\n",
       "M 1831 4666 \n",
       "L 2547 4666 \n",
       "L 4325 0 \n",
       "L 3669 0 \n",
       "L 3244 1197 \n",
       "L 1141 1197 \n",
       "L 716 0 \n",
       "L 50 0 \n",
       "L 1831 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-41\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 2059 -325 \n",
       "Q 1816 -950 1584 -1140 \n",
       "Q 1353 -1331 966 -1331 \n",
       "L 506 -1331 \n",
       "L 506 -850 \n",
       "L 844 -850 \n",
       "Q 1081 -850 1212 -737 \n",
       "Q 1344 -625 1503 -206 \n",
       "L 1606 56 \n",
       "L 191 3500 \n",
       "L 800 3500 \n",
       "L 1894 763 \n",
       "L 2988 3500 \n",
       "L 3597 3500 \n",
       "L 2059 -325 \n",
       "z\n",
       "\" id=\"DejaVuSans-79\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 313 2009 \n",
       "L 1997 2009 \n",
       "L 1997 1497 \n",
       "L 313 1497 \n",
       "L 313 2009 \n",
       "z\n",
       "\" id=\"DejaVuSans-2d\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 3513 2113 \n",
       "L 3513 0 \n",
       "L 2938 0 \n",
       "L 2938 2094 \n",
       "Q 2938 2591 2744 2837 \n",
       "Q 2550 3084 2163 3084 \n",
       "Q 1697 3084 1428 2787 \n",
       "Q 1159 2491 1159 1978 \n",
       "L 1159 0 \n",
       "L 581 0 \n",
       "L 581 4863 \n",
       "L 1159 4863 \n",
       "L 1159 2956 \n",
       "Q 1366 3272 1645 3428 \n",
       "Q 1925 3584 2291 3584 \n",
       "Q 2894 3584 3203 3211 \n",
       "Q 3513 2838 3513 2113 \n",
       "z\n",
       "\" id=\"DejaVuSans-68\" transform=\"scale(0.015625)\"/>\n",
       "      <path d=\"M 2906 2969 \n",
       "L 2906 4863 \n",
       "L 3481 4863 \n",
       "L 3481 0 \n",
       "L 2906 0 \n",
       "L 2906 525 \n",
       "Q 2725 213 2448 61 \n",
       "Q 2172 -91 1784 -91 \n",
       "Q 1150 -91 751 415 \n",
       "Q 353 922 353 1747 \n",
       "Q 353 2572 751 3078 \n",
       "Q 1150 3584 1784 3584 \n",
       "Q 2172 3584 2448 3432 \n",
       "Q 2725 3281 2906 2969 \n",
       "z\n",
       "M 947 1747 \n",
       "Q 947 1113 1208 752 \n",
       "Q 1469 391 1925 391 \n",
       "Q 2381 391 2643 752 \n",
       "Q 2906 1113 2906 1747 \n",
       "Q 2906 2381 2643 2742 \n",
       "Q 2381 3103 1925 3103 \n",
       "Q 1469 3103 1208 2742 \n",
       "Q 947 2381 947 1747 \n",
       "z\n",
       "\" id=\"DejaVuSans-64\" transform=\"scale(0.015625)\"/>\n",
       "     </defs>\n",
       "     <use xlink:href=\"#DejaVuSans-54\"/>\n",
       "     <use x=\"44.583984\" xlink:href=\"#DejaVuSans-77\"/>\n",
       "     <use x=\"126.371094\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "     <use x=\"154.154297\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "     <use x=\"193.363281\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "     <use x=\"232.572266\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "     <use x=\"294.095703\" xlink:href=\"#DejaVuSans-72\"/>\n",
       "     <use x=\"335.208984\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"366.996094\" xlink:href=\"#DejaVuSans-54\"/>\n",
       "     <use x=\"411.080078\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "     <use x=\"470.853516\" xlink:href=\"#DejaVuSans-78\"/>\n",
       "     <use x=\"530.033203\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "     <use x=\"569.242188\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"601.029297\" xlink:href=\"#DejaVuSans-44\"/>\n",
       "     <use x=\"678.03125\" xlink:href=\"#DejaVuSans-75\"/>\n",
       "     <use x=\"741.410156\" xlink:href=\"#DejaVuSans-70\"/>\n",
       "     <use x=\"804.886719\" xlink:href=\"#DejaVuSans-6c\"/>\n",
       "     <use x=\"832.669922\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "     <use x=\"860.453125\" xlink:href=\"#DejaVuSans-63\"/>\n",
       "     <use x=\"915.433594\" xlink:href=\"#DejaVuSans-61\"/>\n",
       "     <use x=\"976.712891\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "     <use x=\"1015.921875\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "     <use x=\"1043.705078\" xlink:href=\"#DejaVuSans-6f\"/>\n",
       "     <use x=\"1104.886719\" xlink:href=\"#DejaVuSans-6e\"/>\n",
       "     <use x=\"1168.265625\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"1200.052734\" xlink:href=\"#DejaVuSans-41\"/>\n",
       "     <use x=\"1268.460938\" xlink:href=\"#DejaVuSans-6e\"/>\n",
       "     <use x=\"1331.839844\" xlink:href=\"#DejaVuSans-61\"/>\n",
       "     <use x=\"1393.119141\" xlink:href=\"#DejaVuSans-6c\"/>\n",
       "     <use x=\"1420.902344\" xlink:href=\"#DejaVuSans-79\"/>\n",
       "     <use x=\"1480.082031\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "     <use x=\"1532.181641\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "     <use x=\"1559.964844\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "     <use x=\"1612.064453\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"1643.851562\" xlink:href=\"#DejaVuSans-2d\"/>\n",
       "     <use x=\"1679.935547\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"1711.722656\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "     <use x=\"1775.345703\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "     <use x=\"1807.132812\" xlink:href=\"#DejaVuSans-37\"/>\n",
       "     <use x=\"1870.755859\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "     <use x=\"1902.542969\" xlink:href=\"#DejaVuSans-54\"/>\n",
       "     <use x=\"1963.626953\" xlink:href=\"#DejaVuSans-68\"/>\n",
       "     <use x=\"2027.005859\" xlink:href=\"#DejaVuSans-72\"/>\n",
       "     <use x=\"2065.869141\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "     <use x=\"2127.392578\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "     <use x=\"2179.492188\" xlink:href=\"#DejaVuSans-68\"/>\n",
       "     <use x=\"2242.871094\" xlink:href=\"#DejaVuSans-6f\"/>\n",
       "     <use x=\"2304.052734\" xlink:href=\"#DejaVuSans-6c\"/>\n",
       "     <use x=\"2331.835938\" xlink:href=\"#DejaVuSans-64\"/>\n",
       "    </g>\n",
       "   </g>\n",
       "   <g id=\"legend_1\">\n",
       "    <g id=\"patch_9\">\n",
       "     <path d=\"M 53.0125 61.953906 \n",
       "L 190.782812 61.953906 \n",
       "Q 192.782812 61.953906 192.782812 59.953906 \n",
       "L 192.782812 31.597656 \n",
       "Q 192.782812 29.597656 190.782812 29.597656 \n",
       "L 53.0125 29.597656 \n",
       "Q 51.0125 29.597656 51.0125 31.597656 \n",
       "L 51.0125 59.953906 \n",
       "Q 51.0125 61.953906 53.0125 61.953906 \n",
       "z\n",
       "\" style=\"fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;\"/>\n",
       "    </g>\n",
       "    <g id=\"patch_10\">\n",
       "     <path d=\"M 55.0125 41.196094 \n",
       "L 75.0125 41.196094 \n",
       "L 75.0125 34.196094 \n",
       "L 55.0125 34.196094 \n",
       "z\n",
       "\" style=\"fill:#d62728;\"/>\n",
       "    </g>\n",
       "    <g id=\"text_13\">\n",
       "     <!-- Near Duplicates - .70 -->\n",
       "     <g transform=\"translate(83.0125 41.196094)scale(0.1 -0.1)\">\n",
       "      <defs>\n",
       "       <path d=\"M 628 4666 \n",
       "L 1478 4666 \n",
       "L 3547 763 \n",
       "L 3547 4666 \n",
       "L 4159 4666 \n",
       "L 4159 0 \n",
       "L 3309 0 \n",
       "L 1241 3903 \n",
       "L 1241 0 \n",
       "L 628 0 \n",
       "L 628 4666 \n",
       "z\n",
       "\" id=\"DejaVuSans-4e\" transform=\"scale(0.015625)\"/>\n",
       "      </defs>\n",
       "      <use xlink:href=\"#DejaVuSans-4e\"/>\n",
       "      <use x=\"74.804688\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "      <use x=\"136.328125\" xlink:href=\"#DejaVuSans-61\"/>\n",
       "      <use x=\"197.607422\" xlink:href=\"#DejaVuSans-72\"/>\n",
       "      <use x=\"238.720703\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"270.507812\" xlink:href=\"#DejaVuSans-44\"/>\n",
       "      <use x=\"347.509766\" xlink:href=\"#DejaVuSans-75\"/>\n",
       "      <use x=\"410.888672\" xlink:href=\"#DejaVuSans-70\"/>\n",
       "      <use x=\"474.365234\" xlink:href=\"#DejaVuSans-6c\"/>\n",
       "      <use x=\"502.148438\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "      <use x=\"529.931641\" xlink:href=\"#DejaVuSans-63\"/>\n",
       "      <use x=\"584.912109\" xlink:href=\"#DejaVuSans-61\"/>\n",
       "      <use x=\"646.191406\" xlink:href=\"#DejaVuSans-74\"/>\n",
       "      <use x=\"685.400391\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "      <use x=\"746.923828\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "      <use x=\"799.023438\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"830.810547\" xlink:href=\"#DejaVuSans-2d\"/>\n",
       "      <use x=\"866.894531\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"898.681641\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "      <use x=\"930.46875\" xlink:href=\"#DejaVuSans-37\"/>\n",
       "      <use x=\"994.091797\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "    <g id=\"patch_11\">\n",
       "     <path d=\"M 55.0125 55.874219 \n",
       "L 75.0125 55.874219 \n",
       "L 75.0125 48.874219 \n",
       "L 55.0125 48.874219 \n",
       "z\n",
       "\" style=\"fill:#2ca02c;\"/>\n",
       "    </g>\n",
       "    <g id=\"text_14\">\n",
       "     <!-- Uniques - .70 -->\n",
       "     <g transform=\"translate(83.0125 55.874219)scale(0.1 -0.1)\">\n",
       "      <use xlink:href=\"#DejaVuSans-55\"/>\n",
       "      <use x=\"73.193359\" xlink:href=\"#DejaVuSans-6e\"/>\n",
       "      <use x=\"136.572266\" xlink:href=\"#DejaVuSans-69\"/>\n",
       "      <use x=\"164.355469\" xlink:href=\"#DejaVuSans-71\"/>\n",
       "      <use x=\"227.832031\" xlink:href=\"#DejaVuSans-75\"/>\n",
       "      <use x=\"291.210938\" xlink:href=\"#DejaVuSans-65\"/>\n",
       "      <use x=\"352.734375\" xlink:href=\"#DejaVuSans-73\"/>\n",
       "      <use x=\"404.833984\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"436.621094\" xlink:href=\"#DejaVuSans-2d\"/>\n",
       "      <use x=\"472.705078\" xlink:href=\"#DejaVuSans-20\"/>\n",
       "      <use x=\"504.492188\" xlink:href=\"#DejaVuSans-2e\"/>\n",
       "      <use x=\"536.279297\" xlink:href=\"#DejaVuSans-37\"/>\n",
       "      <use x=\"599.902344\" xlink:href=\"#DejaVuSans-30\"/>\n",
       "     </g>\n",
       "    </g>\n",
       "   </g>\n",
       "  </g>\n",
       " </g>\n",
       " <defs>\n",
       "  <clipPath id=\"pe2a3fdeeb7\">\n",
       "   <rect height=\"266.112\" width=\"357.12\" x=\"46.0125\" y=\"24.597656\"/>\n",
       "  </clipPath>\n",
       " </defs>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dups_df = pd.DataFrame.from_dict({'Near Duplicates - .70': [dups_70], 'Uniques - .70': [uniques_70]})\n",
    "\n",
    "ax=dups_df.plot(kind = 'bar',y=['Near Duplicates - .70', 'Uniques - .70'], fontsize=10, color=['C3', 'C2'], align='center', width=0.8, xlabel=\"Duplicates vs. Unique\")\n",
    "ax.set_title('Twitter Text Duplication Analysis - 0.7 Threshold', fontsize=15)\n",
    "for p in ax.patches:\n",
    "       ax.annotate(format(p.get_height(), '.1f'), \n",
    "                   (p.get_x() + p.get_width() / 2., p.get_height()/2), \n",
    "                   ha = 'center', va = 'center', \n",
    "                   xytext = (0, 9), \n",
    "                   textcoords = 'offset points') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "0d572117-cf0e-494a-9867-1da18ff1e2cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>distCol</th><th>id_A</th><th>id_B</th><th>tweet_text_A</th><th>tweet_text_B</th></tr>\n",
       "<tr><td>0.6666666666666667</td><td>541</td><td>9025</td><td>{an annual high s...</td><td>{watch: bonfire e...</td></tr>\n",
       "<tr><td>0.6666666666666667</td><td>8209</td><td>12419</td><td>{acting uvalde po...</td><td>{what did police ...</td></tr>\n",
       "<tr><td>0.6666666666666667</td><td>9179</td><td>9968</td><td>{4 franklin high ...</td><td>{about 1,000 stud...</td></tr>\n",
       "<tr><td>0.4545454545454546</td><td>8209</td><td>12487</td><td>{acting uvalde po...</td><td>{the acting uvald...</td></tr>\n",
       "<tr><td>0.6666666666666667</td><td>9179</td><td>12401</td><td>{4 franklin high ...</td><td>{2 students injur...</td></tr>\n",
       "<tr><td>0.6666666666666667</td><td>3120</td><td>4027</td><td>{jon hamm on his ...</td><td>{high school wres...</td></tr>\n",
       "<tr><td>0.6</td><td>3120</td><td>11201</td><td>{jon hamm on his ...</td><td>{thursday&#x27;s high ...</td></tr>\n",
       "<tr><td>0.6666666666666667</td><td>3452</td><td>12401</td><td>{developing: des ...</td><td>{2 students injur...</td></tr>\n",
       "<tr><td>0.6363636363636364</td><td>7595</td><td>10299</td><td>{preseason all-ba...</td><td>{photos: bay area...</td></tr>\n",
       "<tr><td>0.5</td><td>3120</td><td>6404</td><td>{jon hamm on his ...</td><td>{here are the okl...</td></tr>\n",
       "<tr><td>0.4</td><td>3120</td><td>4869</td><td>{jon hamm on his ...</td><td>{week 5 high scho...</td></tr>\n",
       "<tr><td>0.6666666666666667</td><td>3120</td><td>5404</td><td>{jon hamm on his ...</td><td>{vote for the guy...</td></tr>\n",
       "<tr><td>0.5</td><td>3120</td><td>12935</td><td>{jon hamm on his ...</td><td>{houston high sch...</td></tr>\n",
       "<tr><td>0.6666666666666667</td><td>3120</td><td>3697</td><td>{jon hamm on his ...</td><td>{mid-penn tackles...</td></tr>\n",
       "<tr><td>0.5714285714285714</td><td>3120</td><td>5254</td><td>{jon hamm on his ...</td><td>{get ready for to...</td></tr>\n",
       "<tr><td>0.625</td><td>3120</td><td>3838</td><td>{jon hamm on his ...</td><td>{who are oklahoma...</td></tr>\n",
       "<tr><td>0.4545454545454546</td><td>7595</td><td>10414</td><td>{preseason all-ba...</td><td>{bay area news gr...</td></tr>\n",
       "<tr><td>0.6666666666666667</td><td>3120</td><td>11648</td><td>{jon hamm on his ...</td><td>{high school scor...</td></tr>\n",
       "<tr><td>0.625</td><td>3120</td><td>11482</td><td>{jon hamm on his ...</td><td>{there&#x27;s a new no...</td></tr>\n",
       "<tr><td>0.625</td><td>3120</td><td>11630</td><td>{jon hamm on his ...</td><td>{mid-penn high sc...</td></tr>\n",
       "</table>\n",
       "only showing top 20 rows\n"
      ],
      "text/plain": [
       "+------------------+----+-----+--------------------+--------------------+\n",
       "|           distCol|id_A| id_B|        tweet_text_A|        tweet_text_B|\n",
       "+------------------+----+-----+--------------------+--------------------+\n",
       "|0.6666666666666667| 541| 9025|{an annual high s...|{watch: bonfire e...|\n",
       "|0.6666666666666667|8209|12419|{acting uvalde po...|{what did police ...|\n",
       "|0.6666666666666667|9179| 9968|{4 franklin high ...|{about 1,000 stud...|\n",
       "|0.4545454545454546|8209|12487|{acting uvalde po...|{the acting uvald...|\n",
       "|0.6666666666666667|9179|12401|{4 franklin high ...|{2 students injur...|\n",
       "|0.6666666666666667|3120| 4027|{jon hamm on his ...|{high school wres...|\n",
       "|               0.6|3120|11201|{jon hamm on his ...|{thursday's high ...|\n",
       "|0.6666666666666667|3452|12401|{developing: des ...|{2 students injur...|\n",
       "|0.6363636363636364|7595|10299|{preseason all-ba...|{photos: bay area...|\n",
       "|               0.5|3120| 6404|{jon hamm on his ...|{here are the okl...|\n",
       "|               0.4|3120| 4869|{jon hamm on his ...|{week 5 high scho...|\n",
       "|0.6666666666666667|3120| 5404|{jon hamm on his ...|{vote for the guy...|\n",
       "|               0.5|3120|12935|{jon hamm on his ...|{houston high sch...|\n",
       "|0.6666666666666667|3120| 3697|{jon hamm on his ...|{mid-penn tackles...|\n",
       "|0.5714285714285714|3120| 5254|{jon hamm on his ...|{get ready for to...|\n",
       "|             0.625|3120| 3838|{jon hamm on his ...|{who are oklahoma...|\n",
       "|0.4545454545454546|7595|10414|{preseason all-ba...|{bay area news gr...|\n",
       "|0.6666666666666667|3120|11648|{jon hamm on his ...|{high school scor...|\n",
       "|             0.625|3120|11482|{jon hamm on his ...|{there's a new no...|\n",
       "|             0.625|3120|11630|{jon hamm on his ...|{mid-penn high sc...|\n",
       "+------------------+----+-----+--------------------+--------------------+\n",
       "only showing top 20 rows"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dups_70_tweet_text.unpersist()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c927c86c-caa4-4296-b93e-e1384692562c",
   "metadata": {},
   "source": [
    "### Comparing side by side"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "a0f2fde5-c519-4d46-bdb7-5abfd30b2c5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 276:====================================================>(199 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "483\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "df_dups_30_id = df_dups_30_tweet_text.select(col('id_A').alias('id_30_A'), col('id_B').alias('id_30_B'))\n",
    "print(df_dups_30_id.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "29ca656d-d411-477f-86b9-47c8ba66fbc3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 291:====================================================>(199 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------+\n",
      "|id_50_A|id_50_B|\n",
      "+-------+-------+\n",
      "|   8209|  12487|\n",
      "|   3120|   4869|\n",
      "|   7595|  10414|\n",
      "|   3120|   6907|\n",
      "|   6123|  12935|\n",
      "+-------+-------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "df_dups_50_id = df_dups_50_tweet_text.select(col('id_A').alias('id_50_A'), col('id_B').alias('id_50_B'))\n",
    "df_dups_50_id.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "449a68d3-10e6-4540-a371-118173ca8595",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 301:====================================================>(197 + 3) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------------+-------+-------+\n",
      "|        tweet_text_A|        tweet_text_B|id_70_A|id_70_B|\n",
      "+--------------------+--------------------+-------+-------+\n",
      "|{this weekend mar...|{high school foot...|   4823|   5269|\n",
      "|{this weekend mar...|{the tennessee hi...|   4823|  12578|\n",
      "|{this weekend mar...|{ex-giant kerry c...|   4823|   6850|\n",
      "|{this weekend mar...|{high school http...|   4823|   7962|\n",
      "|{this weekend mar...|{the latest tn hi...|   4823|  12369|\n",
      "+--------------------+--------------------+-------+-------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "df_dups_70_id = df_dups_70_tweet_text.select(col('tweet_text_A'), col('tweet_text_B'),\\\n",
    "                                  col('id_A').alias('id_70_A'), col('id_B').alias('id_70_B'))\n",
    "df_dups_70_id.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "b53b5043-2b9f-4806-9fff-e404f9805d16",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_text_A</th>\n",
       "      <th>tweet_text_B</th>\n",
       "      <th>id_70_A</th>\n",
       "      <th>id_70_B</th>\n",
       "      <th>id_50_A</th>\n",
       "      <th>id_50_B</th>\n",
       "      <th>id_30_A</th>\n",
       "      <th>id_30_B</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(this weekend marks the end of the regular high school football season. https://t.co/c3idscccln,)</td>\n",
       "      <td>(high school https://t.co/a2a0tbwnqc,)</td>\n",
       "      <td>4823</td>\n",
       "      <td>7962</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(this weekend marks the end of the regular high school football season. https://t.co/c3idscccln,)</td>\n",
       "      <td>(high school football is back! https://t.co/nnn0zdq66h,)</td>\n",
       "      <td>4823</td>\n",
       "      <td>5269</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(this weekend marks the end of the regular high school football season. https://t.co/c3idscccln,)</td>\n",
       "      <td>(the tennessee high school football scores from week 1 of the tssaa 2022 regular season are in.\\n\\nhttps://t.co/azd8wcibv8,)</td>\n",
       "      <td>4823</td>\n",
       "      <td>12578</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(this weekend marks the end of the regular high school football season. https://t.co/c3idscccln,)</td>\n",
       "      <td>(the latest tn high school football ap rankings are out.\\n\\nhttps://t.co/qtpr7wjnw0\\n\\n@ohspatsfootball  @wcsrhsfootball  @caneridgefball @pagefootball @wcsnhsathletics @wcsnhspd @eastnash_fb @lam...</td>\n",
       "      <td>4823</td>\n",
       "      <td>12369</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(this weekend marks the end of the regular high school football season. https://t.co/c3idscccln,)</td>\n",
       "      <td>(ex-giant kerry collins now a high school football coach #nyg #giants #nfl https://t.co/kltogu7l97,)</td>\n",
       "      <td>4823</td>\n",
       "      <td>6850</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                        tweet_text_A  \\\n",
       "0  (this weekend marks the end of the regular high school football season. https://t.co/c3idscccln,)   \n",
       "1  (this weekend marks the end of the regular high school football season. https://t.co/c3idscccln,)   \n",
       "2  (this weekend marks the end of the regular high school football season. https://t.co/c3idscccln,)   \n",
       "3  (this weekend marks the end of the regular high school football season. https://t.co/c3idscccln,)   \n",
       "4  (this weekend marks the end of the regular high school football season. https://t.co/c3idscccln,)   \n",
       "\n",
       "                                                                                                                                                                                              tweet_text_B  \\\n",
       "0                                                                                                                                                                   (high school https://t.co/a2a0tbwnqc,)   \n",
       "1                                                                                                                                                 (high school football is back! https://t.co/nnn0zdq66h,)   \n",
       "2                                                                             (the tennessee high school football scores from week 1 of the tssaa 2022 regular season are in.\\n\\nhttps://t.co/azd8wcibv8,)   \n",
       "3  (the latest tn high school football ap rankings are out.\\n\\nhttps://t.co/qtpr7wjnw0\\n\\n@ohspatsfootball  @wcsrhsfootball  @caneridgefball @pagefootball @wcsnhsathletics @wcsnhspd @eastnash_fb @lam...   \n",
       "4                                                                                                     (ex-giant kerry collins now a high school football coach #nyg #giants #nfl https://t.co/kltogu7l97,)   \n",
       "\n",
       "   id_70_A  id_70_B  id_50_A  id_50_B  id_30_A  id_30_B  \n",
       "0     4823     7962      NaN      NaN      NaN      NaN  \n",
       "1     4823     5269      NaN      NaN      NaN      NaN  \n",
       "2     4823    12578      NaN      NaN      NaN      NaN  \n",
       "3     4823    12369      NaN      NaN      NaN      NaN  \n",
       "4     4823     6850      NaN      NaN      NaN      NaN  "
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dups_combined = df_dups_70_id.join(df_dups_50_id, on=((df_dups_70_id.id_70_A == df_dups_50_id.id_50_A) & (df_dups_70_id.id_70_B == df_dups_50_id.id_50_B)), how=\"left_outer\")\\\n",
    ".join(df_dups_30_id, on=((df_dups_70_id.id_70_A == df_dups_30_id.id_30_A) & (df_dups_70_id.id_70_B == df_dups_30_id.id_30_B)), how=\"left_outer\")\n",
    "\n",
    "dups_combined.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "258a6654-3848-432c-91f3-4a8c4711b510",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                ]\r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_text_A</th>\n",
       "      <th>tweet_text_B</th>\n",
       "      <th>threshold_30</th>\n",
       "      <th>threshold_50</th>\n",
       "      <th>threshold_70</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(friday night spotlight: oregon high school football week 2 scores, schedule, links https://t.co/0steze958p,)</td>\n",
       "      <td>(photos: bay area high school football week 6, 2022 https://t.co/vpa9sa7ttt,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(udaipur: students lock school gate over transfer of teacher https://t.co/lunutczj9f,)</td>\n",
       "      <td>(#iranrevolution\\ntehran, #iran's capital—#iranprotests against #mahsaamini's murder, day 19.\\nhigh school students protest, chanting:\\n\"no fear! we're all together!\"https://t.co/fkrcrnpxjk,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>('my son was due to graduate from kyiv university - now he's fighting the war'\\nhttps://t.co/mjbgtzi4pl https://t.co/ixqzg384fq,)</td>\n",
       "      <td>(\"the top uk university for graduate outcomes\" https://t.co/mkkw8l8r15 https://t.co/dbygno0plh,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(michigan kicks off its 2022 high school football season on thursday, aug. 25. https://t.co/tu0ipt7sgx,)</td>\n",
       "      <td>(tennessee high school football schedule for week 3 of tssaa 2022 season https://t.co/0juinxzhdj,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(high school football team under fire for 'blue lives matter' memorial on 9/11 https://t.co/qpl2wbf9dd,)</td>\n",
       "      <td>(robinson girls flag football best high school team in florida? https://t.co/xg1ucst0iz @robinsonflag @hillsboroughsch @buccaneers,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                        tweet_text_A  \\\n",
       "0                      (friday night spotlight: oregon high school football week 2 scores, schedule, links https://t.co/0steze958p,)   \n",
       "1                                             (udaipur: students lock school gate over transfer of teacher https://t.co/lunutczj9f,)   \n",
       "2  ('my son was due to graduate from kyiv university - now he's fighting the war'\\nhttps://t.co/mjbgtzi4pl https://t.co/ixqzg384fq,)   \n",
       "3                           (michigan kicks off its 2022 high school football season on thursday, aug. 25. https://t.co/tu0ipt7sgx,)   \n",
       "4                           (high school football team under fire for 'blue lives matter' memorial on 9/11 https://t.co/qpl2wbf9dd,)   \n",
       "\n",
       "                                                                                                                                                                                      tweet_text_B  \\\n",
       "0                                                                                                                    (photos: bay area high school football week 6, 2022 https://t.co/vpa9sa7ttt,)   \n",
       "1  (#iranrevolution\\ntehran, #iran's capital—#iranprotests against #mahsaamini's murder, day 19.\\nhigh school students protest, chanting:\\n\"no fear! we're all together!\"https://t.co/fkrcrnpxjk,)   \n",
       "2                                                                                                 (\"the top uk university for graduate outcomes\" https://t.co/mkkw8l8r15 https://t.co/dbygno0plh,)   \n",
       "3                                                                                               (tennessee high school football schedule for week 3 of tssaa 2022 season https://t.co/0juinxzhdj,)   \n",
       "4                                                             (robinson girls flag football best high school team in florida? https://t.co/xg1ucst0iz @robinsonflag @hillsboroughsch @buccaneers,)   \n",
       "\n",
       "  threshold_30 threshold_50 threshold_70  \n",
       "0      Non-Dup      Non-Dup    Duplicate  \n",
       "1      Non-Dup      Non-Dup    Duplicate  \n",
       "2      Non-Dup      Non-Dup    Duplicate  \n",
       "3      Non-Dup      Non-Dup    Duplicate  \n",
       "4      Non-Dup      Non-Dup    Duplicate  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dups_combined = dups_combined.\\\n",
    "withColumn(\"threshold_30\",\\\n",
    "           when(col(\"id_30_A\").isNotNull(), \"Duplicate\").\n",
    "           otherwise(\"Non-Dup\")).\\\n",
    "withColumn(\"threshold_50\",\\\n",
    "           when(col(\"id_50_A\").isNotNull(), \"Duplicate\").\n",
    "           otherwise(\"Non-Dup\")).\\\n",
    "withColumn(\"threshold_70\",\\\n",
    "           when(col(\"id_70_A\").isNotNull(), \"Duplicate\").\n",
    "           otherwise(\"Non-Dup\")).\\\n",
    "drop('id_30_A', 'id_30_B', 'id_50_A', 'id_50_B', 'id_70_A', 'id_70_B')\n",
    "\n",
    "dups_combined.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "15fd4c0d-6833-4e98-86b2-0cb65d702e04",
   "metadata": {},
   "outputs": [],
   "source": [
    "dups_combined_sample = \\\n",
    "dups_combined.filter((\"threshold_30 == 'Duplicate'\")).sample(False, 0.1, 12345).limit(10).\\\n",
    "union\\\n",
    "(dups_combined.filter((\"threshold_30 == 'Non-Dup' and threshold_50 == 'Duplicate'\")).sample(False, 0.1, 12345).limit(10)).\\\n",
    "union\\\n",
    "(dups_combined.filter((\"threshold_50 == 'Non-Dup'\")).sample(False, 0.1, 12345).limit(10)).\\\n",
    "orderBy('threshold_30', 'threshold_50', 'threshold_70')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "c7f3e1ab-6480-4355-8a1f-4579f556c592",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                ]\r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_text_A</th>\n",
       "      <th>tweet_text_B</th>\n",
       "      <th>threshold_30</th>\n",
       "      <th>threshold_50</th>\n",
       "      <th>threshold_70</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>(high school scores and top performers from monday, dec. 19 https://t.co/2ljyavcopl,)</td>\n",
       "      <td>(high school scores and top performers from tuesday, feb. 7 https://t.co/jd1dodpeck,)</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(while not detailing how the distribution might be funded, gov. abbott said the medication, narcan, should be distributed to law enforcement agencies as well as some hospitals and schools. https:/...</td>\n",
       "      <td>(while not detailing how the distribution might be funded, abbott said the medication, narcan, should be distributed to law enforcement agencies as well as some hospitals and schools. https://t.co...</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(the last nizam of telangana &amp;amp; his last order prohibited public programmes, speech, &amp;amp; open schools. this proved to be his last dictate. likewise, kcrs's prohibitory orders are the last one...</td>\n",
       "      <td>(the last nizam of telangana &amp;amp; his last order prohibited public programmes, speech, &amp;amp; open schools. this proved to be his last dictate. likewise, kcrs's prohibitory orders are the last one...</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(a hardback old school manual packed with history, intelligence, technique and insight. illustrated with beautiful hand drawn images across 165 pages of knock out knowledge!\\nin the uk order direc...</td>\n",
       "      <td>(a hardback old school manual packed with history, intelligence, technique and insight. illustrated with beautiful hand drawn images across 165 pages of knock out knowledge!\\nin the uk order direc...</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(congratulations to these 22 high school football players who made the week 6 friday night drive team of the week https://t.co/snxrwwqyck,)</td>\n",
       "      <td>(congratulations to these 22 high school football players who made the week 8 friday night drive team of the week https://t.co/srhld2fhjq,)</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>(new list of ‘best value’ colleges released: is your alma mater on it? https://t.co/fhjc8wady5,)</td>\n",
       "      <td>(new list of 'best value' colleges released: is your alma mater on it? https://t.co/ogtavua4kk,)</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>(high school football week 8 area schedule https://t.co/paayqndtlz,)</td>\n",
       "      <td>(photos: bay area high school football week 6, 2022 https://t.co/vpa9sa7ttt,)</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>(mom sues fla. schools, claiming district banned her over onlyfans posts\\nhttps://t.co/qhxlwvzgll,)</td>\n",
       "      <td>(mom sues florida schools, claiming district banned her over onlyfans posts https://t.co/xu53qot5je,)</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>(a teenager accused of killing four fellow students and injuring more at a michigan high school is expected to plead guilty to murder next week, authorities said friday. https://t.co/iqt1f1qadw,)</td>\n",
       "      <td>(a teenager accused of killing four fellow students and injuring more at a michigan high school is expected to plead guilty to murder next week, authorities said friday.\\n\\nhttps://t.co/symnj4yni3,)</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>(eastern massachusetts high school scores and highlights from wednesday https://t.co/il2rqvxwe7,)</td>\n",
       "      <td>(eastern massachusetts high school scores and highlights from friday https://t.co/xbhyugoni6,)</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>(\"it looks so real that it’s a little scary.\"\\ninstagram picture of teen flashing a gun in bethlehem high school leads to arrest https://t.co/mtv1ctdjgh,)</td>\n",
       "      <td>(instagram picture of teen flashing a gun in bethlehem high school leads to arrest https://t.co/mtv1ctdjgh,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>(as part of our d&amp;amp;i campaign, we’re presenting d&amp;amp;i initiatives @ utokyo. featured: center for research on counseling &amp;amp; support services, graduate schools of frontier sciences &amp;amp; pub...</td>\n",
       "      <td>(as part of our d&amp;amp;i campaign, we’re presenting d&amp;amp;i initiatives at utokyo. featured: graduate schools of frontier sciences and economics &amp;amp; international research center for neurointelli...</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>(high school football scores https://t.co/c57ecdfsb1,)</td>\n",
       "      <td>(high school scores and top performers from tuesday, oct. 25 https://t.co/jxmvbvix5j,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>(week 11 tennessee high school football scores.  https://t.co/prstwqztxo,)</td>\n",
       "      <td>(tennessee high school football rankings, associated press statewide poll for week 5 https://t.co/aysfx7kmkn,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>(here is this week’s ⁦⁦@opswamedia high school football notebook⁩!\\n\\nhttps://t.co/wqpymf2vos,)</td>\n",
       "      <td>(happy high school football friday y’all  https://t.co/xzeia5zutr,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>(saturday’s houston high school football roundup\\nhttps://t.co/sfilvoenx2,)</td>\n",
       "      <td>(1st &amp;amp; 10: week 10 of acadiana’s high school football [video] https://t.co/olsvjzukuq,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>(high school https://t.co/a2a0tbwnqc,)</td>\n",
       "      <td>(and that’s a sunset of high school football. https://t.co/kltovfgapo,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>(high school football stars of the week https://t.co/ridsvtchld,)</td>\n",
       "      <td>(week 12 connecticut high school football top performers https://t.co/jffypznmef,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>(here are the nominees for the one hour heating &amp;amp; air girls high school athlete of the week. poll closes friday.\\n\\nvote now. vote often\\n\\nhttps://t.co/83izsobbqz,)</td>\n",
       "      <td>(here are the nominees for @theobserver boys high school athlete of the week. poll closes friday.\\n\\nvote now. vote often\\n\\nhttps://t.co/a01l3h5dut,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>(this is a friendly reminder that there is no school november 23-25, 2022. division schools and offices will be closed.  mcps schools and offices will return to normal operations monday, november ...</td>\n",
       "      <td>(we are a week away from fall break! there will be no school november 23-25, 2022. division schools and offices will be closed.  mcps schools and offices will return to normal operations monday, n...</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>(get ready for tonight's high school football state quarterfinals:\\n\\npairings https://t.co/vxvtxutwb5\\n\\npicks https://t.co/erircrobrn\\n\\nclass a-class 5a previews https://t.co/r4fifuvydx\\n\\nclas...</td>\n",
       "      <td>(wiaa level two high school football highlights here! 🏈 https://t.co/bduc7pwzl2,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>(for final scores from saturday's high school football action from across the state, check the live scores page: https://t.co/qhxafeqdew #copreps,)</td>\n",
       "      <td>(breaking down all of week 4 high school football action for southcoast teams https://t.co/jducztpq3i,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>(high school football is back! https://t.co/nnn0zdq66h,)</td>\n",
       "      <td>(high school scores and top performers from tuesday, feb. 7 https://t.co/jd1dodpeck,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>(just in: ohio county schools closed jan 3, 2023.,)</td>\n",
       "      <td>(reminder: friday, oct. 7 is a pa day for elementary and secondary students. #hdsb schools will be closed to students. https://t.co/srymwi6eir,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>(high school https://t.co/q3caeejgxy,)</td>\n",
       "      <td>(the artist, who is a high school teacher, got a frenzy of likes after posting the art on facebook.  https://t.co/zaygdydjja,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>(2022 pine belt high school football day-by-day schedules https://t.co/j2xkc4lm0i,)</td>\n",
       "      <td>(houston high school football rankings: week 8 https://t.co/cfdvriinp2 via @houstonchron | #txhsfb,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>(the 2022 high school football season is getting closer.  https://t.co/4js7uook5m,)</td>\n",
       "      <td>(high school football stars of the week https://t.co/ridsvtchld,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>(a stockton high school was locked down monday after a “trespasser” stabbed and killed a student, police said.\\n\\nhttps://t.co/bw59sqbqpj,)</td>\n",
       "      <td>(alycia reynaga was stabbed in front of her high school monday and died later at a hospital. https://t.co/hg3psbszox,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>(what's your favorite high school subject? (finals),)</td>\n",
       "      <td>(saturday’s houston high school football roundup\\nhttps://t.co/sfilvoenx2,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>(aye what middle school did steve lacy go to?,)</td>\n",
       "      <td>(today's cool school is anderson middle school in new iberia! https://t.co/5crhpunw8b https://t.co/acyofjfgaj,)</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Non-Dup</td>\n",
       "      <td>Duplicate</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                               tweet_text_A  \\\n",
       "0                                                                                                                     (high school scores and top performers from monday, dec. 19 https://t.co/2ljyavcopl,)   \n",
       "1   (while not detailing how the distribution might be funded, gov. abbott said the medication, narcan, should be distributed to law enforcement agencies as well as some hospitals and schools. https:/...   \n",
       "2   (the last nizam of telangana &amp; his last order prohibited public programmes, speech, &amp; open schools. this proved to be his last dictate. likewise, kcrs's prohibitory orders are the last one...   \n",
       "3   (a hardback old school manual packed with history, intelligence, technique and insight. illustrated with beautiful hand drawn images across 165 pages of knock out knowledge!\\nin the uk order direc...   \n",
       "4                                                               (congratulations to these 22 high school football players who made the week 6 friday night drive team of the week https://t.co/snxrwwqyck,)   \n",
       "5                                                                                                          (new list of ‘best value’ colleges released: is your alma mater on it? https://t.co/fhjc8wady5,)   \n",
       "6                                                                                                                                      (high school football week 8 area schedule https://t.co/paayqndtlz,)   \n",
       "7                                                                                                       (mom sues fla. schools, claiming district banned her over onlyfans posts\\nhttps://t.co/qhxlwvzgll,)   \n",
       "8       (a teenager accused of killing four fellow students and injuring more at a michigan high school is expected to plead guilty to murder next week, authorities said friday. https://t.co/iqt1f1qadw,)   \n",
       "9                                                                                                         (eastern massachusetts high school scores and highlights from wednesday https://t.co/il2rqvxwe7,)   \n",
       "10                                               (\"it looks so real that it’s a little scary.\"\\ninstagram picture of teen flashing a gun in bethlehem high school leads to arrest https://t.co/mtv1ctdjgh,)   \n",
       "11  (as part of our d&amp;i campaign, we’re presenting d&amp;i initiatives @ utokyo. featured: center for research on counseling &amp; support services, graduate schools of frontier sciences &amp; pub...   \n",
       "12                                                                                                                                                   (high school football scores https://t.co/c57ecdfsb1,)   \n",
       "13                                                                                                                               (week 11 tennessee high school football scores.  https://t.co/prstwqztxo,)   \n",
       "14                                                                                                          (here is this week’s ⁦⁦@opswamedia high school football notebook⁩!\\n\\nhttps://t.co/wqpymf2vos,)   \n",
       "15                                                                                                                              (saturday’s houston high school football roundup\\nhttps://t.co/sfilvoenx2,)   \n",
       "16                                                                                                                                                                   (high school https://t.co/a2a0tbwnqc,)   \n",
       "17                                                                                                                                        (high school football stars of the week https://t.co/ridsvtchld,)   \n",
       "18                                (here are the nominees for the one hour heating &amp; air girls high school athlete of the week. poll closes friday.\\n\\nvote now. vote often\\n\\nhttps://t.co/83izsobbqz,)   \n",
       "19  (this is a friendly reminder that there is no school november 23-25, 2022. division schools and offices will be closed.  mcps schools and offices will return to normal operations monday, november ...   \n",
       "20  (get ready for tonight's high school football state quarterfinals:\\n\\npairings https://t.co/vxvtxutwb5\\n\\npicks https://t.co/erircrobrn\\n\\nclass a-class 5a previews https://t.co/r4fifuvydx\\n\\nclas...   \n",
       "21                                                      (for final scores from saturday's high school football action from across the state, check the live scores page: https://t.co/qhxafeqdew #copreps,)   \n",
       "22                                                                                                                                                 (high school football is back! https://t.co/nnn0zdq66h,)   \n",
       "23                                                                                                                                                      (just in: ohio county schools closed jan 3, 2023.,)   \n",
       "24                                                                                                                                                                   (high school https://t.co/q3caeejgxy,)   \n",
       "25                                                                                                                      (2022 pine belt high school football day-by-day schedules https://t.co/j2xkc4lm0i,)   \n",
       "26                                                                                                                      (the 2022 high school football season is getting closer.  https://t.co/4js7uook5m,)   \n",
       "27                                                              (a stockton high school was locked down monday after a “trespasser” stabbed and killed a student, police said.\\n\\nhttps://t.co/bw59sqbqpj,)   \n",
       "28                                                                                                                                                    (what's your favorite high school subject? (finals),)   \n",
       "29                                                                                                                                                          (aye what middle school did steve lacy go to?,)   \n",
       "\n",
       "                                                                                                                                                                                               tweet_text_B  \\\n",
       "0                                                                                                                     (high school scores and top performers from tuesday, feb. 7 https://t.co/jd1dodpeck,)   \n",
       "1   (while not detailing how the distribution might be funded, abbott said the medication, narcan, should be distributed to law enforcement agencies as well as some hospitals and schools. https://t.co...   \n",
       "2   (the last nizam of telangana &amp; his last order prohibited public programmes, speech, &amp; open schools. this proved to be his last dictate. likewise, kcrs's prohibitory orders are the last one...   \n",
       "3   (a hardback old school manual packed with history, intelligence, technique and insight. illustrated with beautiful hand drawn images across 165 pages of knock out knowledge!\\nin the uk order direc...   \n",
       "4                                                               (congratulations to these 22 high school football players who made the week 8 friday night drive team of the week https://t.co/srhld2fhjq,)   \n",
       "5                                                                                                          (new list of 'best value' colleges released: is your alma mater on it? https://t.co/ogtavua4kk,)   \n",
       "6                                                                                                                             (photos: bay area high school football week 6, 2022 https://t.co/vpa9sa7ttt,)   \n",
       "7                                                                                                     (mom sues florida schools, claiming district banned her over onlyfans posts https://t.co/xu53qot5je,)   \n",
       "8    (a teenager accused of killing four fellow students and injuring more at a michigan high school is expected to plead guilty to murder next week, authorities said friday.\\n\\nhttps://t.co/symnj4yni3,)   \n",
       "9                                                                                                            (eastern massachusetts high school scores and highlights from friday https://t.co/xbhyugoni6,)   \n",
       "10                                                                                             (instagram picture of teen flashing a gun in bethlehem high school leads to arrest https://t.co/mtv1ctdjgh,)   \n",
       "11  (as part of our d&amp;i campaign, we’re presenting d&amp;i initiatives at utokyo. featured: graduate schools of frontier sciences and economics &amp; international research center for neurointelli...   \n",
       "12                                                                                                                   (high school scores and top performers from tuesday, oct. 25 https://t.co/jxmvbvix5j,)   \n",
       "13                                                                                           (tennessee high school football rankings, associated press statewide poll for week 5 https://t.co/aysfx7kmkn,)   \n",
       "14                                                                                                                                      (happy high school football friday y’all  https://t.co/xzeia5zutr,)   \n",
       "15                                                                                                              (1st &amp; 10: week 10 of acadiana’s high school football [video] https://t.co/olsvjzukuq,)   \n",
       "16                                                                                                                                  (and that’s a sunset of high school football. https://t.co/kltovfgapo,)   \n",
       "17                                                                                                                       (week 12 connecticut high school football top performers https://t.co/jffypznmef,)   \n",
       "18                                                   (here are the nominees for @theobserver boys high school athlete of the week. poll closes friday.\\n\\nvote now. vote often\\n\\nhttps://t.co/a01l3h5dut,)   \n",
       "19  (we are a week away from fall break! there will be no school november 23-25, 2022. division schools and offices will be closed.  mcps schools and offices will return to normal operations monday, n...   \n",
       "20                                                                                                                        (wiaa level two high school football highlights here! 🏈 https://t.co/bduc7pwzl2,)   \n",
       "21                                                                                                  (breaking down all of week 4 high school football action for southcoast teams https://t.co/jducztpq3i,)   \n",
       "22                                                                                                                    (high school scores and top performers from tuesday, feb. 7 https://t.co/jd1dodpeck,)   \n",
       "23                                                         (reminder: friday, oct. 7 is a pa day for elementary and secondary students. #hdsb schools will be closed to students. https://t.co/srymwi6eir,)   \n",
       "24                                                                           (the artist, who is a high school teacher, got a frenzy of likes after posting the art on facebook.  https://t.co/zaygdydjja,)   \n",
       "25                                                                                                     (houston high school football rankings: week 8 https://t.co/cfdvriinp2 via @houstonchron | #txhsfb,)   \n",
       "26                                                                                                                                        (high school football stars of the week https://t.co/ridsvtchld,)   \n",
       "27                                                                                   (alycia reynaga was stabbed in front of her high school monday and died later at a hospital. https://t.co/hg3psbszox,)   \n",
       "28                                                                                                                              (saturday’s houston high school football roundup\\nhttps://t.co/sfilvoenx2,)   \n",
       "29                                                                                          (today's cool school is anderson middle school in new iberia! https://t.co/5crhpunw8b https://t.co/acyofjfgaj,)   \n",
       "\n",
       "   threshold_30 threshold_50 threshold_70  \n",
       "0     Duplicate    Duplicate    Duplicate  \n",
       "1     Duplicate    Duplicate    Duplicate  \n",
       "2     Duplicate    Duplicate    Duplicate  \n",
       "3     Duplicate    Duplicate    Duplicate  \n",
       "4     Duplicate    Duplicate    Duplicate  \n",
       "5     Duplicate    Duplicate    Duplicate  \n",
       "6     Duplicate    Duplicate    Duplicate  \n",
       "7     Duplicate    Duplicate    Duplicate  \n",
       "8     Duplicate    Duplicate    Duplicate  \n",
       "9     Duplicate    Duplicate    Duplicate  \n",
       "10      Non-Dup    Duplicate    Duplicate  \n",
       "11      Non-Dup    Duplicate    Duplicate  \n",
       "12      Non-Dup    Duplicate    Duplicate  \n",
       "13      Non-Dup    Duplicate    Duplicate  \n",
       "14      Non-Dup    Duplicate    Duplicate  \n",
       "15      Non-Dup    Duplicate    Duplicate  \n",
       "16      Non-Dup    Duplicate    Duplicate  \n",
       "17      Non-Dup    Duplicate    Duplicate  \n",
       "18      Non-Dup    Duplicate    Duplicate  \n",
       "19      Non-Dup    Duplicate    Duplicate  \n",
       "20      Non-Dup      Non-Dup    Duplicate  \n",
       "21      Non-Dup      Non-Dup    Duplicate  \n",
       "22      Non-Dup      Non-Dup    Duplicate  \n",
       "23      Non-Dup      Non-Dup    Duplicate  \n",
       "24      Non-Dup      Non-Dup    Duplicate  \n",
       "25      Non-Dup      Non-Dup    Duplicate  \n",
       "26      Non-Dup      Non-Dup    Duplicate  \n",
       "27      Non-Dup      Non-Dup    Duplicate  \n",
       "28      Non-Dup      Non-Dup    Duplicate  \n",
       "29      Non-Dup      Non-Dup    Duplicate  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dups_combined_sample.limit(30).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98d7eeeb-29b7-4b12-a242-8d5fc75de1fc",
   "metadata": {},
   "source": [
    "### Because of verified posts, the duplicity between tweets was low. A Jaccard distance of 0.5 was employed to find near duplicates using count-vectorizer and locality sensitive hashing"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}