socialbakers_scraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Web scraping\n",
    "\n",
    "## Introduction\n",
    "Scrapping twitter handles from the [social bakers site.](https://www.socialbakers.com/statistics/twitter/profiles/kenya)\n",
    "The site has classified twitter users into categories like:\n",
    "* Brands.\n",
    "* Celebrities.\n",
    "* Community.\n",
    "* Entertainment.\n",
    "* Media\n",
    "* Places\n",
    "* Society\n",
    "* Sports\n",
    "\n",
    "The handles listed are for some of the most popular users.\n",
    "Each category has its unique URL. \n",
    "\n",
    "## Table of Contents\n",
    "1. [Scraper](#Scraper)\n",
    "2. Categories:\n",
    "   * [Sports](#sports)\n",
    "   * [Society](#Society)\n",
    "   * [Place](#Place)\n",
    "   * [Restaurants and cafes](#Restaurants-and-cafes)\n",
    "   * [Media](#Media)\n",
    "   * [Entertainment](#Entertainment)\n",
    "   * [Community](#Community)\n",
    "   * [Celebrities](#Celebrities)\n",
    "   * [Brands](#Brands)\n",
    "   \n",
    "3. [Creating Final DF](#Concating-the-dataframes-to-one-mega-df.)\n",
    "   \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from requests import get\n",
    "from requests.exceptions import RequestException\n",
    "from contextlib import closing\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import os, sys\n",
    "\n",
    "import fire\n",
    "\n",
    "import sys\n",
    "import os\n",
    "import json\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "import string\n",
    "\n",
    "import matplotlib.dates as mdates\n",
    "import seaborn as sns\n",
    "\n",
    "# to view all columns\n",
    "pd.set_option(\"display.max.columns\", None)\n",
    "\n",
    "import preprocessor as p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function: [credits](https://github.com/yabebalFantaye)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simple_get(url):\n",
    "    \"\"\"\n",
    "    Attempts to get the content at `url` by making an HTTP GET request.\n",
    "    If the content-type of response is some kind of HTML/XML, return the\n",
    "    text content, otherwise return None.\n",
    "    \"\"\"\n",
    "    try:\n",
    "        with closing(get(url, stream=True)) as resp:\n",
    "            if is_good_response(resp):\n",
    "                return resp.content  #.encode(BeautifulSoup.original_encoding)\n",
    "            else:\n",
    "                return None\n",
    "\n",
    "    except RequestException as e:\n",
    "        log_error('Error during requests to {0} : {1}'.format(url, str(e)))\n",
    "        return None\n",
    "\n",
    "\n",
    "def is_good_response(resp):\n",
    "    \"\"\"\n",
    "    Returns True if the response seems to be HTML, False otherwise.\n",
    "    \"\"\"\n",
    "    content_type = resp.headers['Content-Type'].lower()\n",
    "    return (resp.status_code == 200 \n",
    "            and content_type is not None \n",
    "            and content_type.find('html') > -1)\n",
    "\n",
    "\n",
    "def log_error(e):\n",
    "    \"\"\"\n",
    "    It is always a good idea to log errors. \n",
    "    This function just prints them, but you can\n",
    "    make it do anything.\n",
    "    \"\"\"\n",
    "    print(e)\n",
    "    \n",
    "def get_elements(url, tag='',search={}, fname=None):\n",
    "    \"\"\"\n",
    "    Downloads a page specified by the url parameter\n",
    "    and returns a list of strings, one per tag element\n",
    "    \"\"\"\n",
    "    \n",
    "    if isinstance(url,str):\n",
    "        response = simple_get(url)\n",
    "    else:\n",
    "        #if already it is a loaded html page\n",
    "        response = url\n",
    "\n",
    "    if response is not None:\n",
    "        html = BeautifulSoup(response, 'html.parser')\n",
    "        \n",
    "        res = []\n",
    "        if tag:    \n",
    "            for li in html.select(tag):\n",
    "                for name in li.text.split('\\n'):\n",
    "                    if len(name) > 0:\n",
    "                        res.append(name.strip())\n",
    "                       \n",
    "                \n",
    "        if search:\n",
    "            soup = html            \n",
    "            \n",
    "            \n",
    "            r = ''\n",
    "            if 'find' in search.keys():\n",
    "                print('findaing',search['find'])\n",
    "                soup = soup.find(**search['find'])\n",
    "                r = soup\n",
    "\n",
    "                \n",
    "            if 'find_all' in search.keys():\n",
    "                print('findaing all of',search['find_all'])\n",
    "                r = soup.find_all(**search['find_all'])\n",
    "   \n",
    "            if r:\n",
    "                for x in list(r):\n",
    "                    if len(x) > 0:\n",
    "                        res.extend(x)\n",
    "            \n",
    "        return res\n",
    "    \n",
    "def get_tag_elements(url, tag='h2'):\n",
    "    \"\"\"\n",
    "    Downloads a page specified by the url parameter\n",
    "    and returns a list of strings, one per tag element\n",
    "    \"\"\"\n",
    "    \n",
    "    response = simple_get(url)\n",
    "\n",
    "    if response is not None:\n",
    "        html = BeautifulSoup(response, 'html.parser')\n",
    "        names = set()\n",
    "        for li in html.select(tag):\n",
    "            for name in li.text.split('\\n'):\n",
    "                if len(name) > 0:\n",
    "                    names.add(name.strip())\n",
    "        return list(names)\n",
    "\n",
    "    # Raise an exception if we failed to get any data from the url\n",
    "    raise Exception('Error retrieving contents at {}'.format(url)) \n",
    "    \n",
    "    \n",
    "if get_ipython().__class__.__name__ == '__main__':\n",
    "    fire(get_tag_elements)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id=\"sports\"></a>\n",
    "### sports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/sport'\n",
    "sports = get_elements(url, tag = 'a', search = {})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@OfficialKRU</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@Nondies</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@ingweleopards</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@Gor_MahiaFC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@Monks_Rugby</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          handles\n",
       "0    @OfficialKRU\n",
       "1        @Nondies\n",
       "2  @ingweleopards\n",
       "3    @Gor_MahiaFC\n",
       "4    @Monks_Rugby"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sport = []\n",
    "for element in sports:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            sport.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "sporting = []\n",
    "for element in sport:\n",
    "    element = trim_word(element, 1,1)\n",
    "    sporting.append(element)\n",
    "    \n",
    "sporting_df = pd.DataFrame({'handles':sporting})\n",
    "sporting_df.to_csv('sporting_df.csv')\n",
    "sporting_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Society"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@StrathU</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@CUEA_OFFICIAL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@DaystarUni</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@moringaschool</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@Maseno_Uni</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          handles\n",
       "0        @StrathU\n",
       "1  @CUEA_OFFICIAL\n",
       "2     @DaystarUni\n",
       "3  @moringaschool\n",
       "4     @Maseno_Uni"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/society/education'\n",
    "edu = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "education = []\n",
    "for element in edu:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            education.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "educ = []\n",
    "for element in education:\n",
    "    element = trim_word(element, 1,1)\n",
    "    educ.append(element)\n",
    "    \n",
    "edu_df = pd.DataFrame({'handles':educ})\n",
    "edu_df.to_csv('edu_df.csv')\n",
    "edu_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@StateHouseKenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@IEBCKenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@NPSOfficial_KE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@InteriorKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@USEmbassyKenya</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            handles\n",
       "0  @StateHouseKenya\n",
       "1        @IEBCKenya\n",
       "2   @NPSOfficial_KE\n",
       "3       @InteriorKE\n",
       "4   @USEmbassyKenya"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# gov\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/society/governmental/page-1-3'\n",
    "gov = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "g = []\n",
    "for element in gov:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            g.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "gv = []\n",
    "for element in g:\n",
    "    element = trim_word(element, 1,1)\n",
    "    gv.append(element)\n",
    "    \n",
    "gov_df = pd.DataFrame({'handles':gv})\n",
    "gov_df.to_csv('gov_df.csv')\n",
    "gov_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@TheAGRF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@KenyaHomesExpo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@SMWNairobi</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           handles\n",
       "0         @TheAGRF\n",
       "1  @KenyaHomesExpo\n",
       "2      @SMWNairobi"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#conferences\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/society/conference'\n",
    "conf = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "c = []\n",
    "for element in conf:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            c.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "cf = []\n",
    "for element in c:\n",
    "    element = trim_word(element, 1,1)\n",
    "    cf.append(element)\n",
    "    \n",
    "conf_df = pd.DataFrame({'handles':cf})\n",
    "conf_df.to_csv('conf_df.csv')\n",
    "conf_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@WilliamsRuto</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@RailaOdinga</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@MikeSonko</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@MarthaKarua</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@KideroEvans</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         handles\n",
       "0  @WilliamsRuto\n",
       "1   @RailaOdinga\n",
       "2     @MikeSonko\n",
       "3   @MarthaKarua\n",
       "4   @KideroEvans"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# politics\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/society/politics'\n",
    "politics = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in politics:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "politics_df = pd.DataFrame({'handles':pl})\n",
    "politics_df.to_csv('politics_df.csv')\n",
    "politics_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@KenyaRedCross</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@SheldrickTrust</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@MediaCouncilK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@UNHABITAT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@thekhrc</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           handles\n",
       "0   @KenyaRedCross\n",
       "1  @SheldrickTrust\n",
       "2   @MediaCouncilK\n",
       "3       @UNHABITAT\n",
       "4         @thekhrc"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ngo\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/society/ngo'\n",
    "ngo = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in ngo:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "ngo_df = pd.DataFrame({'handles':pl})\n",
    "ngo_df.to_csv('ngo_df.csv')\n",
    "ngo_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@PROSAK_Kenya</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         handles\n",
       "0  @PROSAK_Kenya"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prof\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/society/professional-association'\n",
    "prof = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in prof:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "prof_df = pd.DataFrame({'handles':pl})\n",
    "prof_df.to_csv('ngo_df.csv')\n",
    "prof_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Place"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@KResearcher</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@magicalkenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@kot</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@MakeItKenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@KenyaBuzz</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         handles\n",
       "0   @KResearcher\n",
       "1  @magicalkenya\n",
       "2           @kot\n",
       "3   @MakeItKenya\n",
       "4     @KenyaBuzz"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# country\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/place/country'\n",
    "country = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in country:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "country_df = pd.DataFrame({'handles':pl})\n",
    "country_df.to_csv('country_df.csv')\n",
    "country_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@nhifkenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@mmtckenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@thenairobihosp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@AAR_Healthcare</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@GertrudesHosp</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           handles\n",
       "0       @nhifkenya\n",
       "1       @mmtckenya\n",
       "2  @thenairobihosp\n",
       "3  @AAR_Healthcare\n",
       "4   @GertrudesHosp"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# medical centre\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/place/medical-center'\n",
    "med = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "\n",
    "p = []\n",
    "for element in med:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "med_df = pd.DataFrame({'handles':pl})\n",
    "med_df.to_csv('med_df.csv')\n",
    "med_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Restaurants and cafes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@urban_kichen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@Quepasakaren</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         handles\n",
       "0  @urban_kichen\n",
       "1  @Quepasakaren"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cafe\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/place/restaurant-cafe'\n",
    "cafe = get_elements(url, tag = 'a', search = {})\n",
    "# cafe\n",
    "\n",
    "p = []\n",
    "for element in cafe:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "cafe_df = pd.DataFrame({'handles':pl})\n",
    "cafe_df.to_csv('cafe_df.csv')\n",
    "cafe_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Media"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@dailynation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@StandardKenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@TheStarKenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@BD_Africa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@Kenyans</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          handles\n",
       "0    @dailynation\n",
       "1  @StandardKenya\n",
       "2   @TheStarKenya\n",
       "3      @BD_Africa\n",
       "4        @Kenyans"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# daily news\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/media/daily-news'\n",
    "daily = get_elements(url, tag = 'a', search = {})\n",
    "# daily\n",
    "\n",
    "\n",
    "p = []\n",
    "for element in daily:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "daily_df = pd.DataFrame({'handles':pl})\n",
    "daily_df.to_csv('daily_df.csv')\n",
    "daily_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@NationMediaGrp</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           handles\n",
       "0  @NationMediaGrp"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# media house\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/media/media-house'\n",
    "media = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in media:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "media_df = pd.DataFrame({'handles':pl})\n",
    "media_df.to_csv('media_df.csv')\n",
    "media_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@CapitalFMKenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@radiomaisha</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@Kiss100kenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@HomeboyzRadio</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@Lit360KE_</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           handles\n",
       "0  @CapitalFMKenya\n",
       "1     @radiomaisha\n",
       "2    @Kiss100kenya\n",
       "3   @HomeboyzRadio\n",
       "4       @Lit360KE_"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# radio media\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/media/radio-media'\n",
    "radio = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in radio:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "radio_df = pd.DataFrame({'handles':pl})\n",
    "radio_df.to_csv('radio_df.csv')\n",
    "radio_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@GameYetu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@SportpesaNews</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@DN_Football</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@Nation_Sport</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          handles\n",
       "0       @GameYetu\n",
       "1  @SportpesaNews\n",
       "2    @DN_Football\n",
       "3   @Nation_Sport"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sportsMedia\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/media/sports-media'\n",
    "sportsM = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in sportsM:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "sportsM_df = pd.DataFrame({'handles':pl})\n",
    "sportsM_df.to_csv('sportsM_df.csv')\n",
    "sportsM_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Entertainment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@Ma3Route</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@EasyTaxiKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@OChargeKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@olizaKE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       handles\n",
       "0    @Ma3Route\n",
       "1  @EasyTaxiKE\n",
       "2   @OChargeKE\n",
       "3     @olizaKE"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# apps\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/entertainment/apps'\n",
    "apps = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in apps:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "apps_df = pd.DataFrame({'handles':pl})\n",
    "apps_df.to_csv('apps_df.csv')\n",
    "apps_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@KBCChannel1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@theTrendLive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@AMLiveNTV</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@NTVSasa</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         handles\n",
       "0   @KBCChannel1\n",
       "1  @theTrendLive\n",
       "2     @AMLiveNTV\n",
       "3       @NTVSasa"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# shows\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/entertainment/broadcast-show'\n",
    "shows = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in shows:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "shows_df = pd.DataFrame({'handles':pl})\n",
    "shows_df.to_csv('shows_df.csv')\n",
    "shows_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Community"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@tafaqari</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     handles\n",
       "0  @tafaqari"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fun\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/community/fun'\n",
    "fun = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in fun:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "fun_df = pd.DataFrame({'handles':pl})\n",
    "fun_df.to_csv('fun_df.csv')\n",
    "fun_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Zuku_WeCar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Zuku_WeCar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Zuku_WeCar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Zuku_WeCar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@AfricanProverbs</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            handles\n",
       "0        Zuku_WeCar\n",
       "1        Zuku_WeCar\n",
       "2        Zuku_WeCar\n",
       "3        Zuku_WeCar\n",
       "4  @AfricanProverbs"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# hobbies\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/community/hobbies'\n",
    "hobbies = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in hobbies:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "hobbies_df = pd.DataFrame({'handles':pl})\n",
    "hobbies_df.to_csv('hobbies_df.csv')\n",
    "hobbies_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@xtiandela</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@ItsJacksonKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@EveWomanKenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@6lvcsupreme</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@lizmarami</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          handles\n",
       "0      @xtiandela\n",
       "1   @ItsJacksonKE\n",
       "2  @EveWomanKenya\n",
       "3    @6lvcsupreme\n",
       "4      @lizmarami"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# life style\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/community/life-style'\n",
    "life = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in life:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "life_df = pd.DataFrame({'handles':pl})\n",
    "life_df.to_csv('life_df.csv')\n",
    "life_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Celebrities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@MwalimChurchill</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@iamedigathegi</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            handles\n",
       "0  @MwalimChurchill\n",
       "1    @iamedigathegi"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# actors\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/celebrities/actor'\n",
    "actor = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in actor:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "actor_df = pd.DataFrame({'handles':pl})\n",
    "actor_df.to_csv('actor_df.csv')\n",
    "actor_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@nhifkenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@mmtckenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@thenairobihosp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@AAR_Healthcare</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@GertrudesHosp</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           handles\n",
       "0       @nhifkenya\n",
       "1       @mmtckenya\n",
       "2  @thenairobihosp\n",
       "3  @AAR_Healthcare\n",
       "4   @GertrudesHosp"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# broadcasts\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/celebrities/broadcast-star'\n",
    "broadcast = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in med:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "med_df = pd.DataFrame({'handles':pl})\n",
    "med_df.to_csv('med_df.csv')\n",
    "med_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@DJSADIC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@DeejayCeleb</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        handles\n",
       "0      @DJSADIC\n",
       "1  @DeejayCeleb"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dj\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/celebrities/disc-jockey'\n",
    "dj = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in dj:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "dj_df = pd.DataFrame({'handles':pl})\n",
    "dj_df.to_csv('dj_df.csv')\n",
    "dj_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@sautisol</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@BahatiKenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@H_ARTTHEBAND</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Sailors25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@sailors254</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         handles\n",
       "0      @sautisol\n",
       "1   @BahatiKenya\n",
       "2  @H_ARTTHEBAND\n",
       "3      Sailors25\n",
       "4    @sailors254"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# musician\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/celebrities/musician'\n",
    "musician = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in musician:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "musician_df = pd.DataFrame({'handles':pl})\n",
    "musician_df.to_csv('musician_df.csv')\n",
    "musician_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@RabbitTheKing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@Sanaipei_Tande</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@KarunMusic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@DynoxxPetrah</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@Kamnao</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           handles\n",
       "0   @RabbitTheKing\n",
       "1  @Sanaipei_Tande\n",
       "2      @KarunMusic\n",
       "3    @DynoxxPetrah\n",
       "4          @Kamnao"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# singer\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/celebrities/singer'\n",
    "singer = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in singer:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "singer_df = pd.DataFrame({'handles':pl})\n",
    "singer_df.to_csv('singer_df.csv')\n",
    "singer_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Brands"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@serenahotels</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@SarovaHotelsKen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@SarovaStanley</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@HeritageKenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@VillaRosaKempin</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            handles\n",
       "0     @serenahotels\n",
       "1  @SarovaHotelsKen\n",
       "2    @SarovaStanley\n",
       "3    @HeritageKenya\n",
       "4  @VillaRosaKempin"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# accom\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/brands/accommodation'\n",
    "accom = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in accom:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "accom_df = pd.DataFrame({'handles':pl})\n",
    "accom_df.to_csv('accom_df.csv')\n",
    "accom_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@KenyaAirways</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@FlyJambojet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@Flysafarilink</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@silverstoneair</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@KLM_KE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           handles\n",
       "0    @KenyaAirways\n",
       "1     @FlyJambojet\n",
       "2   @Flysafarilink\n",
       "3  @silverstoneair\n",
       "4          @KLM_KE"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# airlines\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/brands/airlines'\n",
    "airlines = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in airlines:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "air_df = pd.DataFrame({'handles':pl})\n",
    "air_df.to_csv('air_df.csv')\n",
    "air_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@TuskerCider</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@TuskerLager</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@GuinnessKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@KenyaCaneSpirit</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@TuskerMaltLager</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            handles\n",
       "0      @TuskerCider\n",
       "1      @TuskerLager\n",
       "2       @GuinnessKE\n",
       "3  @KenyaCaneSpirit\n",
       "4  @TuskerMaltLager"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# alcohol\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/brands/alcohol'\n",
    "alcohol = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in alcohol:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "alcohol_df = pd.DataFrame({'handles':pl})\n",
    "alcohol_df.to_csv('alcohol_df.csv')\n",
    "alcohol_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@FordinKenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@IsuzuKenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@SubaruKenya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@chlorideexide</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@BMW_Kenya</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          handles\n",
       "0    @FordinKenya\n",
       "1     @IsuzuKenya\n",
       "2    @SubaruKenya\n",
       "3  @chlorideexide\n",
       "4      @BMW_Kenya"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# auto\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/brands/auto'\n",
    "auto = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in auto:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "auto_df = pd.DataFrame({'handles':pl})\n",
    "auto_df.to_csv('auto_df.csv')\n",
    "auto_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@RexonaEA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@NyweleCreative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           handles\n",
       "0        @RexonaEA\n",
       "1  @NyweleCreative"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# beauty\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/brands/beauty'\n",
    "beauty = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in beauty:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "beauty_df = pd.DataFrame({'handles':pl})\n",
    "beauty_df.to_csv('beauty_df.csv')\n",
    "beauty_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>@CocaColaAfrica</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>@NescafeKe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>@KerichoGold</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>@KetepaLtd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>@SpriteKenya</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           handles\n",
       "0  @CocaColaAfrica\n",
       "1       @NescafeKe\n",
       "2     @KerichoGold\n",
       "3       @KetepaLtd\n",
       "4     @SpriteKenya"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# bev\n",
    "url = 'https://www.socialbakers.com/statistics/twitter/profiles/kenya/brands/beverages'\n",
    "bev = get_elements(url, tag = 'a', search = {})\n",
    "\n",
    "p = []\n",
    "for element in bev:\n",
    "    for string in element.split(' '):\n",
    "        if '@' in string:\n",
    "            p.append(string)\n",
    "        \n",
    "def trim_word(word, from_start=0, from_end=0):\n",
    "    return word[from_start:len(word) - from_end]\n",
    "pl = []\n",
    "for element in p:\n",
    "    element = trim_word(element, 1,1)\n",
    "    pl.append(element)\n",
    "    \n",
    "bev_df = pd.DataFrame({'handles':pl})\n",
    "bev_df.to_csv('bev_df.csv')\n",
    "bev_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concating the dataframes to one mega df."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(172, 1)"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mega_dff = pd.concat([accom_df, actor_df, air_df, alcohol_df, apps_df, auto_df, beauty_df, bev_df, cafe_df, conf_df, country_df,\n",
    "                     daily_df, edu_df, events_df, fun_df, gov_df, hobbies_df, life_df, mags_df, med_df, media_df, musician_df, ngo_df,\n",
    "                     politics_df, radio_df, shows_df, singer_df, sporting_df, sportsM_df, dj_df, prof_df])\n",
    "mega_dff = mega_dff.drop_duplicates(subset = ['handles'])\n",
    "mega_dff.to_csv('mega1.csv')\n",
    "mega_dff.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [Back to top](#Web-scraping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from selenium import webdriver\n",
    "# from selenium import webdriver\n",
    "# from selenium.webdriver.firefox.options import Options as FirefoxOptions\n",
    "\n",
    "# driver = webdriver.Firefox(\"/home/ada/10academy/training /week2/monday/kernels/geckodriver\")\n",
    "\n",
    "# import time\n",
    "\n",
    "# driver.get(\"https://www.socialbakers.com/statistics/twitter/profiles/kenya/society\")\n",
    "# more_buttons = driver.find_elements_by_class_name(\"show-more-button btn-nw btn-nw--lg btn-nw--invisible \")\n",
    "# for x in range(len(more_buttons)):\n",
    "#     if more_buttons[x].is_displayed():\n",
    "#         driver.execute_script(\"arguments[0].click();\", more_buttons[x])\n",
    "#         time.sleep(1)\n",
    "        \n",
    "# page_source = driver.page_source\n",
    "\n",
    "# from bs4 import BeautifulSoup\n",
    "\n",
    "# soup = BeautifulSoup(page_source, 'lxml')\n",
    "# reviews = []\n",
    "# reviews_selector = soup.find_all('a')\n",
    "# # for review_selector in reviews_selector:\n",
    "# #     review_div = review_selector.find('div', class_='dyn_full_review')\n",
    "# #     if review_div is None:\n",
    "# #         review_div = review_selector.find('div', class_='basic_review')\n",
    "# #     review = review_div.find('div', class_='entry').find('p').get_text()\n",
    "# #     review = review.strip()\n",
    "# #     reviews.append(review)\n",
    "# reviews\n",
    "\n",
    "\n",
    "\n",
    "# def configure_firefox_driver():\n",
    "#     # Add additional Options to the webdriver\n",
    "#     firefox_options = FirefoxOptions()\n",
    "#     # add the argument and make the browser Headless.\n",
    "#     firefox_options.add_argument(\"--headless\")\n",
    "\n",
    "#     # Instantiate the Webdriver: Mention the executable path of the webdriver you have downloaded\n",
    "#     # if driver is in PATH, no need to provide executable_path\n",
    "#     driver = webdriver.Firefox(executable_path = \"./geckodriver\", options = firefox_options)\n",
    "#     return driver\n",
    "\n",
    "\n",
    "# dr = configure_firefox_driver()\n",
    "\n",
    "\n",
    "\n",
    "# from selenium.common.exceptions import TimeoutException, NoSuchElementException\n",
    "# from selenium.webdriver.common.by import By\n",
    "# from selenium.webdriver.support.ui import WebDriverWait\n",
    "# from selenium.webdriver.support import expected_conditions as EC\n",
    "\n",
    "\n",
    "# def get_field_text_if_exists(item, selector):\n",
    "#     \"\"\"Extracts a field by a CSS selector if exists.\"\"\"\n",
    "#     try:\n",
    "#         return item.find_element_by_css_selector(selector).text\n",
    "#     except NoSuchElementException:\n",
    "#         return \"\"\n",
    "\n",
    "\n",
    "# def get_link_if_exists(item, selector):\n",
    "#     \"\"\"Extracts an href attribute value by a CSS selector if exists.\"\"\"\n",
    "#     try:\n",
    "#         return item.find_element_by_css_selector(selector).get_attribute(\"href\")\n",
    "#     except NoSuchElementException:\n",
    "#         return \"\"\n",
    "\n",
    "\n",
    "# wait = WebDriverWait(dr, 10)\n",
    "\n",
    "# dr.get(\"https://www.socialbakers.com/statistics/twitter/profiles/kenya/society\")\n",
    "\n",
    "\n",
    "# soup = BeautifulSoup(dr.page_source,'html.parser')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### [Back to top](#Web-scraping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}