snscrape_tutorial_with_code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1bd243f3-80ae-46c0-99d9-3adbef3ef779",
   "metadata": {},
   "source": [
    "# Scrape Millions of Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "759e3b3e-e63a-4fc3-a2bc-1e4685d10f17",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip3 -qqq install snscrape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3775d3c4-174c-4305-86ce-09ff70b5d35e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import snscrape.modules.twitter as snt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "85565a05-45e7-41d5-a248-346b85e47c98",
   "metadata": {},
   "outputs": [],
   "source": [
    "world_cup_scraper = snt.TwitterSearchScraper(\"#worldcup\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7dd38927-e038-4c8d-9b08-08cbf42e34c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<snscrape.modules.twitter.TwitterSearchScraper at 0x114a25750>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "world_cup_scraper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6e4aea5b-75b7-4fad-98af-da9c04e852ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "for tweet in world_cup_scraper.get_items():\n",
    "    this_tweet = tweet\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d60f6b97-df1c-4dfb-8049-9278d7d19c0b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Tweet(url='https://twitter.com/Amaan_Aqeel10/status/1601549346092482560', date=datetime.datetime(2022, 12, 10, 12, 8, 14, tzinfo=datetime.timezone.utc), content='You won my heart EMI ♥️ \\n\\n#EmilianoMartinez  #WorldCup #Argentina #Messi𓃵 https://t.co/zsC8KhN6Se', renderedContent='You won my heart EMI ♥️ \\n\\n#EmilianoMartinez  #WorldCup #Argentina #Messi𓃵 twitter.com/nocontextfooty…', id=1601549346092482560, user=User(username='Amaan_Aqeel10', id=2594539483, displayname='Rⱥgnⱥr Lo†hbrok ➳ 🇵🇰', description='✧I LOVE N͢͢͢𝓐ṨA✧ 🚀🛰️\\n\\nғɪɢʜᴛᴇʀ🥊 FoͥOtͣbͫaller⚽ Captai͢͢͢П🌎 𝔅𝔦𝔤 𝔇͢͢͢𝔯𝔢𝔞𝔪𝔢𝔯🎲 \\n 𝕿𝖍𝖊 𝕭𝖔𝖔𝖌𝖊𝖞𝖒𝖆𝖓 ☠  Alw͢͢͢αys Angry Ⲙคn❃👊', rawDescription='✧I LOVE N͢͢͢𝓐ṨA✧ 🚀🛰️\\n\\nғɪɢʜᴛᴇʀ🥊 FoͥOtͣbͫaller⚽ Captai͢͢͢П🌎 𝔅𝔦𝔤 𝔇͢͢͢𝔯𝔢𝔞𝔪𝔢𝔯🎲 \\n 𝕿𝖍𝖊 𝕭𝖔𝖔𝖌𝖊𝖞𝖒𝖆𝖓 ☠  Alw͢͢͢αys Angry Ⲙคn❃👊', descriptionUrls=None, verified=False, created=datetime.datetime(2014, 6, 29, 9, 15, 27, tzinfo=datetime.timezone.utc), followersCount=431, friendsCount=555, statusesCount=6754, favouritesCount=15948, listedCount=0, mediaCount=710, location='Multan, Pakistan', protected=False, linkUrl=None, linkTcourl=None, profileImageUrl='https://pbs.twimg.com/profile_images/1531762341892788227/rUrwtW4k_normal.jpg', profileBannerUrl='https://pbs.twimg.com/profile_banners/2594539483/1581606622', label=None), replyCount=0, retweetCount=0, likeCount=0, quoteCount=0, conversationId=1601549346092482560, lang='en', source='<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>', sourceUrl='https://mobile.twitter.com', sourceLabel='Twitter Web App', outlinks=['https://twitter.com/nocontextfooty/status/1601530724657811456'], tcooutlinks=['https://t.co/zsC8KhN6Se'], media=None, retweetedTweet=None, quotedTweet=Tweet(url='https://twitter.com/nocontextfooty/status/1601530724657811456', date=datetime.datetime(2022, 12, 10, 10, 54, 14, tzinfo=datetime.timezone.utc), content='https://t.co/cJaXKcTFg7', renderedContent='https://t.co/cJaXKcTFg7', id=1601530724657811456, user=User(username='nocontextfooty', id=1111976778065723393, displayname='Out Of Context Football', description='DM requests. ⚽️', rawDescription='DM requests. ⚽️', descriptionUrls=None, verified=False, created=datetime.datetime(2019, 3, 30, 13, 1, 38, tzinfo=datetime.timezone.utc), followersCount=2733069, friendsCount=0, statusesCount=10381, favouritesCount=0, listedCount=4685, mediaCount=10452, location='nocontextfooty@gmail.com', protected=False, linkUrl='http://youtube.com/@ncfwscontext', linkTcourl='https://t.co/wyhFNjaRdN', profileImageUrl='https://pbs.twimg.com/profile_images/1112323410837815296/wxhDfc5D_normal.jpg', profileBannerUrl='https://pbs.twimg.com/profile_banners/1111976778065723393/1658467510', label=None), replyCount=95, retweetCount=542, likeCount=9878, quoteCount=141, conversationId=1601530724657811456, lang='zxx', source='<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>', sourceUrl='http://twitter.com/download/iphone', sourceLabel='Twitter for iPhone', outlinks=None, tcooutlinks=None, media=[Video(thumbnailUrl='https://pbs.twimg.com/ext_tw_video_thumb/1601423282170957825/pu/img/iimyOxKENVjCg71K.jpg', variants=[VideoVariant(contentType='video/mp4', url='https://video.twimg.com/ext_tw_video/1601423282170957825/pu/vid/480x270/Nxi_f73pZhRU5BZr.mp4?tag=12', bitrate=256000), VideoVariant(contentType='application/x-mpegURL', url='https://video.twimg.com/ext_tw_video/1601423282170957825/pu/pl/AmLLKWgK4I0ipJa7.m3u8?tag=12&container=fmp4', bitrate=None), VideoVariant(contentType='video/mp4', url='https://video.twimg.com/ext_tw_video/1601423282170957825/pu/vid/1280x720/uucYGop6vrZKTsOt.mp4?tag=12', bitrate=2176000), VideoVariant(contentType='video/mp4', url='https://video.twimg.com/ext_tw_video/1601423282170957825/pu/vid/640x360/BST4W9Cwhsu_a10p.mp4?tag=12', bitrate=832000)], duration=15.53, views=208232)], retweetedTweet=None, quotedTweet=None, inReplyToTweetId=None, inReplyToUser=None, mentionedUsers=None, coordinates=None, place=None, hashtags=None, cashtags=None), inReplyToTweetId=None, inReplyToUser=None, mentionedUsers=None, coordinates=None, place=None, hashtags=['EmilianoMartinez', 'WorldCup', 'Argentina', 'Messi𓃵'], cashtags=None)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "this_tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "0edc5b8e-f2f9-4e98-9401-60f69b06b415",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "column_names = ['url', 'date', 'content', 'username','displayname',\n",
    "                'description', 'followersCount', 'friendsCount',\n",
    "                'likeCount', 'retweetCount']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "84e841a0-e391-4a73-9012-777ef7990b8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Putting all togeter\n",
    "def grab_tweets(total_number):\n",
    "    \n",
    "    \n",
    "    final_tweets = []\n",
    "    \n",
    "    for index, world_cup_tweet in enumerate(world_cup_scraper.get_items()):\n",
    "\n",
    "        user = world_cup_tweet.user\n",
    "\n",
    "        tweet_data = [world_cup_tweet.url, \n",
    "                      world_cup_tweet.date, \n",
    "                      world_cup_tweet.content, \n",
    "                      user.username, \n",
    "                      user.displayname,\n",
    "                      user.description, \n",
    "                      user.followersCount,\n",
    "                      user.friendsCount,\n",
    "                      world_cup_tweet.likeCount, \n",
    "                      world_cup_tweet.retweetCount\n",
    "                      ]\n",
    "\n",
    "        final_tweets.append(tweet_data)\n",
    "\n",
    "        if(index == total_number):\n",
    "            break\n",
    "            \n",
    "        \n",
    "    # Create the dataframe\n",
    "    final_tweets_df = pd.DataFrame(final_tweets, columns = column_names)\n",
    "    \n",
    "    return final_tweets_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b8505a37-0e1c-4104-88b0-c3a78ea2c7b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_tweets_data = grab_tweets(200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "a75a6d85-2335-40fa-8046-02226d64514b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(201, 10)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_tweets_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "844edfe6-accc-4598-b579-472b18f3b9d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>date</th>\n",
       "      <th>content</th>\n",
       "      <th>username</th>\n",
       "      <th>displayname</th>\n",
       "      <th>description</th>\n",
       "      <th>followersCount</th>\n",
       "      <th>friendsCount</th>\n",
       "      <th>likeCount</th>\n",
       "      <th>retweetCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://twitter.com/cryptoadiks/status/1601550...</td>\n",
       "      <td>2022-12-10 12:14:32+00:00</td>\n",
       "      <td>Join us to predict the #WorldCup winners and s...</td>\n",
       "      <td>cryptoadiks</td>\n",
       "      <td>cryptoadiks</td>\n",
       "      <td>roaming the internet for worthy endeavour</td>\n",
       "      <td>2628</td>\n",
       "      <td>3821</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://twitter.com/StanTheAnomaly/status/1601...</td>\n",
       "      <td>2022-12-10 12:14:31+00:00</td>\n",
       "      <td>#Messi𓃵 #FIFAWorldCup #WorldCup #Argentina htt...</td>\n",
       "      <td>StanTheAnomaly</td>\n",
       "      <td>Suvrahadip Ghosh</td>\n",
       "      <td>Bestselling Author of #RUINS. \\nOtaku. Poetry....</td>\n",
       "      <td>45</td>\n",
       "      <td>346</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://twitter.com/Yakan_so_ko_nin/status/160...</td>\n",
       "      <td>2022-12-10 12:14:17+00:00</td>\n",
       "      <td>敗退直後、「アルゼンチンは優勝できると思うか?」との問いにファン・ダイクは…\\nhttps:...</td>\n",
       "      <td>Yakan_so_ko_nin</td>\n",
       "      <td>夜間走行人</td>\n",
       "      <td>野球・サッカー・旅・探索・写真・情報発信・寝不足・？・？・？\\n全ての力を結集し、前に向かっ...</td>\n",
       "      <td>94</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://twitter.com/loirenordfccom/status/1601...</td>\n",
       "      <td>2022-12-10 12:14:17+00:00</td>\n",
       "      <td>Fifa FIFA World Cup on Instagram: « 🌟 Stars sh...</td>\n",
       "      <td>loirenordfccom</td>\n",
       "      <td>loirenordfc.com</td>\n",
       "      <td>loirenordfc.comLOIRENORDFC - Football Club Loi...</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://twitter.com/Elde_fut/status/1601550826...</td>\n",
       "      <td>2022-12-10 12:14:07+00:00</td>\n",
       "      <td>🏆#WorldCup🏆\\n\\nHwang Hee Chan va avoir le droi...</td>\n",
       "      <td>Elde_fut</td>\n",
       "      <td>Eldé #FUT #FIFA23</td>\n",
       "      <td>▶ Actu, News, Leaks #FIFA23 #FUT23 #FUT\\n▶ Ene...</td>\n",
       "      <td>13310</td>\n",
       "      <td>77</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 url  \\\n",
       "0  https://twitter.com/cryptoadiks/status/1601550...   \n",
       "1  https://twitter.com/StanTheAnomaly/status/1601...   \n",
       "2  https://twitter.com/Yakan_so_ko_nin/status/160...   \n",
       "3  https://twitter.com/loirenordfccom/status/1601...   \n",
       "4  https://twitter.com/Elde_fut/status/1601550826...   \n",
       "\n",
       "                       date  \\\n",
       "0 2022-12-10 12:14:32+00:00   \n",
       "1 2022-12-10 12:14:31+00:00   \n",
       "2 2022-12-10 12:14:17+00:00   \n",
       "3 2022-12-10 12:14:17+00:00   \n",
       "4 2022-12-10 12:14:07+00:00   \n",
       "\n",
       "                                             content         username  \\\n",
       "0  Join us to predict the #WorldCup winners and s...      cryptoadiks   \n",
       "1  #Messi𓃵 #FIFAWorldCup #WorldCup #Argentina htt...   StanTheAnomaly   \n",
       "2  敗退直後、「アルゼンチンは優勝できると思うか?」との問いにファン・ダイクは…\\nhttps:...  Yakan_so_ko_nin   \n",
       "3  Fifa FIFA World Cup on Instagram: « 🌟 Stars sh...   loirenordfccom   \n",
       "4  🏆#WorldCup🏆\\n\\nHwang Hee Chan va avoir le droi...         Elde_fut   \n",
       "\n",
       "         displayname                                        description  \\\n",
       "0        cryptoadiks          roaming the internet for worthy endeavour   \n",
       "1   Suvrahadip Ghosh  Bestselling Author of #RUINS. \\nOtaku. Poetry....   \n",
       "2              夜間走行人  野球・サッカー・旅・探索・写真・情報発信・寝不足・？・？・？\\n全ての力を結集し、前に向かっ...   \n",
       "3    loirenordfc.com  loirenordfc.comLOIRENORDFC - Football Club Loi...   \n",
       "4  Eldé #FUT #FIFA23  ▶ Actu, News, Leaks #FIFA23 #FUT23 #FUT\\n▶ Ene...   \n",
       "\n",
       "   followersCount  friendsCount  likeCount  retweetCount  \n",
       "0            2628          3821          0             0  \n",
       "1              45           346          0             0  \n",
       "2              94             0          0             0  \n",
       "3               0            20          0             0  \n",
       "4           13310            77          2             0  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_tweets_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87ebbc51-7006-485f-9256-cdcd10ca1ecd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}