Harvey Geo Extractor.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract Geo Tagged Tweets from Harvey Collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json, sys, pprint, os, urllib\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First, go to epic_utils/parallel_get and run: \n",
    "\n",
    "    ruby get_geo.rb --start=2017-08-30 \"2017 Hurricane Harvey\" 24\n",
    "\n",
    "When that finishes, run:\n",
    "\n",
    "    cat tweets* > ~/HurricaneHarvey/new_tweets.jsonl\n",
    "    \n",
    "### Once that is all complete... do this:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found 9000 geotagged of 10935 processed"
     ]
    },
    {
     "data": {
      "text/plain": [
       "9616"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "these_tweets = []\n",
    "count = 0;\n",
    "with open('all_geo.jsonl') as inFile:\n",
    "    for line in inFile:\n",
    "        count += 1\n",
    "        try:\n",
    "            t = json.loads(line.strip())\n",
    "            if 'coordinates' in t:\n",
    "                if t['coordinates']:\n",
    "                    these_tweets.append(t)\n",
    "                    if len(these_tweets)%1000==0:\n",
    "                        sys.stderr.write(\"\\rFound {0} geotagged of {1} processed\".format(len(these_tweets),count))\n",
    "        except:\n",
    "            print(\"FAIL: \")\n",
    "            pprint.pprint(line.strip())\n",
    "            continue\n",
    "len(these_tweets)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 9616 tweets in previous file\n",
      "ids:  ['902941559062118400', '902947612852092928']\n"
     ]
    }
   ],
   "source": [
    "# Read in previous geojson.\n",
    "previous = json.load(open('/data/www/jennings/harvey.geojson','r'))\n",
    "print(\"Found {0} tweets in previous file\".format(len(previous['features'])))\n",
    "existing_ids = [x['properties']['id'] for x in previous['features']]\n",
    "print(\"ids: \", existing_ids[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New Tweets:  0\n"
     ]
    }
   ],
   "source": [
    "new_tweets = []\n",
    "for x in these_tweets:\n",
    "    if x['id_str'] not in existing_ids:\n",
    "        new_tweets.append(x)\n",
    "print(\"New Tweets: \",len(new_tweets))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def extract_hashtags(t):\n",
    "    if 'hashtags' in t['entities']:\n",
    "        tags = ['#'+x['text'].lower() for x in t['entities']['hashtags']]\n",
    "        return tags\n",
    "    else:\n",
    "        return []\n",
    "def extract_media(t):\n",
    "    if 'media' in t['entities']:\n",
    "        media = [x['media_url'] for x in t['entities']['media']]\n",
    "        return media\n",
    "    else:\n",
    "        return []\n",
    "def tweet_to_feature(t):\n",
    "    feat = {\n",
    "        \"type\":\"Feature\",\n",
    "        \"geometry\" : {\"type\" : \"Point\", \"coordinates\" : t['coordinates']['coordinates']},\n",
    "        \"properties\":{\n",
    "            \"created_at\" : t['created_at'],\n",
    "            \"text\" : urllib.parse.quote_plus(t['text']),\n",
    "            \"user\" : t['user']['screen_name'],\n",
    "            \"timestamp\"  : int(pd.Timestamp(t['created_at']).timestamp()),\n",
    "            \"id\"   : t['id_str'],\n",
    "            \"coords\" : t['coordinates']['coordinates']\n",
    "        }\n",
    "    }\n",
    "    for tag in extract_hashtags(t):\n",
    "        feat['properties'][tag] = 1\n",
    "       \n",
    "    media = extract_media(t)\n",
    "    if len(media) > 0:\n",
    "        feat['properties']['media'] = media\n",
    "\n",
    "    return feat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#previous['features'] = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_features = []\n",
    "#for t in new_tweets:\n",
    "for t in these_tweets:    \n",
    "    new_features.append(tweet_to_feature(t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "New Collection tweets:   9616\n"
     ]
    }
   ],
   "source": [
    "collection = {\"type\" : \"FeatureCollection\",\n",
    "              \"features\" : new_features+previous['features']}\n",
    "print(\"New Collection tweets:  \", len(collection['features']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/data/www/jennings/harvey.geojson','w') as outFile:\n",
    "    json.dump(collection, outFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(\"latest.geojsonl\",'w') as oFile:\n",
    "    for f in collection['features']:\n",
    "        oFile.write(json.dumps(f)+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tile it locally\n",
    "os.system(\"tippecanoe -Pf -Z0 -B8 -z18 -o harvey-latest.mbtiles -r1 --drop-fraction-as-needed --named-layer=harvey-tweets:latest.geojsonl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.system(\"/home/anderstj/upload-tiles.js --name=harvey-latest harvey-latest.mbtiles\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "# DataFrame? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#1000yearevent</th>\n",
       "      <th>#11pm</th>\n",
       "      <th>#13lovelylavenderlady</th>\n",
       "      <th>#19h30rts</th>\n",
       "      <th>#1on1</th>\n",
       "      <th>#24kmagicworldtour</th>\n",
       "      <th>#25ago</th>\n",
       "      <th>#26agosto</th>\n",
       "      <th>#2esport</th>\n",
       "      <th>#2k17hurricaneharvey</th>\n",
       "      <th>...</th>\n",
       "      <th>#안전요원</th>\n",
       "      <th>#주황</th>\n",
       "      <th>#허리케인하비</th>\n",
       "      <th>coords</th>\n",
       "      <th>created_at</th>\n",
       "      <th>id</th>\n",
       "      <th>media</th>\n",
       "      <th>text</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>user</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[-97.396381, 27.8005828]</td>\n",
       "      <td>Wed Aug 30 17:10:18 +0000 2017</td>\n",
       "      <td>902941559062118400</td>\n",
       "      <td>NaN</td>\n",
       "      <td>See our latest #CorpusChristi, TX #job and cli...</td>\n",
       "      <td>1504113018</td>\n",
       "      <td>tmj_TX_transp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[-97.19439697, 33.00699997]</td>\n",
       "      <td>Wed Aug 30 17:34:21 +0000 2017</td>\n",
       "      <td>902947612852092928</td>\n",
       "      <td>[http://pbs.twimg.com/media/DIfqEXbW4AA_qpZ.jpg]</td>\n",
       "      <td>Hurricane Harvey Leaves Pets Homeless https://...</td>\n",
       "      <td>1504114461</td>\n",
       "      <td>BLifeWestlake</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[-96.62439728, 33.01779938]</td>\n",
       "      <td>Wed Aug 30 17:39:27 +0000 2017</td>\n",
       "      <td>902948897752002564</td>\n",
       "      <td>[http://pbs.twimg.com/media/DIfrPFuWsAMm-tB.jpg]</td>\n",
       "      <td>Hurricane Harvey Leaves Pets Homeless https://...</td>\n",
       "      <td>1504114767</td>\n",
       "      <td>BLifeMurphy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[4.4768, 50.501]</td>\n",
       "      <td>Wed Aug 30 17:40:04 +0000 2017</td>\n",
       "      <td>902949052202983424</td>\n",
       "      <td>NaN</td>\n",
       "      <td>⒈ #kernuitstap\\n⒉ #USOpen\\n⒊ #Harvey\\n⒋ Housto...</td>\n",
       "      <td>1504114804</td>\n",
       "      <td>trendinaliaBE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[-96.865092, 32.984314]</td>\n",
       "      <td>Wed Aug 30 17:47:52 +0000 2017</td>\n",
       "      <td>902951015259017216</td>\n",
       "      <td>NaN</td>\n",
       "      <td>For all those affected in the wake of Hurrican...</td>\n",
       "      <td>1504115272</td>\n",
       "      <td>WCTDRE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2616 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   #1000yearevent  #11pm  #13lovelylavenderlady  #19h30rts  #1on1  \\\n",
       "0             NaN    NaN                    NaN        NaN    NaN   \n",
       "1             NaN    NaN                    NaN        NaN    NaN   \n",
       "2             NaN    NaN                    NaN        NaN    NaN   \n",
       "3             NaN    NaN                    NaN        NaN    NaN   \n",
       "4             NaN    NaN                    NaN        NaN    NaN   \n",
       "\n",
       "   #24kmagicworldtour  #25ago  #26agosto  #2esport  #2k17hurricaneharvey  \\\n",
       "0                 NaN     NaN        NaN       NaN                   NaN   \n",
       "1                 NaN     NaN        NaN       NaN                   NaN   \n",
       "2                 NaN     NaN        NaN       NaN                   NaN   \n",
       "3                 NaN     NaN        NaN       NaN                   NaN   \n",
       "4                 NaN     NaN        NaN       NaN                   NaN   \n",
       "\n",
       "       ...        #안전요원  #주황  #허리케인하비                       coords  \\\n",
       "0      ...          NaN  NaN      NaN     [-97.396381, 27.8005828]   \n",
       "1      ...          NaN  NaN      NaN  [-97.19439697, 33.00699997]   \n",
       "2      ...          NaN  NaN      NaN  [-96.62439728, 33.01779938]   \n",
       "3      ...          NaN  NaN      NaN             [4.4768, 50.501]   \n",
       "4      ...          NaN  NaN      NaN      [-96.865092, 32.984314]   \n",
       "\n",
       "                       created_at                  id  \\\n",
       "0  Wed Aug 30 17:10:18 +0000 2017  902941559062118400   \n",
       "1  Wed Aug 30 17:34:21 +0000 2017  902947612852092928   \n",
       "2  Wed Aug 30 17:39:27 +0000 2017  902948897752002564   \n",
       "3  Wed Aug 30 17:40:04 +0000 2017  902949052202983424   \n",
       "4  Wed Aug 30 17:47:52 +0000 2017  902951015259017216   \n",
       "\n",
       "                                              media  \\\n",
       "0                                               NaN   \n",
       "1  [http://pbs.twimg.com/media/DIfqEXbW4AA_qpZ.jpg]   \n",
       "2  [http://pbs.twimg.com/media/DIfrPFuWsAMm-tB.jpg]   \n",
       "3                                               NaN   \n",
       "4                                               NaN   \n",
       "\n",
       "                                                text   timestamp  \\\n",
       "0  See our latest #CorpusChristi, TX #job and cli...  1504113018   \n",
       "1  Hurricane Harvey Leaves Pets Homeless https://...  1504114461   \n",
       "2  Hurricane Harvey Leaves Pets Homeless https://...  1504114767   \n",
       "3  ⒈ #kernuitstap\\n⒉ #USOpen\\n⒊ #Harvey\\n⒋ Housto...  1504114804   \n",
       "4  For all those affected in the wake of Hurrican...  1504115272   \n",
       "\n",
       "            user  \n",
       "0  tmj_TX_transp  \n",
       "1  BLifeWestlake  \n",
       "2    BLifeMurphy  \n",
       "3  trendinaliaBE  \n",
       "4         WCTDRE  \n",
       "\n",
       "[5 rows x 2616 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame([x['properties'] for x in collection['features']])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9616"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_csv('/data/www/jennings/harvey_geo_tweets.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}