Coursera Capstone- Toronto Assignment Part 2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<H1> Coursera Capstone- Toronto Neighbourhood Assignment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook is in response to Coursera Capstone Week 3 assignment. It will utilize Foursquare API and newly acquired skills from the course."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting package metadata (repodata.json): ...working... done\n",
      "Solving environment: ...working... done\n",
      "\n",
      "# All requested packages already installed.\n",
      "\n",
      "Libraries imported.\n"
     ]
    }
   ],
   "source": [
    "#getting the imports out of the way\n",
    "import numpy as np \n",
    "\n",
    "import pandas as pd \n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)\n",
    "\n",
    "import json\n",
    "\n",
    "!conda install -c conda-forge geopy --yes\n",
    "from geopy.geocoders import Nominatim \n",
    "\n",
    "import requests \n",
    "from pandas.io.json import json_normalize\n",
    "\n",
    "import matplotlib.cm as cm\n",
    "import matplotlib.colors as colors\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.datasets.samples_generator import make_blobs\n",
    "\n",
    "#!conda install -c conda-forge folium=0.5.0 --yes\n",
    "import folium\n",
    "\n",
    "#!conda install -c anaconda beautifulsoup4\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import lxml\n",
    "print('Libraries imported.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<H1> Scraping Website"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that the libraries are imported, we can begin scraping the website."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Postcode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighbourhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M3A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Parkwoods</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M4A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Victoria Village</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M5A</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Regent Park, Harbourfront</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M6A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Lawrence Heights, Lawrence Manor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M7A</td>\n",
       "      <td>Queen's Park (Toronto)</td>\n",
       "      <td>Queen's Park</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Postcode                 Borough                     Neighbourhood\n",
       "0      M3A              North York                         Parkwoods\n",
       "1      M4A              North York                  Victoria Village\n",
       "2      M5A        Downtown Toronto        Regent Park, Harbourfront \n",
       "3      M6A              North York  Lawrence Heights, Lawrence Manor\n",
       "4      M7A  Queen's Park (Toronto)                     Queen's Park "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#linking and scraping website\n",
    "r= requests.get('https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M')\n",
    "soup = BeautifulSoup(r.text, 'html.parser')\n",
    "table= soup.find('table', attrs={'class':'wikitable sortable'})\n",
    "\n",
    "headers=table.findAll('th')\n",
    "for i, head in enumerate(headers): headers[i]=str(headers[i]).replace(\"<th>\",\"\").replace(\"</th>\",\"\").replace(\"\\n\",\"\")\n",
    "\n",
    "rows=table.findAll('tr')\n",
    "rows=rows[1:len(rows)]\n",
    "\n",
    "for i, row in enumerate(rows): rows[i] = str(rows[i]).replace(\"\\n</td></tr>\",\"\").replace(\"<tr>\\n<td>\",\"\")\n",
    "for i, row in enumerate(rows): rows[i] = str(rows[i]).replace(\"<https://>\", \"\")\n",
    "\n",
    "#Creating the Data Frame\n",
    "df=pd.DataFrame(rows)\n",
    "df[headers] = df[0].str.split(\"</td>\\n<td>\", n = 2, expand = True) \n",
    "df.drop(columns=[0],inplace=True)\n",
    "\n",
    "df=df.drop(df[(df.Borough== \"Not assigned\")].index)\n",
    "df.Neighbourhood.replace(\"Not assigned\", df.Borough, inplace=True)\n",
    "\n",
    "df.Neighbourhood.fillna(df.Borough, inplace=True)\n",
    "df=df.drop_duplicates()\n",
    "\n",
    "df.update(\n",
    "    df.Neighbourhood.loc[\n",
    "        lambda x: x.str.contains('title')\n",
    "    ].str.extract('title=\\\"([^\\\"]*)',expand=False))\n",
    "\n",
    "df.update(\n",
    "    df.Borough.loc[\n",
    "        lambda x: x.str.contains('title')\n",
    "    ].str.extract('title=\\\"([^\\\"]*)',expand=False))\n",
    "\n",
    "df.update(\n",
    "    df.Neighbourhood.loc[\n",
    "        lambda x: x.str.contains('Toronto')\n",
    "    ].str.replace(\", Toronto\",\"\"))\n",
    "df.update(\n",
    "    df.Neighbourhood.loc[\n",
    "        lambda x: x.str.contains('Toronto')\n",
    "    ].str.replace(\"\\(Toronto\\)\",\"\"))\n",
    "\n",
    "Toronto = pd.DataFrame({'Postcode':df.Postcode.unique()})\n",
    "Toronto['Borough']=pd.DataFrame(list(set(df['Borough'].loc[df['Postcode'] == x['Postcode']])) for i, x in Toronto.iterrows())\n",
    "Toronto['Neighbourhood']=pd.Series(list(set(df['Neighbourhood'].loc[df['Postcode'] == x['Postcode']])) for i, x in Toronto.iterrows())\n",
    "Toronto['Neighbourhood']=Toronto['Neighbourhood'].apply(lambda x: ', '.join(x))\n",
    "Toronto.dtypes\n",
    "\n",
    "Toronto.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Postalcode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighbourhood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M1B</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>Islington Avenue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M1B</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Islington Avenue, Rouge, Rouge, Rouge, Malvern</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M1C</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>West Deane Park</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M1C</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>West Deane Park, Highland Creek, Highland Cree...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M1E</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>Old Burnhamthorpe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>M1E</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Old Burnhamthorpe, Guildwood, Guildwood, Guild...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>M1G</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Scarborough, Woburn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>M1H</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Scarborough, Cedarbrae</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>M1J</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Scarborough, Scarborough Village</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>M1K</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Scarborough, East Birchmount Park, East Birchm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>M1L</td>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Scarborough, Clairlea, Clairlea, Clairlea, Gol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>M1M</td>\n",
       "      <td>North York</td>\n",
       "      <td>Humber Summit</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Postalcode      Borough                                      Neighbourhood\n",
       "0         M1B    Etobicoke                                   Islington Avenue\n",
       "1         M1B  Scarborough     Islington Avenue, Rouge, Rouge, Rouge, Malvern\n",
       "2         M1C    Etobicoke                                    West Deane Park\n",
       "3         M1C  Scarborough  West Deane Park, Highland Creek, Highland Cree...\n",
       "4         M1E    Etobicoke                                  Old Burnhamthorpe\n",
       "5         M1E  Scarborough  Old Burnhamthorpe, Guildwood, Guildwood, Guild...\n",
       "6         M1G  Scarborough                                Scarborough, Woburn\n",
       "7         M1H  Scarborough                             Scarborough, Cedarbrae\n",
       "8         M1J  Scarborough                   Scarborough, Scarborough Village\n",
       "9         M1K  Scarborough  Scarborough, East Birchmount Park, East Birchm...\n",
       "10        M1L  Scarborough  Scarborough, Clairlea, Clairlea, Clairlea, Gol...\n",
       "11        M1M   North York                                      Humber Summit"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table_columns=['Postalcode', 'Borough', 'Neighbourhood']\n",
    "toronto = pd.DataFrame(columns = table_columns)\n",
    "content = soup.find('div', class_='mw-parser-output')\n",
    "table= content.table.tbody\n",
    "postcode=0\n",
    "borough=0\n",
    "neighborhood=0\n",
    "\n",
    "for tr in table.find_all('tr'):\n",
    "    i=0\n",
    "    for td in tr.find_all('td'):\n",
    "        if i == 0:\n",
    "            postcode = td.text\n",
    "            i= i+1\n",
    "        elif i==1:\n",
    "            borough = td.text\n",
    "            i=i+1\n",
    "        elif i==2:\n",
    "            neighborhood = td.text.strip('\\n').replace(']','')\n",
    "        toronto = toronto.append({'Postalcode': postcode, 'Borough': borough, 'Neighbourhood':neighborhood}, ignore_index=True)\n",
    "\n",
    "toronto = toronto[toronto.Borough!='Not assigned']\n",
    "toronto = toronto[toronto.Borough!=0]\n",
    "toronto.reset_index(drop=True, inplace=True)\n",
    "i=0\n",
    "for i in range(0,toronto.shape[0]):\n",
    "    if toronto.iloc[i][2] == 'Not assigned':\n",
    "        toronto.iloc[i][2]= toronto.iloc[i][1]\n",
    "        i=i+1\n",
    "df= toronto.groupby(['Postalcode','Borough'])['Neighbourhood'].apply(', '.join).reset_index()\n",
    "df.head(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Postcode</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Neighbourhood</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M3A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Parkwoods</td>\n",
       "      <td>43.753259</td>\n",
       "      <td>-79.329656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M4A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Victoria Village</td>\n",
       "      <td>43.725882</td>\n",
       "      <td>-79.315572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M5A</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Regent Park, Harbourfront</td>\n",
       "      <td>43.654260</td>\n",
       "      <td>-79.360636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M6A</td>\n",
       "      <td>North York</td>\n",
       "      <td>Lawrence Heights, Lawrence Manor</td>\n",
       "      <td>43.718518</td>\n",
       "      <td>-79.464763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M7A</td>\n",
       "      <td>Queen's Park (Toronto)</td>\n",
       "      <td>Queen's Park</td>\n",
       "      <td>43.662301</td>\n",
       "      <td>-79.389494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>M9A</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>Islington Avenue</td>\n",
       "      <td>43.667856</td>\n",
       "      <td>-79.532242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>M1B</td>\n",
       "      <td>Scarborough, Toronto</td>\n",
       "      <td>Malvern, Rouge</td>\n",
       "      <td>43.806686</td>\n",
       "      <td>-79.194353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>M3B</td>\n",
       "      <td>North York</td>\n",
       "      <td>Don Mills North</td>\n",
       "      <td>43.745906</td>\n",
       "      <td>-79.352188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>M4B</td>\n",
       "      <td>East York</td>\n",
       "      <td>Parkview Hill, Woodbine Gardens</td>\n",
       "      <td>43.706397</td>\n",
       "      <td>-79.309937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>M5B</td>\n",
       "      <td>Downtown Toronto</td>\n",
       "      <td>Garden District, Ryerson</td>\n",
       "      <td>43.657162</td>\n",
       "      <td>-79.378937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>M6B</td>\n",
       "      <td>North York</td>\n",
       "      <td>Glencairn</td>\n",
       "      <td>43.709577</td>\n",
       "      <td>-79.445073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>M9B</td>\n",
       "      <td>Etobicoke</td>\n",
       "      <td>Cloverdale, Princess Gardens, Islington, Marti...</td>\n",
       "      <td>43.650943</td>\n",
       "      <td>-79.554724</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Postcode                 Borough  \\\n",
       "0       M3A              North York   \n",
       "1       M4A              North York   \n",
       "2       M5A        Downtown Toronto   \n",
       "3       M6A              North York   \n",
       "4       M7A  Queen's Park (Toronto)   \n",
       "5       M9A               Etobicoke   \n",
       "6       M1B    Scarborough, Toronto   \n",
       "7       M3B              North York   \n",
       "8       M4B               East York   \n",
       "9       M5B        Downtown Toronto   \n",
       "10      M6B              North York   \n",
       "11      M9B               Etobicoke   \n",
       "\n",
       "                                        Neighbourhood   Latitude  Longitude  \n",
       "0                                           Parkwoods  43.753259 -79.329656  \n",
       "1                                    Victoria Village  43.725882 -79.315572  \n",
       "2                          Regent Park, Harbourfront   43.654260 -79.360636  \n",
       "3                    Lawrence Heights, Lawrence Manor  43.718518 -79.464763  \n",
       "4                                       Queen's Park   43.662301 -79.389494  \n",
       "5                                    Islington Avenue  43.667856 -79.532242  \n",
       "6                                      Malvern, Rouge  43.806686 -79.194353  \n",
       "7                                     Don Mills North  43.745906 -79.352188  \n",
       "8                     Parkview Hill, Woodbine Gardens  43.706397 -79.309937  \n",
       "9                            Garden District, Ryerson  43.657162 -79.378937  \n",
       "10                                          Glencairn  43.709577 -79.445073  \n",
       "11  Cloverdale, Princess Gardens, Islington, Marti...  43.650943 -79.554724  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfll= pd.read_csv(\"http://cocl.us/Geospatial_data\")\n",
    "dfll.rename(columns={'Postal Code':'Postcode'}, inplace=True)\n",
    "dfll.set_index(\"Postcode\")\n",
    "Toronto.set_index(\"Postcode\")\n",
    "toronto_data=pd.merge(Toronto, dfll)\n",
    "toronto_data.head(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The geograpical coordinate of Toronto, ON, Canada are 43.653963, -79.387207.\n"
     ]
    }
   ],
   "source": [
    "address = 'Toronto, ON, Canada'\n",
    "\n",
    "geolocator = Nominatim(user_agent=\"Jupyter\")\n",
    "location = geolocator.geocode(address)\n",
    "latitude = location.latitude\n",
    "longitude = location.longitude\n",
    "print('The geograpical coordinate of Toronto, ON, Canada are {}, {}.'.format(latitude, longitude))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}