02_Scrape_info_for_each_condo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import packages\n",
    "\n",
    "from datetime import datetime\n",
    "import time\n",
    "import pandas as pd\n",
    "import pickle as pk\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# open the output text file from '01_Scrape_All_Links_gh.ipynb'\n",
    "\n",
    "with open('condo_links_all.txt') as f:\n",
    "    condo_links_all = f.read().splitlines()\n",
    "condo_links_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write function to retrive info, using BeautifulSoup package.\n",
    "# This process took some time to carefully extract the info you needed from the soup.\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "\n",
    "def retrieve(link):\n",
    "    page = requests.get(link)\n",
    "    print(link)\n",
    "    soup = BeautifulSoup(page.content, 'html.parser')\n",
    "    \n",
    "    graph_data=soup.find(id=\"graph1\").get_text().strip()\n",
    "    \n",
    "    # If there is no price chart in the page, return 'None'\n",
    "    if graph_data != \"Not enough data to build the graph\":\n",
    "        name=soup.find(itemprop=\"name\").get_text().strip()\n",
    "        district=soup.find(itemprop=\"streetAddress\").get_text().strip().split(\",\")[-1].strip()\n",
    "        latitude=str(soup.find(itemprop=\"latitude\")).split(\"=\")[1].split()[0]\n",
    "        longitude=str(soup.find(itemprop=\"longitude\")).split(\"=\")[1].split()[0]\n",
    "    \n",
    "        description=str(soup.find(class_=\"property-description__content\"))\n",
    "        year_built=description.split(\". Condominium\")[0].split()[-1]\n",
    "        buildings=description.split(\"building\")[0].split()[-1]\n",
    "        floors=description.split(\"floors\")[0].split()[-1]\n",
    "        units=description.split(\"units\")[0].split()[-1]\n",
    "        print(name,district,latitude,longitude,\"\\n\",year_built,buildings,floors,units)\n",
    "        \n",
    "        neighborhood=[]\n",
    "        for i in range(0,15):\n",
    "            x=soup.find(class_=\"property-description__content\").findAll('li')[i].get_text()\n",
    "            neighborhood.append(x)\n",
    "        \n",
    "        shops=neighborhood[0:5]\n",
    "        #for x in shops: print(x)\n",
    "            \n",
    "        schools=neighborhood[5:10]\n",
    "        #for x in schools: print(x)\n",
    "        \n",
    "        restaurants=neighborhood[10:15]\n",
    "        #for x in restaurants: print(x)\n",
    "        \n",
    "        hospital=soup.find(class_=\"property-description__content\").findAll('p')[-3].get_text()\n",
    "        #print(hospital)\n",
    "        \n",
    "        # Amenities section\n",
    "        # Elevator,Parking,Security,CCTV,Pool,Sauna,Gym,Garden,Playground,Shop,Restaurant,Wifi\n",
    "        amenities=[]\n",
    "        for i in range(0,12):\n",
    "            if ('yes' in str(soup.find(class_=\"amenities\").findAll('li')[i])):\n",
    "                amenities.append(1)\n",
    "            else:\n",
    "                amenities.append(0)\n",
    "        #print(amenities)\n",
    "        \n",
    "        # Location and Neighborhood\n",
    "        transportation=[]\n",
    "        for i in range(0,5):\n",
    "            tran_type=soup.findAll(class_=\"media neighborhood-destination\")[i].find(class_=\"icon\").i['class'][1]\n",
    "            trans_name=soup.findAll(class_=\"media-heading\")[i].get_text()\n",
    "            trans_dist=soup.findAll(class_=\"media neighborhood-destination\")[i].find('small').get_text()\n",
    "            transportation.append((tran_type,trans_name,trans_dist))\n",
    "            \n",
    "        # Market Stats\n",
    "        price_sqm=soup.find(class_=\"indicator__amount\").find(class_=\"money\").get_text().strip('฿').replace(',',\"\")\n",
    "        change_last_q=soup.findAll(class_=\"indicator__amount\")[1].get_text().replace('\\n',\"\").strip()\n",
    "        change_last_y=soup.findAll(class_=\"indicator__amount\")[2].get_text().replace('\\n',\"\").strip()\n",
    "        rental_yield=soup.findAll(class_=\"indicator__amount\")[3].get_text().replace('\\n',\"\").strip()\n",
    "        change_last_y_rental_price=soup.findAll(class_=\"indicator__amount\")[4].get_text().replace('\\n',\"\").strip()\n",
    "        #print(price_sqm,change_last_q,change_last_y,rental_yield,change_last_y_rental_price)\n",
    "        \n",
    "        # price history graph\n",
    "        price_hist=soup.find(class_=\"row-fluid background-color-gray project__graph-container\").find('script').get_text().split('\\n')[3].strip().strip(',').replace('data: ',\"\")\n",
    "        #print(price_hist)\n",
    "        \n",
    "        return (name,district,latitude,longitude,year_built,buildings,floors,units,\\\n",
    "           shops,schools,restaurants,hospital,amenities,transportation,\\\n",
    "           price_sqm,change_last_q,change_last_y,rental_yield,change_last_y_rental_price,price_hist)\n",
    "    else:\n",
    "        print(\"---------Not enough data to build the graph----------\",'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run the loop to retrieve data and store data as DataFrame, save as pickle.\n",
    "\n",
    "start_time = datetime.now()\n",
    "\n",
    "condo_list=[]\n",
    "i=0\n",
    "\n",
    "for link in condo_links_all:\n",
    "    try:\n",
    "        condo_list.append(retrieve(link))\n",
    "    except Exception: # Let the codes go if there is any error.\n",
    "        pass\n",
    "    print(i)\n",
    "    time_elapsed = datetime.now() - start_time\n",
    "    print('Time elapsed (hh:mm:ss.ms) {}'.format(time_elapsed))\n",
    "    \n",
    "    ### Give the 'sleep' time. Space out each request so the server isn’t overwhelmed.\n",
    "    time.sleep(5)\n",
    "    i=i+1\n",
    "\n",
    "    # This is the preventive step...\n",
    "    # You can even clear the list and name a new file to save processing memory.\n",
    "    # Dump the data periodically every 5 iterations.\n",
    "    if (i%5==0):\n",
    "        # Delete 'None' elements from the list.\n",
    "        condo_list = [c for c in condo_list if c is not None]\n",
    "        df = pd.DataFrame(condo_list)\n",
    "        with open('df.pkl', 'wb') as f:\n",
    "            pk.dump(df, f)\n",
    "        # Print out i,len(condo_list), so we can trace back if error occur.\n",
    "        # i is the index of 'condo_links_all'\n",
    "        print('------------------------ dump @ i = ',i,len(condo_list))         \n",
    "print(\"completed\")\n",
    "\n",
    "# Once complete, dump to pickle and save as 'df_completed.pkl'.\n",
    "condo_list = [c for c in condo_list if c is not None]\n",
    "df_completed = pd.DataFrame(condo_list)\n",
    "with open('df_completed.pkl', 'wb') as f:\n",
    "    pk.dump(df_completed, f) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read the output file\n",
    "\n",
    "with open('df.pkl','rb') as f:  # Python 3: open(..., 'rb')\n",
    "    (df) = pk.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}