Git_OLXscrape.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load required libraries\n",
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from bs4 import BeautifulSoup, NavigableString\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from selenium import webdriver\n",
    "import requests\n",
    "import re  \n",
    "from time import sleep\n",
    "from selenium.common.exceptions import NoSuchElementException"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set-up web driver\n",
    "\n",
    "driver = webdriver.PhantomJS()\n",
    "options = Options()\n",
    "options.add_argument('--headless')\n",
    "options.add_argument('--no-sandbox')\n",
    "options.add_argument('--disable-gpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Open browser\n",
    "browser = webdriver.Chrome(ChromeDriverManager().install())\n",
    "browser.implicitly_wait(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Open link https://www.olx.co.id/jakarta-dki_g2000007/indekos_c4833?sorting=desc-creation\n",
    "browser.get(\"https://www.olx.co.id/jakarta-dki_g2000007/indekos_c4833?sorting=desc-creation\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to navigate and click infinite scroll button\n",
    "def run_script():\n",
    "    browser.find_element_by_xpath('//*[@data-aut-id=\"btnLoadMore\"]').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loop to repetively click (and wait 7 seconds in between) the infinite scroll button. \n",
    "# Here I assume the max click as 100\n",
    "\n",
    "count = 0\n",
    "while count < 100:\n",
    "    try:\n",
    "        run_script()\n",
    "        count+=1\n",
    "        sleep(7)\n",
    "    except NoSuchElementException:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load required libraries to read html\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read html file\n",
    "soup = BeautifulSoup(browser.page_source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find selected items of interest\n",
    "\n",
    "i_price = soup.find_all('span',{'class':'_89yzn'})\n",
    "i_name = soup.find_all('span',{'class': '_2tW1I'})\n",
    "i_area = soup.find_all('span', {'class': 'tjgMj'})\n",
    "i_date = soup.find_all('span', {'class': 'zLvFQ'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Store findings in a dataframe\n",
    "df = pd.DataFrame(zip(i_price, i_name, i_area, i_date),\n",
    "                     columns=['i_price', 'i_name', 'i_area', 'i_date'])\n",
    "print(df.head())\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Close and end browser\n",
    "driver.quit()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}