second_data_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import time\n",
    "import csv\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrape_main(link):\n",
    "    '''\n",
    "    a function that request the webpage and store it in response object. then passing lxml parser to parse over the webpage.\n",
    "    \n",
    "    here lxml parser defines the speed to parse the webpage.\n",
    "    \n",
    "    if it throws any errors in using lxml parser. just install it by: \" pip intall lxml \"\n",
    "    '''\n",
    "    \n",
    "    response = requests.get(link)\n",
    "    return(BeautifulSoup(response.text, 'lxml' ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reading first csv file which contains the links\n",
    "\n",
    "df = pd.read_csv(\"first_links.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>source</th>\n",
       "      <th>location</th>\n",
       "      <th>job_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>internshala</td>\n",
       "      <td>work_from_home</td>\n",
       "      <td>https://internshala.com/internship/detail/mobi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>internshala</td>\n",
       "      <td>work_from_home</td>\n",
       "      <td>https://internshala.com/internship/detail/cont...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>internshala</td>\n",
       "      <td>work_from_home</td>\n",
       "      <td>https://internshala.com/internship/detail/ente...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>internshala</td>\n",
       "      <td>work_from_home</td>\n",
       "      <td>https://internshala.com/internship/detail/busi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>internshala</td>\n",
       "      <td>work_from_home</td>\n",
       "      <td>https://internshala.com/internship/detail/grap...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0       source        location  \\\n",
       "0           0  internshala  work_from_home   \n",
       "1           1  internshala  work_from_home   \n",
       "2           2  internshala  work_from_home   \n",
       "3           3  internshala  work_from_home   \n",
       "4           4  internshala  work_from_home   \n",
       "\n",
       "                                            job_link  \n",
       "0  https://internshala.com/internship/detail/mobi...  \n",
       "1  https://internshala.com/internship/detail/cont...  \n",
       "2  https://internshala.com/internship/detail/ente...  \n",
       "3  https://internshala.com/internship/detail/busi...  \n",
       "4  https://internshala.com/internship/detail/grap...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#checking 5 job records\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "row_heading = ['source', 'location', 'job_link', 'job_title','company_name', 'imp_fields','description_headings','description']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "93"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# timestamp as file name by using time library and with prefix as internshala\n",
    "file_name = 'internshala_second_raw_data_' + time.strftime(\"%d_%m_%Y_%H_%M_%S\") + \".csv\" \n",
    "\n",
    "\n",
    "# opening file in write mode and connecting csv writer to file\n",
    "file = open(file_name,'w')\n",
    "writer = csv.writer(file)\n",
    "\n",
    "\n",
    "# initially writing header of csv file\n",
    "writer.writerow(row_heading)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# looping over dataframe link column with index value to give job location and source to new data frame\n",
    "for index,link in enumerate(df.job_link):\n",
    "    # passing  url to scrape on each job link\n",
    "    soup = scrape_main(link)\n",
    "    \n",
    "    # getting job title by finding unique class name in webpage\n",
    "    if soup.find('div',{'class':'heading_4_5 profile'}) == None:\n",
    "        continue\n",
    "    \n",
    "    job_title = soup.find('div',{'class':'heading_4_5 profile'}).text.strip()\n",
    "    \n",
    "    # getting company name by unique class name of div tag\n",
    "    company_name = soup.find('div',{'class':'heading_6 company_name'}).text.strip()\n",
    "    \n",
    "    \n",
    "    '''\n",
    "    # getting important fields in the job posting as list\n",
    "    fields are:\n",
    "    1. start date of joining/mode of vacancy\n",
    "    2. duration of job\n",
    "    3. incentives/stipend\n",
    "    4. last date to apply\n",
    "    5. types of doing job \n",
    "    \n",
    "    \n",
    "    '''\n",
    "    imp_fields = []\n",
    "    for i in soup.find_all('div',{'class':'item_body'}):\n",
    "        imp_fields.append(i.get_text().strip())\n",
    "        \n",
    "    # this list is extracting for further process to do in description to get valuable information.\n",
    "    description_headings = []\n",
    "    for i in soup.find_all('div',{'class':'section_heading heading_5_5'}):\n",
    "        description_headings.append(i.get_text().strip())\n",
    "        \n",
    "    # complete description of job\n",
    "    description = soup.find('div',{'class':'internship_details'}).get_text().strip()\n",
    "    \n",
    "    # writing to the server\n",
    "    writer.writerow([df.source[index], df.location[index], df.job_link[index],job_title,company_name, imp_fields,description_headings, description])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loading extracted csv file to the dataframe\n",
    "new_df = pd.read_csv(file_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>location</th>\n",
       "      <th>job_link</th>\n",
       "      <th>job_title</th>\n",
       "      <th>company_name</th>\n",
       "      <th>imp_fields</th>\n",
       "      <th>description_headings</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6040</th>\n",
       "      <td>internshala</td>\n",
       "      <td>work_from_home</td>\n",
       "      <td>https://internshala.com/internship/detail/ente...</td>\n",
       "      <td>Enterprise Accounts</td>\n",
       "      <td>Dunzo</td>\n",
       "      <td>['Starts\\xa0immediatelyImmediately', '6 Months...</td>\n",
       "      <td>['About Dunzo', 'About the work from home job/...</td>\n",
       "      <td>About Dunzo        \\r\\n\\r\\nhttp://dunzo.in/ \\r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7531</th>\n",
       "      <td>internshala</td>\n",
       "      <td>bangalore</td>\n",
       "      <td>https://internshala.com/internship/detail/web-...</td>\n",
       "      <td>Web Development</td>\n",
       "      <td>SKMTECHNOSOLZ Private Limited</td>\n",
       "      <td>['Starts\\xa0immediatelyImmediately', '2 Months...</td>\n",
       "      <td>['About SKMTECHNOSOLZ Private Limited', 'About...</td>\n",
       "      <td>About SKMTECHNOSOLZ Private Limited        \\r\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1919</th>\n",
       "      <td>internshala</td>\n",
       "      <td>work_from_home</td>\n",
       "      <td>https://internshala.com/internship/details/pro...</td>\n",
       "      <td>Product Management</td>\n",
       "      <td>Forbes Center For Excellence</td>\n",
       "      <td>['Starts\\xa0immediatelyImmediately', '3 Months...</td>\n",
       "      <td>['About Forbes Center For Excellence', 'About ...</td>\n",
       "      <td>About Forbes Center For Excellence        \\r\\n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7351</th>\n",
       "      <td>internshala</td>\n",
       "      <td>bangalore</td>\n",
       "      <td>https://internshala.com/internship/detail/rese...</td>\n",
       "      <td>Research</td>\n",
       "      <td>Unacademy</td>\n",
       "      <td>['Starts\\xa0immediatelyImmediately', '3 Months...</td>\n",
       "      <td>['About Unacademy', 'About the internship', 'W...</td>\n",
       "      <td>About Unacademy        \\r\\n\\r\\nhttps://unacade...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>internshala</td>\n",
       "      <td>work_from_home</td>\n",
       "      <td>https://internshala.com/internship/detail/busi...</td>\n",
       "      <td>Business Development (Sales)</td>\n",
       "      <td>Urban Company</td>\n",
       "      <td>['Starts\\xa0immediatelyImmediately', '1 Month'...</td>\n",
       "      <td>['About Urban Company', 'About the work from h...</td>\n",
       "      <td>About Urban Company        \\r\\n\\r\\nhttp://urba...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           source        location  \\\n",
       "6040  internshala  work_from_home   \n",
       "7531  internshala       bangalore   \n",
       "1919  internshala  work_from_home   \n",
       "7351  internshala       bangalore   \n",
       "83    internshala  work_from_home   \n",
       "\n",
       "                                               job_link  \\\n",
       "6040  https://internshala.com/internship/detail/ente...   \n",
       "7531  https://internshala.com/internship/detail/web-...   \n",
       "1919  https://internshala.com/internship/details/pro...   \n",
       "7351  https://internshala.com/internship/detail/rese...   \n",
       "83    https://internshala.com/internship/detail/busi...   \n",
       "\n",
       "                         job_title                   company_name  \\\n",
       "6040           Enterprise Accounts                          Dunzo   \n",
       "7531               Web Development  SKMTECHNOSOLZ Private Limited   \n",
       "1919            Product Management   Forbes Center For Excellence   \n",
       "7351                      Research                      Unacademy   \n",
       "83    Business Development (Sales)                  Urban Company   \n",
       "\n",
       "                                             imp_fields  \\\n",
       "6040  ['Starts\\xa0immediatelyImmediately', '6 Months...   \n",
       "7531  ['Starts\\xa0immediatelyImmediately', '2 Months...   \n",
       "1919  ['Starts\\xa0immediatelyImmediately', '3 Months...   \n",
       "7351  ['Starts\\xa0immediatelyImmediately', '3 Months...   \n",
       "83    ['Starts\\xa0immediatelyImmediately', '1 Month'...   \n",
       "\n",
       "                                   description_headings  \\\n",
       "6040  ['About Dunzo', 'About the work from home job/...   \n",
       "7531  ['About SKMTECHNOSOLZ Private Limited', 'About...   \n",
       "1919  ['About Forbes Center For Excellence', 'About ...   \n",
       "7351  ['About Unacademy', 'About the internship', 'W...   \n",
       "83    ['About Urban Company', 'About the work from h...   \n",
       "\n",
       "                                            description  \n",
       "6040  About Dunzo        \\r\\n\\r\\nhttp://dunzo.in/ \\r...  \n",
       "7531  About SKMTECHNOSOLZ Private Limited        \\r\\...  \n",
       "1919  About Forbes Center For Excellence        \\r\\n...  \n",
       "7351  About Unacademy        \\r\\n\\r\\nhttps://unacade...  \n",
       "83    About Urban Company        \\r\\n\\r\\nhttp://urba...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "new_df.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8115, 8)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# getting size of csv file\n",
    "new_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### To save it into normal csv file just run below cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df.to_csv('second_data.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### To save file - pipe '|' as delimiter : just run the below cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_df.to_csv('first_pipe_format.csv',sep ='|')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}