Web Scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accounting and Finance \n",
      "BSc (Hons)\n",
      "Cass Business School\n",
      "3 years\n",
      "NN43 / N4N3 with placement\n",
      "-------------\n",
      "\n",
      "Actuarial Science \n",
      "BSc (Hons)\n",
      "Cass Business School\n",
      "3 years\n",
      "G322 / G321 with professional placement\n",
      "-------------\n",
      "\n",
      "Adult Nursing \n",
      "BSc (Hons)\n",
      "School of Health Sciences\n",
      "Full-time: three years\n",
      "B701\n",
      "-------------\n",
      "\n",
      "Aeronautical Engineering \n",
      "BEng (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "3 years\n",
      "H410\n",
      "-------------\n",
      "\n",
      "Aeronautical Engineering \n",
      "MEng (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "4 years or 5 years including a&nbsp;professional placement. \n",
      "H403 MEng (Hons), H405 MEng (Hons) with professional placement.\n",
      "-------------\n",
      "\n",
      "Banking and International Finance \n",
      "BSc (Hons)\n",
      "Cass Business School\n",
      "3 years\n",
      "N302 / N312 with placement\n",
      "-------------\n",
      "\n",
      "Biomedical Engineering \n",
      "MEng (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "4 years or 5 years including a professional placement\n",
      "BH82; BH11 with professional placement\n",
      "-------------\n",
      "\n",
      "Biomedical Engineering \n",
      "BEng (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "3 years\n",
      "BH81\n",
      "-------------\n",
      "\n",
      "Business Management \n",
      "BSc (Hons)\n",
      "Cass Business School\n",
      "3 years\n",
      "N102 / N121 with professional placement\n",
      "-------------\n",
      "\n",
      "Business Management, Digital Innovation and Entrepreneurship \n",
      "BSc (Hons)\n",
      "Cass Business School\n",
      "3 years\n",
      "N103 / N113 with professional placement\n",
      "-------------\n",
      "\n",
      "Business with Finance \n",
      "BSc (Hons)\n",
      "Cass Business School\n",
      "3 years\n",
      "N1N3 / N131 with professional placement\n",
      "-------------\n",
      "\n",
      "Business with Marketing \n",
      "BSc (Hons)\n",
      "Cass Business School\n",
      "3 years\n",
      "N1N5 / N151 with professional placement\n",
      "-------------\n",
      "\n",
      "Child Nursing \n",
      "BSc (Hons)\n",
      "School of Health Sciences\n",
      "Full-time: three years\n",
      "B703\n",
      "-------------\n",
      "\n",
      "Civil Engineering \n",
      "BEng (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "3 years\n",
      "H200\n",
      "-------------\n",
      "\n",
      "Civil Engineering \n",
      "MEng (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "4 years or 5 years including a professional placement\n",
      "H204; H205 with professional placement\n",
      "-------------\n",
      "\n",
      "Computer Science \n",
      "MSci (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "4 years, or 5 years with an industrial placement.\n",
      "G401\n",
      "-------------\n",
      "\n",
      "Computer Science \n",
      "BSc (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "3 years or 4 years with a 1 year placement, or the Professional Pathway Scheme (combining the degree with 3 years of IT work experience)\n",
      "G400\n",
      "-------------\n",
      "\n",
      "Computer Science with Cyber Security \n",
      "MSci (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "4 years, or 5 years with an industrial placement.\n",
      "G4G0\n",
      "-------------\n",
      "\n",
      "Computer Science with Games Technology \n",
      "MSci (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "4 years, or 5 years with an industrial placement\n",
      "GG49\n",
      "-------------\n",
      "\n",
      "Computer Science with Games Technology \n",
      "BSc (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "3 years, or 4 years with an industrial placement\n",
      "G490\n",
      "-------------\n",
      "\n",
      "Criminology \n",
      "BSc (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 or 4 years with a work placement option or study abroad\n",
      "4T82\n",
      "-------------\n",
      "\n",
      "Criminology and Psychology \n",
      "BSc (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 years 4 years with a work placement year or study abroad year, to be undertaken between Part 2 and Part 3 of the course. \n",
      "L3C8\n",
      "-------------\n",
      "\n",
      "Criminology and Sociology \n",
      "BSc (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 years or 4 years with a work placement option or study abroad.\n",
      "L390\n",
      "-------------\n",
      "\n",
      "Data Analytics and Actuarial Science \n",
      "BSc (Hons)\n",
      "Cass Business School\n",
      "3 years\n",
      "G3G1\n",
      "-------------\n",
      "\n",
      "Data Science \n",
      "MSci (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "Four years or five years including a professional placement\n",
      "G102\n",
      "-------------\n",
      "\n",
      "Economics \n",
      "BSc (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 years or 4 years with a work placement option.\n",
      "L100\n",
      "-------------\n",
      "\n",
      "Economics with Accounting \n",
      "BSc (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 years or 4 years with a work placement option.\n",
      "LN14\n",
      "-------------\n",
      "\n",
      "Electrical and Electronic Engineering \n",
      "BEng (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "3 years\n",
      "H602\n",
      "-------------\n",
      "\n",
      "Electrical and Electronic Engineering \n",
      "MEng (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "4 years or 5 years including a&nbsp;professional placement. \n",
      "H607; H609 with a professional placement\n",
      "-------------\n",
      "\n",
      "Engineering \n",
      "BEng (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "3 years\n",
      "H101\n",
      "-------------\n",
      "\n",
      "Engineering \n",
      "MEng (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "4 years or 5 years including a professional placement. \n",
      "H102; H103 with professional placement\n",
      "-------------\n",
      "\n",
      "English \n",
      "BA (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 years, or 4 years with a work placement option or study abroad. \n",
      "Q300\n",
      "-------------\n",
      "\n",
      "Finance \n",
      "BSc (Hons)\n",
      "Cass Business School\n",
      "3 years\n",
      "N3N3 / N3N1 with professional placement\n",
      "-------------\n",
      "\n",
      "Finance with Actuarial Science \n",
      "BSc (Hons)\n",
      "Cass Business School\n",
      "3 years\n",
      "G3N3\n",
      "-------------\n",
      "\n",
      "Financial Economics \n",
      "BSc (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 years or 4 years with a work placement option.\n",
      "L111\n",
      "-------------\n",
      "\n",
      "Health and Social Care \n",
      "BSc (Hons)\n",
      "School of Health Sciences\n",
      "3 years full-time\n",
      "L510\n",
      "-------------\n",
      "\n",
      "History \n",
      "BA (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 years, or 4 years with a work placement option or study abroad. \n",
      "V100\n",
      "-------------\n",
      "\n",
      "International Business \n",
      "BSc (Hons)\n",
      "Cass Business School\n",
      "3 years with Term Abroad\n",
      "N120: with Term Abroad <br> N111: with Study Abroad year<br> N112:  with Term Abroad & Professional Placement\n",
      "-------------\n",
      "\n",
      "International Political Economy \n",
      "BSc (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 years or 4 years with a work placement option.\n",
      "4J80\n",
      "-------------\n",
      "\n",
      "International Politics \n",
      "BSc (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 years or 4 years with professional placement.\n",
      "L240\n",
      "-------------\n",
      "\n",
      "International Politics and Sociology \n",
      "BSc (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 years or 4 years with a work placement option.\n",
      "LL23\n",
      "-------------\n",
      "\n",
      "Investment and Financial Risk Management \n",
      "BSc (Hons)\n",
      "Cass Business School\n",
      "3 years\n",
      "N390 / N319 with professional placement\n",
      "-------------\n",
      "\n",
      "Journalism \n",
      "BA (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 years or 4 years with a work placement option or study abroad.\n",
      "P500\n",
      "-------------\n",
      "\n",
      "Law \n",
      "LLB (Hons)\n",
      "The City Law School\n",
      "3 years\n",
      "M100\n",
      "-------------\n",
      "\n",
      "LLB in Legal Practice (Online) \n",
      "LLB (Hons)\n",
      "The City Law School\n",
      "\n",
      "\n",
      "-------------\n",
      "\n",
      "Mathematics \n",
      "BSc (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "3 years or 4 years including a professional placement.\n",
      "G100\n",
      "-------------\n",
      "\n",
      "Mathematics and Finance \n",
      "BSc (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "3 years or 4 years including a professional placement.\n",
      "GN13\n",
      "-------------\n",
      "\n",
      "Mathematics with Finance and Economics \n",
      "BSc (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "3 years or 4 years including a professional placement.\n",
      "G1L1\n",
      "-------------\n",
      "\n",
      "Mechanical Engineering \n",
      "BEng (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "Three years\n",
      "H300\n",
      "-------------\n",
      "\n",
      "Mechanical Engineering \n",
      "MEng (Hons)\n",
      "School of Mathematics, Computer Science and Engineering\n",
      "Four years or five years including a professional placement. \n",
      "H304 MEng (Hons), H305 MEng (Hons) with professional placement\n",
      "-------------\n",
      "\n",
      "Media, Communications and Sociology \n",
      "BSc (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 or 4 years with a work placement option or study abroad\n",
      "PL33\n",
      "-------------\n",
      "\n",
      "Mental Health Nursing \n",
      "BSc (Hons)\n",
      "School of Health Sciences\n",
      "Full-time: three years\n",
      "B702\n",
      "-------------\n",
      "\n",
      "Midwifery \n",
      "BSc (Hons)\n",
      "School of Health Sciences\n",
      "Full-time: 3 years\n",
      "B715\n",
      "-------------\n",
      "\n",
      "Music \n",
      "BMus (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 years or 4 years with one of the two optional sandwich year routes: a work placement year or study abroad year, to be undertaken between years two and three of the course.\n",
      "W300\n",
      "-------------\n",
      "\n",
      "Music, Sound and Technology \n",
      "BSc (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 years or 4 years with one of the two optional sandwich year routes: a work placement year or study abroad year, to be undertaken between years two and three of the course. \n",
      "W3W7\n",
      "-------------\n",
      "\n",
      "Nursing Studies \n",
      "BSc (Hons)\n",
      "School of Health Sciences\n",
      "Full-time: 2 years Part-time: 3 - 5 years \n",
      "\n",
      "-------------\n",
      "\n",
      "Optometry \n",
      "BSc (Hons)\n",
      "School of Health Sciences\n",
      "3 years (BSc only)\n",
      "B510\n",
      "-------------\n",
      "\n",
      "Politics \n",
      "BSc (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 years or 4 years with professional placement.\n",
      "L200\n",
      "-------------\n",
      "\n",
      "Psychology \n",
      "BSc (Hons)\n",
      "School of Arts and Social Sciences\n",
      "3 years 4 years with a work placement year or study abroad year, to be undertaken between Part 2 and Part 3 of the course\n",
      "C800\n",
      "-------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import urllib.request as ur\n",
    "\n",
    "#loop through the 7 pages of undergraduate courses\n",
    "for i in range (0,6):\n",
    "\n",
    "    #read and open url to scrape\n",
    "    urlToScrape = \"https://www.city.ac.uk/courses?level=Undergraduate&p=\" + str(i * 10 + 1)\n",
    "    r = ur.urlopen(urlToScrape).read()\n",
    "    soup = BeautifulSoup(r, \"lxml\")\n",
    "\n",
    "    #attributes for each course\n",
    "    courseList = soup.find_all('div', attrs={'class': 'course-finder__results__item course-finder__results__item--undergraduate'})\n",
    "\n",
    "    #loop through each course\n",
    "    for courseListItem in courseList:\n",
    "        \n",
    "        #get course name\n",
    "        try:\n",
    "            courseNameElement = courseListItem.find('div', attrs={'class': \"col-sm-24 col-md-18 col-lg-20\"})\n",
    "            courseName = courseNameElement.find('a').text\n",
    "        except Exception as e:\n",
    "            courseName = ''\n",
    "            #get degree name\n",
    "        try:\n",
    "            degreeTypeElement = courseListItem.find('div', attrs={'class': \"course-finder__results__item__award\"})\n",
    "            degreeName = degreeTypeElement.text\n",
    "        except Exception as e:\n",
    "            degreeName = ''\n",
    "        \n",
    "        #get school the course is part of\n",
    "        try:\n",
    "            courseSchoolElement = courseListItem.find('div', attrs={'class': \"course-finder__results__item__md course-finder__results__item__md--school\"})\n",
    "            courseSchool = courseSchoolElement.find('a').text\n",
    "        except Exception as e:\n",
    "            courseSchool = ''\n",
    "            \n",
    "        #get course duration    \n",
    "        try:\n",
    "            courseDurationElement = courseListItem.find('div', attrs={'class': \"course-finder__results__item__md course-finder__results__item__md--duration\"})\n",
    "            courseDuration = courseDurationElement.find_all('span')[2].text\n",
    "        except Exception as e:\n",
    "            courseDuration = ''\n",
    "        \n",
    "        #get course code\n",
    "        try:\n",
    "            courseCodeElement = courseListItem.find('div', attrs={'class': \"course-finder__results__item__md course-finder__results__item__md--code\"})\n",
    "            courseCode = courseCodeElement.find_all('span')[2].text\n",
    "        except Exception as e:\n",
    "            courseCode = ''\n",
    "        \n",
    "        print(courseName)\n",
    "        print(degreeName)\n",
    "        print(courseSchool)\n",
    "        print(courseDuration)\n",
    "        print(courseCode)\n",
    "        print (\"-------------\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}