discover_previous_years.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sigpy import get_faculty\n",
    "from sigpy.parser import *\n",
    "from lxml.html import fromstring\n",
    "import sys\n",
    "\n",
    "# MAIN CONFIGURATIONS\n",
    "ERASMUS_PAGE = \"https://sigarra.up.pt/feup/pt/coop_candidatura_geral.ver_colocacoes_aluno?pi_processo_view_id=%s\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# WHAT TO EXTRACT\n",
    "extract = {\n",
    "    \"model\": \"erasmus\",\n",
    "    \"attributes\": {\n",
    "        \"page_time\": {\"regex\": \"Data das Colocações provisórias: (.*)</p>\"},\n",
    "        \"students\": {\n",
    "            \"model\": \"student\",\n",
    "            \"list\": \"True\",\n",
    "            \"css\": \"#conteudoinner > table tr.d\",\n",
    "            \"attributes\": {\n",
    "                \"phase\": {\"xpath\": \".//td[1]\"},\n",
    "                \"order\": {\"xpath\": \".//td[2]\"},\n",
    "                \"name\": {\"xpath\": \".//td[3]\"},\n",
    "                \"id\": {\"xpath\": \".//td[4]\"},\n",
    "                \"grade\": {\"xpath\": \".//td[5]\"},\n",
    "                \"country\": {\"xpath\": \".//td[6]\"},\n",
    "                \"university\": {\"xpath\": \".//td[7]\"},\n",
    "                \"type\": {\"xpath\": \".//td[8]\"},\n",
    "                \"option\": {\"xpath\": \".//td[9]\"},\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LOGIN AND SCRAPE\n",
    "fac = get_faculty(\"feup\", False)\n",
    "fac.set_verbose(True)\n",
    "if not fac.login(201403027):\n",
    "    print(\"Failed login\", file=stderr)\n",
    "    exit(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "live = {}\n",
    "start, end = (990000, 1000000)\n",
    "print(\"Looking from %d to %d (total: %d)\" % (start, end, end - start))\n",
    "for id in range(start, end):\n",
    "    res = parse_class(fromstring(fac.GET(ERASMUS_PAGE % id, False)), extract)\n",
    "    if res.students:\n",
    "        print(\"X\", end=\"\")\n",
    "        live[id] = res\n",
    "    else:\n",
    "        print(\".\", end=\"\")\n",
    "    sys.stdout.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "live.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Previous MIEIC years\n",
    " * [12/13](https://sigarra.up.pt/feup/pt/coop_candidatura_geral.ver_colocacoes_aluno?pi_processo_view_id=1000861) - 1000861\n",
    " * [13/14](https://sigarra.up.pt/feup/pt/coop_candidatura_geral.ver_colocacoes_aluno?pi_processo_view_id=1002664) - 1002664\n",
    " * [14/15](https://sigarra.up.pt/feup/pt/coop_candidatura_geral.ver_colocacoes_aluno?pi_processo_view_id=1004742) - 1004742\n",
    " * [15/16](https://sigarra.up.pt/feup/pt/coop_candidatura_geral.ver_colocacoes_aluno?pi_processo_view_id=1006885) - 1006885\n",
    " * [16/17](https://sigarra.up.pt/feup/pt/coop_candidatura_geral.ver_colocacoes_aluno?pi_processo_view_id=1008349) - 1008349\n",
    " * [17/18](https://sigarra.up.pt/feup/pt/coop_candidatura_geral.ver_colocacoes_aluno?pi_processo_view_id=1010349) - 1010349\n",
    " * [18/19](https://sigarra.up.pt/feup/pt/coop_candidatura_geral.ver_colocacoes_aluno?pi_processo_view_id=1012045) - 1012045\n",
    " "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}