Merge pull request #112 from NIAEFEUP/refactor/use-sigarra-api-link-o…

…n-slot-fetching

Refactor slot and professors spiders in light of sigarra schedule html page changes

src/Makefile

@@ -1,7 +1,8 @@
 .PHONY: all clean
 FILES = $(shell ls)
 
-all: info faculties courses course_units course_metadata classes slots slot_professor professors
+# all: info faculties courses course_units course_metadata classes slots slot_professor professors
+all: info faculties courses course_units course_metadata slots
 
 
 faculties: 
@@ -16,18 +17,9 @@ course_units:
 course_metadata:
 	scrapy crawl course_metadata
 
-classes:
-	scrapy crawl classes
-
 slots:
 	scrapy crawl slots
 
-slot_professor:
-	scrapy crawl slot_professor
-
-professors:
-	scrapy crawl professors
-
 info:
 	python ./scrapper/info.py
 
@@ -43,9 +35,9 @@ upload:
 
 convert_mysql:
 	@echo "Converting dump to mysql..."
-	@bash ./scripts/sqlite3-to-mysql.sh ./scripts/dump/data/dump_sqlite3.sql > ./scripts/dump/data/dump_mysql.sql
+	@bash ./scripts/sqlite3-to-mysql.sh ./scripts/dump/data/dump_sqlite3.sql > ./scripts/dump/data/01_data.sql
 	@echo "Convertion completed!"
 
 clean: 
 	@echo "Removing database.db"
-	@rm ./scrapper/database/dbs/database.db
+	@rm ./scrapper/database/dbs/database.db

src/scrapper/items.py

@@ -7,6 +7,7 @@
 
 import scrapy
 
+
 class Faculty(scrapy.Item):
     acronym = scrapy.Field()
     name = scrapy.Field()
@@ -37,18 +38,22 @@ class CourseUnit(scrapy.Item):
     classes_url = scrapy.Field()
     last_updated = scrapy.Field()
 
+
 class CourseMetadata(scrapy.Item):
     course_id = scrapy.Field()
     course_unit_id = scrapy.Field()
     course_unit_year = scrapy.Field()
     ects = scrapy.Field()
 
+
 class Class(scrapy.Item):
     course_unit_id = scrapy.Field()
     name = scrapy.Field()  # 1MIEIC01
     last_updated = scrapy.Field()
-
+
+
 class Slot(scrapy.Item):
+    id = scrapy.Field()
     lesson_type = scrapy.Field()  # T, TP, PL, etc.
     day = scrapy.Field()  # 0 = monday, 1 = tuesday, .., 5 = saturday (no sunday)
     start_time = scrapy.Field()  # At what time the lesson starts
@@ -64,8 +69,8 @@ class SlotProfessor(scrapy.Item):
     slot_id = scrapy.Field()
     professor_id = scrapy.Field()
 
+
 class Professor(scrapy.Item):
     id = scrapy.Field()
     professor_acronym = scrapy.Field()
     professor_name = scrapy.Field()
-