Fix course unit schema

README.md

@@ -99,7 +99,7 @@ root@00723f950c71:/scrapper# scrapy shell "https://sigarra.up.pt/fcnaup/pt/cur_g
 
 ## :triangular_ruler: Database design
 
-![Image](./docs/schema.2.0.png)
+![Image](./docs/schema.2.1.png)
 
 ## :page_with_curl: More information
 

src/config.ini

@@ -21,12 +21,12 @@ filename=database.db
 # Represents the expected number of elements for each table. 
 [statistics]
 num_faculties=14
-num_courses=313
-num_course_units=6513
-num_course_metadata=8590
-num_schedules=11120
-num_schedule_professor=42737
-num_professors=2688
+num_courses=318
+num_course_units=6679
+num_course_metadata=9427
+num_schedules=21441
+num_schedule_professor=36986
+num_professors=2499
 
 # !! Do NOT change this section !! 
 [dump]

src/requirements.txt

@@ -23,7 +23,7 @@ protobuf==4.22.0
 pyopenssl==22.0.0
 python_bcrypt==0.3.2
 railroad==0.5.0
-Scrapy==2.7.0
+Scrapy==2.9.0
 Sphinx==6.1.3
 tornado==6.2
 tqdm==4.64.1

src/scrapper/database/dbs/create_db_sqlite3.sql

@@ -46,15 +46,14 @@ CREATE TABLE `course` (
 CREATE TABLE `course_unit` (
   `id` INTEGER PRIMARY KEY AUTOINCREMENT,
   `sigarra_id` int(11) NOT NULL,
-  `course_id` int(11) NOT NULL,
   `name` varchar(200) NOT NULL,
   `acronym` varchar(16) NOT NULL,
   `url` varchar(2000) NOT NULL,
   `semester` tinyint(4) NOT NULL,
   `year` smallint(6) NOT NULL,
   `schedule_url` varchar(2000) DEFAULT NULL,
   `last_updated` datetime NOT NULL,
-  FOREIGN KEY (`course_id`) REFERENCES `course`(`id`) ON DELETE CASCADE ON UPDATE CASCADE
+  UNIQUE (`sigarra_id`, `semester`, `year`)
 );
 
 -- --------------------------------------------------------
@@ -119,7 +118,7 @@ CREATE TABLE `schedule_professor` (
 CREATE TABLE `professor` (
   `sigarra_id` INTEGER PRIMARY KEY,
   `professor_acronym` varchar(16),
-  `professor_name` varchar(50)
+  `professor_name` varchar(100)
 );
 
 -- --------------------------------------------------------

src/scrapper/items.py

@@ -27,7 +27,6 @@ class Course(scrapy.Item):
 
 class CourseUnit(scrapy.Item):
     sigarra_id = scrapy.Field()
-    course_id = scrapy.Field()
     name = scrapy.Field()
     acronym = scrapy.Field()
     url = scrapy.Field()

src/scrapper/spiders/course_metadata.py

@@ -32,6 +32,7 @@ def __init__(self, password=None, category=None, *args, **kwargs):
         self.open_config()
         self.user = CONFIG[USERNAME]
         self.password = CONFIG[PASSWORD]
+        self.course_metadatas = set()
 
     def format_login_url(self):
         return '{}?{}'.format(self.login_page_base, urlencode({
@@ -64,32 +65,52 @@ def check_login_response(self, response):
 
 
     def courseRequests(self):
-        print("Gathering course metadata...")
+        print("Gathering courses...")
         db = Database() 
 
-        sql = "SELECT cu.id, cu.url, cu.course_id, c.sigarra_id FROM course_unit cu JOIN course c ON c.id=cu.course_id"
+        sql = "SELECT id, sigarra_id FROM course"
+        db.cursor.execute(sql)
+        self.courses = {sigarra_id: id for id, sigarra_id in db.cursor.fetchall()}
+
+        self.log("Crawling {} courses".format(len(self.courses)))
+
+        print("Gathering course units...")
+        sql = "SELECT id, url FROM course_unit"
         db.cursor.execute(sql)
         self.course_units = db.cursor.fetchall()
         db.connection.close()
 
-        self.log("Crawling {} courses".format(len(self.course_units)))
+        self.log("Crawling {} course units".format(len(self.course_units)))
 
         for course_unit in self.course_units:
             yield scrapy.http.Request(
                 url=course_unit[1],
-                meta={'course_unit_id': course_unit[0],'course_id': course_unit[2], 'course_unit_sigarra_id': course_unit[3]},
+                meta={'course_unit_id': course_unit[0]},
                 callback=self.extractCourseUnitByYears)
 
     def extractCourseUnitByYears(self, response): 
         study_cycles = response.xpath('//h3[text()="Ciclos de Estudo/Cursos"]/following-sibling::table[1]').get()
-        df = pd.read_html(study_cycles, decimal=',', thousands='.', extract_links="all")[0]
+        if study_cycles is None:
+            return
+        df = pd.read_html(study_cycles, decimal=',', thousands='.')[0]
 
         for (_, row) in df.iterrows():
-            # NOTE(luisd & thePeras): We need to filter by the current course otherwise we would get duplicate entries
-            if  f'pv_curso_id={response.meta["course_unit_sigarra_id"]}' in str(row[0][1]):
-                yield CourseMetadata(
-                        course_id = response.meta['course_id'],
-                        course_unit_id = response.meta['course_unit_id'],
-                        course_unit_year = row[3][0],
-                        ects = row[5][0]
-                    )
+            course_unit_id = response.meta['course_unit_id']
+            course_acronym = row['Sigla' if 'Sigla' in row else 'Acronym']
+            course_url = response.xpath(f'//a[text()="{course_acronym}"]/@href').get()
+            if course_url is None:
+                continue
+            course_sigarra_id = int(course_url.split('=')[1].split('&')[0])
+            if course_sigarra_id not in self.courses:
+                continue
+            course_id = self.courses[course_sigarra_id]
+            course_unit_year = row['Anos Curriculares']
+            if (course_id, course_unit_id, course_unit_year) in self.course_metadatas:
+                continue
+            self.course_metadatas.add((course_id, course_unit_id, course_unit_year))
+            yield CourseMetadata(
+                course_id = course_id,
+                course_unit_id = course_unit_id,
+                course_unit_year = row['Anos Curriculares'],
+                ects = row[5]
+            )

src/scrapper/spiders/course_unit_spider.py

@@ -34,6 +34,7 @@ def __init__(self, password=None, category=None, *args, **kwargs):
         self.open_config()
         self.user = CONFIG[USERNAME]
         self.password = CONFIG[PASSWORD]
+        self.sigarra_ids = set()
         logging.getLogger('scrapy').propagate = False
 
     def format_login_url(self):
@@ -113,8 +114,11 @@ def extractCourseUnitInfo(self, response):
         if name == 'Sem Resultados':
             return None  # Return None as yielding would continue the program and crash at the next assert
 
-        course_unit_id = parse_qs(urlparse(response.url).query)[
-            'pv_ocorrencia_id'][0]
+        sigarra_id = parse_qs(urlparse(response.url).query)['pv_ocorrencia_id'][0]
+        if sigarra_id in self.sigarra_ids:
+            return
+        self.sigarra_ids.add(sigarra_id)
+
         acronym = response.xpath(
             '//div[@id="conteudoinner"]/table[@class="formulario"][1]//td[text()="Sigla:"]/following-sibling::td[1]/text()').extract_first()
 
@@ -164,8 +168,7 @@ def extractCourseUnitInfo(self, response):
 
         for semester in semesters:
             yield CourseUnit(
-                sigarra_id=course_unit_id,
-                course_id=response.meta['course_id'],
+                sigarra_id=sigarra_id,
                 name=name,
                 acronym=acronym,
                 url=url,
@@ -174,4 +177,3 @@ def extractCourseUnitInfo(self, response):
                 semester=semester,
                 last_updated=datetime.now()
             )
-

src/scrapper/spiders/schedule_spider.py

@@ -84,9 +84,6 @@ def classUnitRequests(self):
                 meta={'id': class_unit[0]},
                 callback=self.extractSchedule,
                 errback=self.func)
-
-    def func(self):
-        print("Error :(")
 
     def extractSchedule(self, response):
         # Check if there is no schedule available

src/scripts/dump/schema/create_db_sqlite3.sql

@@ -48,11 +48,10 @@ CREATE TABLE `course` (
 CREATE TABLE `course_unit` (
   `id` int(11) NOT NULL,
   `sigarra_id` int(11) NOT NULL,
-  `course_id` int(11) NOT NULL,
   `name` varchar(200) NOT NULL,
   `acronym` varchar(16) NOT NULL,
   `url` varchar(2000) NOT NULL,
-  `semester` int(4) NOT NULL,
+  `semester` tinyint(4) NOT NULL,
   `year` smallint(6) NOT NULL,
   `schedule_url` varchar(2000) DEFAULT NULL,
   `last_updated` datetime NOT NULL
@@ -67,7 +66,7 @@ CREATE TABLE `course_unit` (
 CREATE TABLE `course_metadata` (
   `course_id` int(11) NOT NULL,
   `course_unit_id` int(11) NOT NULL,
-  `course_unit_year` int(4) NOT NULL,
+  `academic_year` int(4) NOT NULL,
   `ects` float(4) NOT NULL
 ) ENGINE=InnoDB CHARSET = utf8 COLLATE = utf8_general_ci;
 
@@ -81,7 +80,7 @@ CREATE TABLE `course_metadata` (
 
 CREATE TABLE `schedule` (
   `id` INTEGER NOT NULL,
-  `day` int(3) NOT NULL,
+  `day` tinyint(3) NOT NULL,
   `duration` decimal(3,1) NOT NULL,
   `start_time` decimal(3,1) NOT NULL,
   `location` varchar(31) NOT NULL,
@@ -126,8 +125,6 @@ alter TABLE course ADD PRIMARY KEY (`id`);
 alter TABLE course ADD FOREIGN KEY (`faculty_id`) REFERENCES `faculty`(`acronym`) on DELETE CASCADE ON UPDATE CASCADE;
 
 alter TABLE course_unit ADD PRIMARY KEY (`id`);
-alter TABLE course_unit ADD UNIQUE (`sigarra_id`, `course_id`);
-alter TABLE course_unit ADD FOREIGN KEY (`course_id`) REFERENCES `course`(`id`) ON DELETE CASCADE ON UPDATE CASCADE;
 
 alter TABLE course_metadata ADD PRIMARY KEY (`course_id`, `course_unit_id`, `course_unit_year`);
 alter TABLE course_metadata ADD FOREIGN KEY (`course_unit_id`) REFERENCES `course_unit`(`id`) ON DELETE CASCADE ON UPDATE CASCADE;
@@ -156,8 +153,7 @@ CREATE INDEX `course_faculty_acronym` ON `course` (`faculty_id`);
 --
 -- Indexes for table `course_unit`
 --
-CREATE UNIQUE INDEX `course_unit_uniqueness` ON `course_unit`  (`sigarra_id`,`course_id`,`year`,`semester`); 
-CREATE INDEX `course_unit_course_id` ON `course_unit` (`course_id`);
+CREATE UNIQUE INDEX `course_unit_uniqueness` ON `course_unit` (`sigarra_id`,`year`,`semester`); 
 
 --
 -- Indexes for table `faculty`

src/scripts/dump/schema/schema_mysql.sql

@@ -48,11 +48,10 @@ CREATE TABLE `course` (
 CREATE TABLE `course_unit` (
   `id` int(11) NOT NULL,
   `sigarra_id` int(11) NOT NULL,
-  `course_id` int(11) NOT NULL,
   `name` varchar(200) NOT NULL,
   `acronym` varchar(16) NOT NULL,
   `url` varchar(2000) NOT NULL,
-  `semester` int(4) NOT NULL,
+  `semester` tinyint(4) NOT NULL,
   `year` smallint(6) NOT NULL,
   `schedule_url` varchar(2000) DEFAULT NULL,
   `last_updated` datetime NOT NULL
@@ -67,7 +66,7 @@ CREATE TABLE `course_unit` (
 CREATE TABLE `course_metadata` (
   `course_id` int(11) NOT NULL,
   `course_unit_id` int(11) NOT NULL,
-  `course_unit_year` int(4) NOT NULL,
+  `academic_year` int(4) NOT NULL,
   `ects` float(4) NOT NULL
 ) ENGINE=InnoDB CHARSET = utf8 COLLATE = utf8_general_ci;
 
@@ -81,7 +80,7 @@ CREATE TABLE `course_metadata` (
 
 CREATE TABLE `schedule` (
   `id` INTEGER NOT NULL,
-  `day` int(3) NOT NULL,
+  `day` tinyint(3) NOT NULL,
   `duration` decimal(3,1) NOT NULL,
   `start_time` decimal(3,1) NOT NULL,
   `location` varchar(31) NOT NULL,
@@ -126,7 +125,6 @@ alter TABLE course ADD PRIMARY KEY (`id`);
 alter TABLE course ADD FOREIGN KEY (`faculty_id`) REFERENCES `faculty`(`acronym`) on DELETE CASCADE ON UPDATE CASCADE;
 
 alter TABLE course_unit ADD PRIMARY KEY (`id`);
-alter TABLE course_unit ADD FOREIGN KEY (`course_id`) REFERENCES `course`(`id`) ON DELETE CASCADE ON UPDATE CASCADE;
 
 alter TABLE course_metadata ADD PRIMARY KEY (`course_id`, `course_unit_id`, `course_unit_year`);
 alter TABLE course_metadata ADD FOREIGN KEY (`course_unit_id`) REFERENCES `course_unit`(`id`) ON DELETE CASCADE ON UPDATE CASCADE;
@@ -155,8 +153,7 @@ CREATE INDEX `course_faculty_acronym` ON `course` (`faculty_id`);
 --
 -- Indexes for table `course_unit`
 --
-CREATE UNIQUE INDEX `course_unit_uniqueness` ON `course_unit`  (`sigarra_id`,`course_id`,`year`,`semester`); 
-CREATE INDEX `course_unit_course_id` ON `course_unit` (`course_id`);
+CREATE UNIQUE INDEX `course_unit_uniqueness` ON `course_unit` (`sigarra_id`,`year`,`semester`); 
 
 --
 -- Indexes for table `faculty`
@@ -173,4 +170,3 @@ CREATE INDEX `schedule_course_unit_id` ON `schedule`(`course_unit_id`);
 -- 
 CREATE INDEX `course_metadata_index` ON `course_metadata`(`course_id`, `course_unit_id`, `course_unit_year`); 
 
-