[add] Table & Heading detection of Non-semantic Agenda pages

FreeCodeCamp-Chengdu · May 4, 2023 · c34afa8 · c34afa8
1 parent c4fe589
commit c34afa8
Show file tree

Hide file tree

Showing 6 changed files with 148 additions and 30 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
     "name": "@fcc-cdc/it-events",
-    "version": "1.3.0-alpha.0",
+    "version": "1.3.0",
     "license": "MIT",
     "author": "[email protected]",
     "description": "IT Events Crawler of China",
@@ -29,7 +29,7 @@
         "commander-jsx": "^0.6.1",
         "file-type": "^16.5.4",
         "fs-extra": "^11.1.1",
-        "jsdom": "^22.0.0",
+        "jsdom": "^21.1.2",
         "node-fetch": "^2.6.9",
         "string-similarity": "^4.0.4",
         "urlpattern-polyfill": "^8.0.2",

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/source/Agenda/HuoDongXing.ts b/source/Agenda/HuoDongXing.ts
@@ -5,3 +5,9 @@ export class HuoDongXingAgenda extends CommonAgendaCrawler {
 
     static schema = new URLPattern(`${this.baseURI}/:event(\\d+)`);
 }
+
+export class OldHuoDongXingAgenda extends CommonAgendaCrawler {
+    static baseURI = 'https://www.huodongxing.com/go';
+
+    static schema = new URLPattern(`${this.baseURI}/:event(\\w+)`);
+}
diff --git a/source/Agenda/common.ts b/source/Agenda/common.ts
@@ -1,11 +1,13 @@
 import { JSDOM } from 'jsdom';
-import { walkDOM, countBy } from 'web-utility';
+import { byteLength, countBy, walkDOM } from 'web-utility';
 
+import { CSSSelectorPrecision, getCSSSelector, sameParentOf } from '../utility';
 import { Agenda, AgendaCrawler } from './core';
-import { getCSSSelector, sameParentOf } from '../utility';
 
 export const TimePattern = /\d{1,2}\s*[:：]\s*\d{2}/;
 
+const HeadingSelector = `h1, h2, h3, h4, h5, h6, strong, b`;
+
 export abstract class CommonAgendaCrawler extends AgendaCrawler {
     document?: Document;
 
@@ -20,7 +22,11 @@ export abstract class CommonAgendaCrawler extends AgendaCrawler {
             walkDOM<Text>(document.body, 3),
             ({ nodeValue, parentElement }) =>
                 TimePattern.test(nodeValue) && {
-                    selector: getCSSSelector(parentElement, document.body)
+                    selector: getCSSSelector(
+                        parentElement,
+                        document.body,
+                        CSSSelectorPrecision.Medium
+                    )
                 }
         ).filter(Boolean);
 
@@ -32,12 +38,21 @@ export abstract class CommonAgendaCrawler extends AgendaCrawler {
         const [first, second] = document.querySelectorAll(agendaTimeSelector);
 
         const agendaBox = sameParentOf(first, second) as Element;
-        const agendaBoxSelector = getCSSSelector(agendaBox);
+        const agendaBoxSelector = getCSSSelector(
+            agendaBox,
+            document.body,
+            CSSSelectorPrecision.High
+        );
 
         for (let i = 0; i < agendaBox.childElementCount; i++)
-            yield await this.getItem(
-                `${agendaBoxSelector} > :nth-child(${i + 1})`
-            );
+            if (agendaBox.tagName.toLowerCase() === 'tbody')
+                yield* this.getItems(
+                    agendaBox.children[i] as HTMLTableRowElement
+                );
+            else
+                yield await this.getItem(
+                    `${agendaBoxSelector} > :nth-child(${i + 1})`
+                );
         this.document = undefined;
     }
 
@@ -48,20 +63,55 @@ export abstract class CommonAgendaCrawler extends AgendaCrawler {
 
         let time = '';
 
-        const [title, name, position, summary] = Array.from(
-            walkDOM<Text>(agendaItem, 3),
-            ({ nodeValue }) => nodeValue.trim()
-        ).filter(text => !TimePattern.test(text) || !(time = text));
+        const [head, body] = Array.from(walkDOM<Text>(agendaItem, 3)).reduce(
+            (group, { parentElement, nodeValue }) => {
+                const isHeading =
+                    parentElement.matches(HeadingSelector) ||
+                    !!parentElement.closest(HeadingSelector);
+
+                if (TimePattern.test(nodeValue)) time = nodeValue.trim();
+                else group[isHeading ? 0 : 1].push(nodeValue.trim());
 
+                return group;
+            },
+            [[], []] as string[][]
+        );
         const [startTime, endTime] = time.split(/[^\d:：]+/),
+            [name, title] = head.sort((a, b) => byteLength(a) - byteLength(b)),
+            [position, summary] = body.sort(
+                (a, b) => byteLength(a) - byteLength(b)
+            ),
             avatar =
                 agendaItem.querySelector<HTMLImageElement>('img[src]')?.src;
 
         return {
             mentor: { name, position, avatar },
             title,
+            summary,
             startTime,
             endTime
         };
     }
+
+    protected getItems({ children }: HTMLTableRowElement): Agenda[] {
+        const [time, ...agendas] = [...children];
+        const [startTime, endTime] = time.textContent.trim().split(/[^\d:：]+/);
+
+        return agendas.map(agendaItem => {
+            const [name, position, title, summary] = agendaItem.textContent
+                    .trim()
+                    .split('\n')
+                    .sort((a, b) => byteLength(a) - byteLength(b)),
+                avatar =
+                    agendaItem.querySelector<HTMLImageElement>('img[src]')?.src;
+
+            return {
+                mentor: { name, position, avatar },
+                title,
+                summary,
+                startTime,
+                endTime
+            };
+        });
+    }
 }
diff --git a/source/Agenda/core.ts b/source/Agenda/core.ts
@@ -13,7 +13,7 @@ export type Duration = Partial<Record<'startTime' | 'endTime', string>>;
 
 export type Forum = Duration & Pick<Mentor, 'name' | 'summary'>;
 
-export interface Agenda extends Duration {
+export interface Agenda extends Duration, Pick<Forum, 'summary'> {
     title?: string;
     mentor?: Mentor;
     forum?: Forum;

diff --git a/source/utility.ts b/source/utility.ts
@@ -75,9 +75,16 @@ export async function saveFile(
     return path;
 }
 
+export enum CSSSelectorPrecision {
+    Low,
+    Medium,
+    High
+}
+
 export function getCSSSelector(
     toElement: Element,
-    fromElement = toElement.getRootNode()
+    fromElement = toElement.getRootNode(),
+    precision = CSSSelectorPrecision.Low
 ) {
     const selectors: string[] = [];
 
@@ -88,9 +95,12 @@ export function getCSSSelector(
             tagName.toLowerCase() +
             (className.trim()
                 ? '.' + className.split(/\s+/).filter(Boolean).join('.')
-                : `:nth-child(${
+                : precision === CSSSelectorPrecision.High ||
+                  (precision === CSSSelectorPrecision.Medium && !selectors[0])
+                ? `:nth-child(${
                       [...parentNode.children].indexOf(toElement) + 1
-                  })`);
+                  })`
+                : '');
         selectors.unshift(selector);
 
         toElement = parentNode as Element;