|
18 | 18 |
|
19 | 19 | import fs from 'node:fs'; |
20 | 20 | import path from 'node:path'; |
21 | | -import crypto from 'node:crypto'; |
22 | 21 | import { fileURLToPath } from 'node:url'; |
| 22 | +import { SOURCES, captureSource } from './lib.mjs'; |
23 | 23 |
|
24 | 24 | const __dirname = path.dirname(fileURLToPath(import.meta.url)); |
25 | 25 | const BASELINE_DIR = path.join(__dirname, 'baseline'); |
26 | 26 |
|
27 | | -const SOURCES = [ |
28 | | - { |
29 | | - id: 'loomy', |
30 | | - name: 'Loomy Docs', |
31 | | - type: 'vitepress', |
32 | | - origin: 'https://loomy.xunfei.cn', |
33 | | - index: 'https://loomy.xunfei.cn/docs', |
34 | | - }, |
35 | | - { |
36 | | - id: 'astronclaw', |
37 | | - name: 'AstronClaw 使用指南', |
38 | | - type: 'single', |
39 | | - origin: 'https://www.xfyun.cn', |
40 | | - pages: [ |
41 | | - 'https://www.xfyun.cn/doc/spark/Agent-AstronClaw%E4%BD%BF%E7%94%A8%E6%8C%87%E5%8D%97.html', |
42 | | - ], |
43 | | - }, |
44 | | -]; |
45 | | - |
46 | | -const UA = |
47 | | - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) doc-watch/1.0'; |
48 | | - |
49 | | -async function fetchText(url) { |
50 | | - const res = await fetch(url, { headers: { 'User-Agent': UA }, redirect: 'follow' }); |
51 | | - if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`); |
52 | | - return res.text(); |
53 | | -} |
54 | | - |
55 | | -// ---------- HTML helpers (regex-based, no deps) ---------- |
56 | | - |
57 | | -function decodeEntities(s) { |
58 | | - return s |
59 | | - .replace(/&/g, '&') |
60 | | - .replace(/</g, '<') |
61 | | - .replace(/>/g, '>') |
62 | | - .replace(/"/g, '"') |
63 | | - .replace(/'/g, "'") |
64 | | - .replace(/ /g, ' ') |
65 | | - .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(+n)) |
66 | | - .replace(/[-]/g, ''); // zero-width chars (VitePress anchors) |
67 | | -} |
68 | | - |
69 | | -function stripTags(html) { |
70 | | - return decodeEntities( |
71 | | - html |
72 | | - .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, ' ') |
73 | | - .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, ' ') |
74 | | - .replace(/<[^>]+>/g, ' ') |
75 | | - ) |
76 | | - .replace(/\s+/g, ' ') |
77 | | - .trim(); |
78 | | -} |
79 | | - |
80 | | -function hash(s) { |
81 | | - return crypto.createHash('sha256').update(s).digest('hex').slice(0, 16); |
82 | | -} |
83 | | - |
84 | | -function absolutize(src, origin) { |
85 | | - if (!src) return src; |
86 | | - if (/^https?:\/\//i.test(src)) return src; |
87 | | - if (src.startsWith('//')) return 'https:' + src; |
88 | | - if (src.startsWith('/')) return origin + src; |
89 | | - return origin + '/' + src; |
90 | | -} |
91 | | - |
92 | | -// Grab the main article region so we don't diff against nav/footer chrome. |
93 | | -function extractMain(html) { |
94 | | - const main = html.match(/<main\b[^>]*>([\s\S]*?)<\/main>/i); |
95 | | - if (main) return main[1]; |
96 | | - const vpdoc = html.match(/<div class="vp-doc[\s\S]*?<\/main>/i); |
97 | | - if (vpdoc) return vpdoc[0]; |
98 | | - // xfyun / generic: try common content containers, else fall back to <body>. |
99 | | - for (const re of [ |
100 | | - /<article\b[^>]*>([\s\S]*?)<\/article>/i, |
101 | | - /<div[^>]*class="[^"]*(?:doc-content|markdown-body|article|content-detail)[^"]*"[^>]*>([\s\S]*?)<\/div>\s*<\/div>/i, |
102 | | - ]) { |
103 | | - const m = html.match(re); |
104 | | - if (m) return m[1]; |
105 | | - } |
106 | | - const body = html.match(/<body\b[^>]*>([\s\S]*?)<\/body>/i); |
107 | | - return body ? body[1] : html; |
108 | | -} |
109 | | - |
110 | | -function extractHeadings(contentHtml) { |
111 | | - const out = []; |
112 | | - const re = /<h([1-4])\b[^>]*>([\s\S]*?)<\/h\1>/gi; |
113 | | - let m; |
114 | | - while ((m = re.exec(contentHtml))) { |
115 | | - // Drop VitePress header-anchor links before flattening to text. |
116 | | - const inner = m[2].replace(/<a\b[^>]*class="[^"]*header-anchor[^"]*"[^>]*>[\s\S]*?<\/a>/gi, ''); |
117 | | - const text = stripTags(inner).trim(); |
118 | | - if (text) out.push(`h${m[1]}:${text}`); |
119 | | - } |
120 | | - return out; |
121 | | -} |
122 | | - |
123 | | -function extractImages(contentHtml, origin) { |
124 | | - const out = new Set(); |
125 | | - const re = /<img\b[^>]*\bsrc=["']([^"']+)["'][^>]*>/gi; |
126 | | - let m; |
127 | | - while ((m = re.exec(contentHtml))) { |
128 | | - const src = absolutize(m[1], origin); |
129 | | - // Skip ubiquitous logo/avatar chrome so it isn't flagged as "content". |
130 | | - if (/loomy-avatar\.png$/i.test(src)) continue; |
131 | | - out.add(src); |
132 | | - } |
133 | | - return [...out]; |
134 | | -} |
135 | | - |
136 | | -// VitePress sidebar carries the full page list. Pull /docs/* hrefs. |
137 | | -function extractSidebarLinks(html, origin, indexUrl) { |
138 | | - const links = new Set(); |
139 | | - const re = /<a\b[^>]*class="[^"]*VPLink[^"]*"[^>]*href="([^"]+)"/gi; |
140 | | - let m; |
141 | | - while ((m = re.exec(html))) { |
142 | | - let href = m[1]; |
143 | | - if (!href.startsWith('/docs')) continue; |
144 | | - href = href.replace(/#.*$/, '').replace(/\/$/, ''); |
145 | | - links.add(absolutize(href, origin)); |
146 | | - } |
147 | | - links.add(indexUrl.replace(/\/$/, '')); |
148 | | - return [...links].sort(); |
149 | | -} |
150 | | - |
151 | | -// ---------- per-source capture ---------- |
152 | | - |
153 | | -async function capturePage(url, origin) { |
154 | | - const html = await fetchText(url); |
155 | | - const titleM = html.match(/<title>([^<]*)<\/title>/i); |
156 | | - const main = extractMain(html); |
157 | | - const text = stripTags(main); |
158 | | - return { |
159 | | - url, |
160 | | - title: titleM ? decodeEntities(titleM[1]).trim() : url, |
161 | | - headings: extractHeadings(main), |
162 | | - images: extractImages(main, origin), |
163 | | - textHash: hash(text), |
164 | | - textLen: text.length, |
165 | | - }; |
166 | | -} |
167 | | - |
168 | | -async function captureSource(src) { |
169 | | - let pageUrls; |
170 | | - if (src.type === 'vitepress') { |
171 | | - const indexHtml = await fetchText(src.index); |
172 | | - pageUrls = extractSidebarLinks(indexHtml, src.origin, src.index); |
173 | | - } else { |
174 | | - pageUrls = src.pages; |
175 | | - } |
176 | | - |
177 | | - const pages = {}; |
178 | | - for (const url of pageUrls) { |
179 | | - try { |
180 | | - pages[url] = await capturePage(url, src.origin); |
181 | | - } catch (e) { |
182 | | - pages[url] = { url, error: String(e.message || e) }; |
183 | | - } |
184 | | - } |
185 | | - return { source: src.id, name: src.name, pageCount: Object.keys(pages).length, pages }; |
186 | | -} |
187 | | - |
188 | 27 | // ---------- diff ---------- |
189 | 28 |
|
190 | 29 | function diffSource(oldSnap, newSnap) { |
|
0 commit comments