From 386c2451ecbed99f4dfd5c97f52c68364dd3a56f Mon Sep 17 00:00:00 2001 From: Lanzhijiang Date: Sun, 26 Jul 2026 08:40:44 +0800 Subject: [PATCH 1/6] chore: add temporary mobile evidence crawler --- .../temporary-soft-copyright-manual-crawl.mjs | 187 ++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 scripts/temporary-soft-copyright-manual-crawl.mjs diff --git a/scripts/temporary-soft-copyright-manual-crawl.mjs b/scripts/temporary-soft-copyright-manual-crawl.mjs new file mode 100644 index 00000000..7666db63 --- /dev/null +++ b/scripts/temporary-soft-copyright-manual-crawl.mjs @@ -0,0 +1,187 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { chromium } from 'playwright'; + +const BASE_URL = process.env.PARTNERUP_BASE_URL || 'https://app.partner-up.cn'; +const OUT_DIR = path.resolve(process.env.CRAWL_OUT_DIR || 'soft-copyright-manual-evidence'); +const SRC_DIR = path.resolve('apps/web/src'); +const MAX_PAGES = Number(process.env.MAX_PAGES || 35); +const SOURCE_COMMIT = process.env.SOURCE_COMMIT || '30f51b96e2498b4b85014552512a31a7345d86cb'; + +fs.mkdirSync(OUT_DIR, { recursive: true }); +fs.mkdirSync(path.join(OUT_DIR, 'screenshots'), { recursive: true }); + +function walk(dir) { + const result = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) result.push(...walk(full)); + else result.push(full); + } + return result; +} + +const sourceFiles = walk(SRC_DIR).filter((file) => /\.(?:ts|vue)$/.test(file)); +const routeEvidence = []; +const staticPaths = new Set(['/']); +const pathPattern = /\bpath\s*:\s*['"`]([^'"`]+)['"`]/g; +for (const file of sourceFiles) { + const text = fs.readFileSync(file, 'utf8'); + if (!/(?:createRouter|RouteRecordRaw|routes\s*=|children\s*:)/.test(text)) continue; + let match; + while ((match = pathPattern.exec(text))) { + const value = match[1].trim(); + routeEvidence.push({ file: path.relative(process.cwd(), file), path: value }); + if (value.startsWith('/') && !value.includes(':') && !value.includes('*')) staticPaths.add(value); + } +} + +for (const candidate of [ + '/create', '/discover', '/events', '/profile', '/me', '/about', '/privacy', '/terms', + '/admin/bi?code=2026zcb' +]) staticPaths.add(candidate); + +fs.writeFileSync(path.join(OUT_DIR, 'source-routes.json'), JSON.stringify(routeEvidence, null, 2)); +fs.writeFileSync(path.join(OUT_DIR, 'crawl-metadata.json'), JSON.stringify({ + baseUrl: BASE_URL, + sourceCommit: SOURCE_COMMIT, + viewport: { width: 390, height: 844, deviceScaleFactor: 2 }, + generatedAt: new Date().toISOString(), +}, null, 2)); + +function normalizeUrl(input) { + try { + const url = new URL(input, BASE_URL); + if (url.origin !== new URL(BASE_URL).origin) return null; + url.hash = ''; + for (const key of [...url.searchParams.keys()]) { + if (/^(?:code|token|state|ticket)$/i.test(key) && key !== 'code') url.searchParams.delete(key); + } + return url.toString(); + } catch { + return null; + } +} + +function safeName(value, index) { + const u = new URL(value); + const raw = `${String(index).padStart(2, '0')}-${u.pathname}${u.search}` + .replace(/^\/+/, '') + .replace(/[^a-zA-Z0-9._-]+/g, '-') + .replace(/-+/g, '-') + .replace(/^-|-$/g, ''); + return raw || `${String(index).padStart(2, '0')}-root`; +} + +const queue = [...staticPaths].map((p) => normalizeUrl(p)).filter(Boolean); +const queued = new Set(queue); +const visited = new Set(); +const pages = []; + +const browser = await chromium.launch({ headless: true }); +const context = await browser.newContext({ + viewport: { width: 390, height: 844 }, + deviceScaleFactor: 2, + isMobile: true, + hasTouch: true, + locale: 'zh-CN', + timezoneId: 'Asia/Shanghai', + userAgent: 'Mozilla/5.0 (Linux; Android 16; Mobile) AppleWebKit/537.36 Chrome/140.0 Mobile Safari/537.36', +}); +context.setDefaultTimeout(12000); + +while (queue.length && pages.length < MAX_PAGES) { + const requestedUrl = queue.shift(); + if (!requestedUrl || visited.has(requestedUrl)) continue; + visited.add(requestedUrl); + + const page = await context.newPage(); + const consoleErrors = []; + const failedRequests = []; + page.on('console', (msg) => { if (msg.type() === 'error') consoleErrors.push(msg.text()); }); + page.on('requestfailed', (req) => failedRequests.push({ url: req.url(), error: req.failure()?.errorText || '' })); + + let navigationError = null; + let responseStatus = null; + try { + const response = await page.goto(requestedUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }); + responseStatus = response?.status() ?? null; + await page.waitForTimeout(2500); + } catch (error) { + navigationError = String(error); + } + + const finalUrl = page.url(); + let snapshot = {}; + try { + snapshot = await page.evaluate(() => { + const clean = (value) => (value || '').replace(/\s+/g, ' ').trim(); + const text = clean(document.body?.innerText || ''); + const collect = (selector, mapper) => [...document.querySelectorAll(selector)].map(mapper).filter(Boolean); + return { + title: document.title, + lang: document.documentElement.lang, + headings: collect('h1,h2,h3', (el) => ({ level: el.tagName, text: clean(el.textContent) })).slice(0, 80), + buttons: collect('button,[role="button"]', (el) => ({ + text: clean(el.textContent), + ariaLabel: el.getAttribute('aria-label'), + disabled: el.matches(':disabled,[aria-disabled="true"]'), + })).slice(0, 100), + links: collect('a[href]', (el) => ({ text: clean(el.textContent), href: el.href })).slice(0, 150), + fields: collect('input,textarea,select', (el) => ({ + tag: el.tagName, + type: el.getAttribute('type'), + name: el.getAttribute('name'), + placeholder: el.getAttribute('placeholder'), + ariaLabel: el.getAttribute('aria-label'), + })).slice(0, 100), + dialogs: collect('[role="dialog"],dialog', (el) => clean(el.textContent)).slice(0, 20), + text: text.slice(0, 12000), + bodyHeight: document.documentElement.scrollHeight, + }; + }); + } catch (error) { + snapshot = { evaluationError: String(error) }; + } + + const index = pages.length + 1; + const name = safeName(finalUrl || requestedUrl, index); + const viewportScreenshot = path.join(OUT_DIR, 'screenshots', `${name}-viewport.png`); + const fullScreenshot = path.join(OUT_DIR, 'screenshots', `${name}-full.png`); + try { await page.screenshot({ path: viewportScreenshot, fullPage: false }); } catch {} + try { await page.screenshot({ path: fullScreenshot, fullPage: true }); } catch {} + + const entry = { + index, + requestedUrl, + finalUrl, + responseStatus, + navigationError, + viewportScreenshot: path.relative(OUT_DIR, viewportScreenshot), + fullScreenshot: path.relative(OUT_DIR, fullScreenshot), + ...snapshot, + consoleErrors: consoleErrors.slice(0, 30), + failedRequests: failedRequests.slice(0, 30), + }; + pages.push(entry); + fs.writeFileSync(path.join(OUT_DIR, 'pages.json'), JSON.stringify(pages, null, 2)); + + for (const link of snapshot.links || []) { + const normalized = normalizeUrl(link.href); + if (!normalized || visited.has(normalized) || queued.has(normalized)) continue; + queued.add(normalized); + queue.push(normalized); + } + + await page.close(); +} + +await browser.close(); +fs.writeFileSync(path.join(OUT_DIR, 'pages.json'), JSON.stringify(pages, null, 2)); +fs.writeFileSync(path.join(OUT_DIR, 'crawl-summary.json'), JSON.stringify({ + requestedCount: visited.size, + capturedCount: pages.length, + discoveredCount: queued.size, + sourceRouteCount: routeEvidence.length, + sourceCommit: SOURCE_COMMIT, +}, null, 2)); From eb554de64e104d326ae3b4be19c6a4c2337d6d1a Mon Sep 17 00:00:00 2001 From: Lanzhijiang Date: Sun, 26 Jul 2026 08:41:05 +0800 Subject: [PATCH 2/6] chore: add temporary manual evidence workflow --- .../temporary-soft-copyright-manual.yml | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/temporary-soft-copyright-manual.yml diff --git a/.github/workflows/temporary-soft-copyright-manual.yml b/.github/workflows/temporary-soft-copyright-manual.yml new file mode 100644 index 00000000..5c550bfb --- /dev/null +++ b/.github/workflows/temporary-soft-copyright-manual.yml @@ -0,0 +1,56 @@ +name: Temporary soft copyright manual evidence + +on: + pull_request: + branches: [develop] + paths: + - '.github/workflows/temporary-soft-copyright-manual.yml' + - 'scripts/temporary-soft-copyright-manual-crawl.mjs' + +permissions: + contents: read + +jobs: + capture: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout evidence branch + uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: '22' + + - name: Install Playwright + run: | + mkdir -p /tmp/partnerup-crawler + cd /tmp/partnerup-crawler + npm init -y + npm install --no-save playwright@1.55.0 + npx playwright install --with-deps chromium + + - name: Capture mobile site evidence + env: + NODE_PATH: /tmp/partnerup-crawler/node_modules + PARTNERUP_BASE_URL: https://app.partner-up.cn + SOURCE_COMMIT: 30f51b96e2498b4b85014552512a31a7345d86cb + MAX_PAGES: '35' + run: | + cp scripts/temporary-soft-copyright-manual-crawl.mjs /tmp/partnerup-crawler/crawl.mjs + node /tmp/partnerup-crawler/crawl.mjs + tar -czf soft-copyright-manual-evidence/apps-web-source.tar.gz \ + apps/web/package.json apps/web/src + git ls-tree -r --name-only 30f51b96e2498b4b85014552512a31a7345d86cb -- apps/web \ + > soft-copyright-manual-evidence/apps-web-files.txt + printf '%s\n' '30f51b96e2498b4b85014552512a31a7345d86cb' \ + > soft-copyright-manual-evidence/source-commit.txt + + - name: Upload evidence + uses: actions/upload-artifact@v4 + with: + name: partner-up-mobile-manual-evidence + path: soft-copyright-manual-evidence + if-no-files-found: error + retention-days: 7 From 54ece3c03d3c610f4ff597610414726b8ae974ec Mon Sep 17 00:00:00 2001 From: Lanzhijiang Date: Sun, 26 Jul 2026 08:44:51 +0800 Subject: [PATCH 3/6] chore: preserve crawler evidence on failure --- .../workflows/temporary-soft-copyright-manual.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/temporary-soft-copyright-manual.yml b/.github/workflows/temporary-soft-copyright-manual.yml index 5c550bfb..f4e18e1d 100644 --- a/.github/workflows/temporary-soft-copyright-manual.yml +++ b/.github/workflows/temporary-soft-copyright-manual.yml @@ -17,6 +17,8 @@ jobs: steps: - name: Checkout evidence branch uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set up Node uses: actions/setup-node@v4 @@ -32,22 +34,30 @@ jobs: npx playwright install --with-deps chromium - name: Capture mobile site evidence + continue-on-error: true env: NODE_PATH: /tmp/partnerup-crawler/node_modules PARTNERUP_BASE_URL: https://app.partner-up.cn SOURCE_COMMIT: 30f51b96e2498b4b85014552512a31a7345d86cb - MAX_PAGES: '35' + MAX_PAGES: '25' run: | + mkdir -p soft-copyright-manual-evidence cp scripts/temporary-soft-copyright-manual-crawl.mjs /tmp/partnerup-crawler/crawl.mjs - node /tmp/partnerup-crawler/crawl.mjs + set +e + node /tmp/partnerup-crawler/crawl.mjs 2>&1 | tee soft-copyright-manual-evidence/crawl.log + crawl_status=${PIPESTATUS[0]} + set -e tar -czf soft-copyright-manual-evidence/apps-web-source.tar.gz \ apps/web/package.json apps/web/src git ls-tree -r --name-only 30f51b96e2498b4b85014552512a31a7345d86cb -- apps/web \ > soft-copyright-manual-evidence/apps-web-files.txt printf '%s\n' '30f51b96e2498b4b85014552512a31a7345d86cb' \ > soft-copyright-manual-evidence/source-commit.txt + printf '%s\n' "$crawl_status" > soft-copyright-manual-evidence/crawl-exit-code.txt + exit "$crawl_status" - name: Upload evidence + if: always() uses: actions/upload-artifact@v4 with: name: partner-up-mobile-manual-evidence From 8840e4b8da4eafdea218f06f2a4c315c1dd10769 Mon Sep 17 00:00:00 2001 From: Lanzhijiang Date: Sun, 26 Jul 2026 09:10:31 +0800 Subject: [PATCH 4/6] chore: preserve crawler evidence on failure From ddf4c0a05b928150e6d0e06e6b44e0f22c06913f Mon Sep 17 00:00:00 2001 From: Lanzhijiang Date: Sun, 26 Jul 2026 09:10:40 +0800 Subject: [PATCH 5/6] chore: temporary cleanup marker --- .../temporary-soft-copyright-manual-crawl.mjs | 186 ------------------ 1 file changed, 186 deletions(-) diff --git a/scripts/temporary-soft-copyright-manual-crawl.mjs b/scripts/temporary-soft-copyright-manual-crawl.mjs index 7666db63..bbcd808f 100644 --- a/scripts/temporary-soft-copyright-manual-crawl.mjs +++ b/scripts/temporary-soft-copyright-manual-crawl.mjs @@ -1,187 +1 @@ import fs from 'node:fs'; -import path from 'node:path'; -import { chromium } from 'playwright'; - -const BASE_URL = process.env.PARTNERUP_BASE_URL || 'https://app.partner-up.cn'; -const OUT_DIR = path.resolve(process.env.CRAWL_OUT_DIR || 'soft-copyright-manual-evidence'); -const SRC_DIR = path.resolve('apps/web/src'); -const MAX_PAGES = Number(process.env.MAX_PAGES || 35); -const SOURCE_COMMIT = process.env.SOURCE_COMMIT || '30f51b96e2498b4b85014552512a31a7345d86cb'; - -fs.mkdirSync(OUT_DIR, { recursive: true }); -fs.mkdirSync(path.join(OUT_DIR, 'screenshots'), { recursive: true }); - -function walk(dir) { - const result = []; - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) result.push(...walk(full)); - else result.push(full); - } - return result; -} - -const sourceFiles = walk(SRC_DIR).filter((file) => /\.(?:ts|vue)$/.test(file)); -const routeEvidence = []; -const staticPaths = new Set(['/']); -const pathPattern = /\bpath\s*:\s*['"`]([^'"`]+)['"`]/g; -for (const file of sourceFiles) { - const text = fs.readFileSync(file, 'utf8'); - if (!/(?:createRouter|RouteRecordRaw|routes\s*=|children\s*:)/.test(text)) continue; - let match; - while ((match = pathPattern.exec(text))) { - const value = match[1].trim(); - routeEvidence.push({ file: path.relative(process.cwd(), file), path: value }); - if (value.startsWith('/') && !value.includes(':') && !value.includes('*')) staticPaths.add(value); - } -} - -for (const candidate of [ - '/create', '/discover', '/events', '/profile', '/me', '/about', '/privacy', '/terms', - '/admin/bi?code=2026zcb' -]) staticPaths.add(candidate); - -fs.writeFileSync(path.join(OUT_DIR, 'source-routes.json'), JSON.stringify(routeEvidence, null, 2)); -fs.writeFileSync(path.join(OUT_DIR, 'crawl-metadata.json'), JSON.stringify({ - baseUrl: BASE_URL, - sourceCommit: SOURCE_COMMIT, - viewport: { width: 390, height: 844, deviceScaleFactor: 2 }, - generatedAt: new Date().toISOString(), -}, null, 2)); - -function normalizeUrl(input) { - try { - const url = new URL(input, BASE_URL); - if (url.origin !== new URL(BASE_URL).origin) return null; - url.hash = ''; - for (const key of [...url.searchParams.keys()]) { - if (/^(?:code|token|state|ticket)$/i.test(key) && key !== 'code') url.searchParams.delete(key); - } - return url.toString(); - } catch { - return null; - } -} - -function safeName(value, index) { - const u = new URL(value); - const raw = `${String(index).padStart(2, '0')}-${u.pathname}${u.search}` - .replace(/^\/+/, '') - .replace(/[^a-zA-Z0-9._-]+/g, '-') - .replace(/-+/g, '-') - .replace(/^-|-$/g, ''); - return raw || `${String(index).padStart(2, '0')}-root`; -} - -const queue = [...staticPaths].map((p) => normalizeUrl(p)).filter(Boolean); -const queued = new Set(queue); -const visited = new Set(); -const pages = []; - -const browser = await chromium.launch({ headless: true }); -const context = await browser.newContext({ - viewport: { width: 390, height: 844 }, - deviceScaleFactor: 2, - isMobile: true, - hasTouch: true, - locale: 'zh-CN', - timezoneId: 'Asia/Shanghai', - userAgent: 'Mozilla/5.0 (Linux; Android 16; Mobile) AppleWebKit/537.36 Chrome/140.0 Mobile Safari/537.36', -}); -context.setDefaultTimeout(12000); - -while (queue.length && pages.length < MAX_PAGES) { - const requestedUrl = queue.shift(); - if (!requestedUrl || visited.has(requestedUrl)) continue; - visited.add(requestedUrl); - - const page = await context.newPage(); - const consoleErrors = []; - const failedRequests = []; - page.on('console', (msg) => { if (msg.type() === 'error') consoleErrors.push(msg.text()); }); - page.on('requestfailed', (req) => failedRequests.push({ url: req.url(), error: req.failure()?.errorText || '' })); - - let navigationError = null; - let responseStatus = null; - try { - const response = await page.goto(requestedUrl, { waitUntil: 'domcontentloaded', timeout: 30000 }); - responseStatus = response?.status() ?? null; - await page.waitForTimeout(2500); - } catch (error) { - navigationError = String(error); - } - - const finalUrl = page.url(); - let snapshot = {}; - try { - snapshot = await page.evaluate(() => { - const clean = (value) => (value || '').replace(/\s+/g, ' ').trim(); - const text = clean(document.body?.innerText || ''); - const collect = (selector, mapper) => [...document.querySelectorAll(selector)].map(mapper).filter(Boolean); - return { - title: document.title, - lang: document.documentElement.lang, - headings: collect('h1,h2,h3', (el) => ({ level: el.tagName, text: clean(el.textContent) })).slice(0, 80), - buttons: collect('button,[role="button"]', (el) => ({ - text: clean(el.textContent), - ariaLabel: el.getAttribute('aria-label'), - disabled: el.matches(':disabled,[aria-disabled="true"]'), - })).slice(0, 100), - links: collect('a[href]', (el) => ({ text: clean(el.textContent), href: el.href })).slice(0, 150), - fields: collect('input,textarea,select', (el) => ({ - tag: el.tagName, - type: el.getAttribute('type'), - name: el.getAttribute('name'), - placeholder: el.getAttribute('placeholder'), - ariaLabel: el.getAttribute('aria-label'), - })).slice(0, 100), - dialogs: collect('[role="dialog"],dialog', (el) => clean(el.textContent)).slice(0, 20), - text: text.slice(0, 12000), - bodyHeight: document.documentElement.scrollHeight, - }; - }); - } catch (error) { - snapshot = { evaluationError: String(error) }; - } - - const index = pages.length + 1; - const name = safeName(finalUrl || requestedUrl, index); - const viewportScreenshot = path.join(OUT_DIR, 'screenshots', `${name}-viewport.png`); - const fullScreenshot = path.join(OUT_DIR, 'screenshots', `${name}-full.png`); - try { await page.screenshot({ path: viewportScreenshot, fullPage: false }); } catch {} - try { await page.screenshot({ path: fullScreenshot, fullPage: true }); } catch {} - - const entry = { - index, - requestedUrl, - finalUrl, - responseStatus, - navigationError, - viewportScreenshot: path.relative(OUT_DIR, viewportScreenshot), - fullScreenshot: path.relative(OUT_DIR, fullScreenshot), - ...snapshot, - consoleErrors: consoleErrors.slice(0, 30), - failedRequests: failedRequests.slice(0, 30), - }; - pages.push(entry); - fs.writeFileSync(path.join(OUT_DIR, 'pages.json'), JSON.stringify(pages, null, 2)); - - for (const link of snapshot.links || []) { - const normalized = normalizeUrl(link.href); - if (!normalized || visited.has(normalized) || queued.has(normalized)) continue; - queued.add(normalized); - queue.push(normalized); - } - - await page.close(); -} - -await browser.close(); -fs.writeFileSync(path.join(OUT_DIR, 'pages.json'), JSON.stringify(pages, null, 2)); -fs.writeFileSync(path.join(OUT_DIR, 'crawl-summary.json'), JSON.stringify({ - requestedCount: visited.size, - capturedCount: pages.length, - discoveredCount: queued.size, - sourceRouteCount: routeEvidence.length, - sourceCommit: SOURCE_COMMIT, -}, null, 2)); From 2fd6c9d3b7ead48f16707979cea59bc51e5a2e8a Mon Sep 17 00:00:00 2001 From: Lanzhijiang Date: Sun, 26 Jul 2026 09:10:52 +0800 Subject: [PATCH 6/6] chore: restore temporary crawler stub before reset --- scripts/temporary-soft-copyright-manual-crawl.mjs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/temporary-soft-copyright-manual-crawl.mjs b/scripts/temporary-soft-copyright-manual-crawl.mjs index bbcd808f..e5903e2f 100644 --- a/scripts/temporary-soft-copyright-manual-crawl.mjs +++ b/scripts/temporary-soft-copyright-manual-crawl.mjs @@ -1 +1,5 @@ import fs from 'node:fs'; +import path from 'node:path'; +import { chromium } from 'playwright'; + +// Temporary read-only evidence crawler. This branch is reset after artifact collection.