diff --git a/.github/workflows/credibility-list-refresh.yml b/.github/workflows/credibility-list-refresh.yml new file mode 100644 index 00000000..800b6b43 --- /dev/null +++ b/.github/workflows/credibility-list-refresh.yml @@ -0,0 +1,104 @@ +name: Credibility List Refresh + +# Scheduled maintenance for the keyless search engine's domain-credibility list +# (src-tauri/src/websearch/credibility.rs). Regenerates the autogenerated +# regions of credibility_domains.txt from upstream license-clean sources and +# opens a reviewable PR when the diff is non-empty. Mirrors the existing +# Renovate engine-bump pattern (#238): automation proposes, a human reviews and +# merges, updates ship with releases. The app itself never fetches these lists +# at runtime; this is dev/CI tooling only. +# +# Note: this PR is opened with the default GITHUB_TOKEN, so per +# https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs +# it will not trigger this repo's other pull_request-triggered CI workflows. +# A reviewer must push an empty commit, or close and reopen the PR, to get CI +# to run on it before merging. + +on: + schedule: + # Weekly, 06:17 UTC on Mondays. Off-the-hour minute to avoid the + # scheduled-workflow thundering herd at :00. + - cron: '17 6 * * 1' + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + refresh: + name: Refresh credibility list + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 1 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.3.11 + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Run credibility list refresh + id: refresh + run: bun scripts/refresh-credibility-list.ts --summary refresh-summary.md + + - name: Append summary to job summary + if: always() + run: | + if [ -f refresh-summary.md ]; then + cat refresh-summary.md >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Check for changes + id: diff + run: | + if git diff --quiet -- src-tauri/src/websearch/credibility_domains.txt; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Open pull request + if: steps.diff.outputs.changed == 'true' + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + branch=chore/credibility-list-refresh + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git checkout -B "$branch" + git add src-tauri/src/websearch/credibility_domains.txt + git commit -s -m 'chore(search): refresh domain-credibility list' + git push --force origin "$branch" + + { + echo "Automated refresh of the autogenerated regions in" + echo "\`src-tauri/src/websearch/credibility_domains.txt\`" + echo "from upstream license-clean sources. See the summary below for" + echo "per-region counts and the added/removed domains." + echo "" + echo "This PR was opened with the default \`GITHUB_TOKEN\`, so it will" + echo "not trigger this repo's other pull_request-triggered CI" + echo "workflows. Push an empty commit, or close and reopen this PR," + echo "to run CI before merging." + echo "" + cat refresh-summary.md + } > pr-body.md + + existing=$(gh pr list --head "$branch" --state open --json number --jq '.[0].number // empty') + if [ -n "$existing" ]; then + gh pr edit "$existing" --body-file pr-body.md + else + gh pr create \ + --base main \ + --head "$branch" \ + --title 'chore(search): refresh domain-credibility list' \ + --body-file pr-body.md + fi diff --git a/docs/built-in-web-search.md b/docs/built-in-web-search.md index 9b412997..cf8b58b3 100644 --- a/docs/built-in-web-search.md +++ b/docs/built-in-web-search.md @@ -559,6 +559,8 @@ Code: `judge.rs`, `orchestrator.rs`. **Example.** Query `openai ceo` → DuckDuckGo ranks A,B,C; Mojeek ranks B,D → B’s RRF score wins → fetch B first. +**Maintenance.** The bulk-imported penalize clusters in `credibility_domains.txt` are refreshed by a scheduled GitHub Action that pulls the latest upstream license-clean spam/copycat lists and opens a reviewable PR when the diff is non-empty; a human still reviews and merges every change. The app itself never fetches these lists at runtime, so the no-phone-home privacy stance is unchanged; updates only ship inside a release. + Code: `engine.rs`, `credibility.rs`. --- diff --git a/docs/search-eval.md b/docs/search-eval.md index 480e0360..16858bd9 100644 --- a/docs/search-eval.md +++ b/docs/search-eval.md @@ -118,6 +118,10 @@ Deterministic-first, never pairwise, never a rubric score: Before trusting the gate, hand-label a sample of 30-40 rows spanning all three graded sources and volatility buckets with your own CORRECT/INCORRECT/NOT_ATTEMPTED judgment against the same predicted answers the harness generated, then run the live judge (majority-of-3) over that identical sample and compute agreement: the fraction of rows where the judge's majority verdict matches your hand label. This is the only way to know whether the local judge model is reliable enough for `CONFIDENTLY_WRONG_GATE` to mean anything, and is a prerequisite for tightening it. It needs a live model and a human rater (Logan), so it is a documented manual procedure, not code in this repository. +## Related automation + +These harnesses are dev-machine tools, run by hand as documented above; no CI workflow executes them. The domain-credibility list has its own scheduled refresh workflow (`.github/workflows/credibility-list-refresh.yml`); see [built-in-web-search.md](./built-in-web-search.md) for that automation. + ## Judging: pairwise superseded by absolute SimpleQA grading An earlier revision of this doc sketched pairwise, position-swapped LLM-as-judge scoring, following [Brave's published search-eval methodology](https://brave.com/blog/), as the intended next step once two capture runs existed to compare: an LLM judge shown both runs' answers in one order, then the swapped order, with a majority vote across both orderings deciding the winner or a tie. diff --git a/scripts/refresh-credibility-list.ts b/scripts/refresh-credibility-list.ts new file mode 100644 index 00000000..b5511c95 --- /dev/null +++ b/scripts/refresh-credibility-list.ts @@ -0,0 +1,565 @@ +// Refreshes the autogenerated regions of src-tauri/src/websearch/credibility_domains.txt +// from upstream license-clean ad/SEO-spam-blocklist sources. Run manually with +// `bun scripts/refresh-credibility-list.ts` or via the scheduled +// credibility-list-refresh GitHub Action (see .github/workflows/credibility-list-refresh.yml). +// +// Only text between "# BEGIN AUTOGEN " / "# END AUTOGEN " +// sentinel comments is rewritten; every other byte in the file (the drop section, +// the boost section, and the hand-curated penalize clusters) is left untouched. +// This mirrors the existing Renovate engine-bump pattern (#238): automation +// proposes a diff, a human reviews and merges it. The app itself never fetches +// these lists at runtime; this script is dev/CI tooling only. + +import { promises as dns } from 'node:dns'; +import { mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +/** Byte cap on any single fetched upstream response, a guard against a hung or hostile server. */ +const MAX_RESPONSE_BYTES = 5 * 1024 * 1024; + +/** Minimum domains a parsed region must yield; fewer trips the truncation/hijack guard. */ +const MIN_REGION_DOMAINS = 10; + +/** Domain count above which a region is flagged in the summary as suspiciously large. */ +const WARN_REGION_MAX_DOMAINS = 5000; + +/** DNS lookup timeout for the drop-section liveness check, per domain. */ +const DNS_TIMEOUT_MS = 4000; + +const repoRoot = fileURLToPath(new URL('../', import.meta.url)); +const CREDIBILITY_FILE = resolve( + repoRoot, + 'src-tauri/src/websearch/credibility_domains.txt', +); + +/** The two upstream line formats an autogen source can be published in. */ +type SourceFormat = 'ublacklist' | 'domain-list'; + +/** One upstream source feeding a single autogen sentinel region. */ +interface Source { + /** Sentinel id, must match "# BEGIN/END AUTOGEN " in the txt file. */ + id: string; + /** Raw-content URL fetched over HTTPS. */ + url: string; + /** Line format the fetched text is in. */ + format: SourceFormat; + /** One-line courtesy credit written at the top of the rewritten region. */ + credit: string; +} + +const SOURCES: readonly Source[] = [ + { + id: 'quenhus-seo-spam', + url: 'https://raw.githubusercontent.com/quenhus/uBlock-Origin-dev-filter/main/dist/other_format/domains/seo_spam.txt', + format: 'domain-list', + credit: + '# source: quenhus/uBlock-Origin-dev-filter (dist/other_format/domains/seo_spam.txt), Unlicense', + }, + { + id: 'quenhus-wikipedia-copycats', + url: 'https://raw.githubusercontent.com/quenhus/uBlock-Origin-dev-filter/main/dist/other_format/domains/wikipedia_copycats.txt', + format: 'domain-list', + credit: + '# source: quenhus/uBlock-Origin-dev-filter (dist/other_format/domains/wikipedia_copycats.txt), Unlicense', + }, + { + id: 'arosh-stackoverflow-copycats', + url: 'https://raw.githubusercontent.com/arosh/ublacklist-stackoverflow-translation/master/uBlacklist.txt', + format: 'ublacklist', + credit: + '# source: arosh/ublacklist-stackoverflow-translation (uBlacklist.txt), CC0-1.0', + }, +]; + +/** Verdict for one drop-list domain's liveness probe. */ +type LivenessVerdict = 'resolves' | 'possibly dead' | 'inconclusive'; + +/** + * Fetches `url` over HTTPS as text, aborting if the response body exceeds + * `maxBytes`. Streams the body rather than trusting a Content-Length header, + * since that header can be absent or wrong. Throws on a non-2xx status, a + * missing body, or the size cap being exceeded. + */ +async function fetchWithCap(url: string, maxBytes: number): Promise { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`fetch ${url} failed: HTTP ${response.status}`); + } + const body = response.body; + if (!body) { + throw new Error(`fetch ${url} returned no response body`); + } + const reader = body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) { + break; + } + total += value.byteLength; + if (total > maxBytes) { + await reader.cancel(); + throw new Error( + `fetch ${url} exceeded ${maxBytes} byte cap (truncated read at ${total} bytes)`, + ); + } + chunks.push(value); + } + const buffer = Buffer.concat(chunks.map((c) => Buffer.from(c))); + return buffer.toString('utf8'); +} + +/** + * Validates and normalizes one extracted domain token. Lowercases the token, + * strips any character outside `[a-z0-9.-]` (defense against comment injection + * or control characters riding in on a hijacked upstream file), drops a + * leading `www.` label (the file's own header contract is "no scheme or + * www"; a `www.`-only entry would not cover the bare domain via the parser's + * host-to-parent suffix walk, silently weakening coverage), then requires the + * result to look like a registrable domain: at least two dot-separated + * labels, each label alphanumeric-with-internal-hyphens only. Only the + * cosmetic `www.` prefix is stripped, not reduced to eTLD+1 in general: the + * list intentionally carries meaningful subdomains (e.g. + * `agecalculator.iamrohit.in`), which a general collapse would over-widen. + * Returns null for anything that fails the shape check, so the caller can + * drop it silently. + */ +function normalizeDomainToken(raw: string): string | null { + const lowered = raw.toLowerCase(); + const stripped = lowered.replace(/[^a-z0-9.-]/g, ''); + const withoutWww = stripped.replace(/^www\./, ''); + const labels = withoutWww.split('.'); + if (labels.length < 2) { + return null; + } + const labelPattern = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; + if (!labels.every((label) => label.length > 0 && labelPattern.test(label))) { + return null; + } + return withoutWww; +} + +/** + * Parses the arosh uBlacklist match-pattern format: lines like + * `*://code-examples.net/*` or a wildcard-subdomain form `*://*.voidcc.com/*`. + * Strips the `*://` scheme prefix, an optional leading `*.` wildcard label, + * and everything from the first `/` onward (path-scoped entries collapse to + * their host), then normalizes what remains as a domain token. + */ +function parseUblacklistFormat(text: string): string[] { + const domains: string[] = []; + for (const rawLine of text.split('\n')) { + const line = rawLine.trim(); + if (line.length === 0 || line.startsWith('#')) { + continue; + } + const withoutScheme = line.replace(/^\*:\/\//, ''); + const withoutWildcardSubdomain = withoutScheme.replace(/^\*\./, ''); + const host = withoutWildcardSubdomain.split('/')[0] ?? ''; + const normalized = normalizeDomainToken(host); + if (normalized) { + domains.push(normalized); + } + } + return domains; +} + +/** + * Parses the quenhus `dist/other_format/domains/*.txt` format: `#`-prefixed + * comments and blank lines are skipped, and each remaining line is a bare + * domain, optionally followed by a path (`brianlovin.com/hn`), which is + * dropped by keeping only the portion before the first `/`. + */ +function parseDomainListFormat(text: string): string[] { + const domains: string[] = []; + for (const rawLine of text.split('\n')) { + const line = rawLine.trim(); + if (line.length === 0 || line.startsWith('#')) { + continue; + } + const host = line.split('/')[0] ?? ''; + const normalized = normalizeDomainToken(host); + if (normalized) { + domains.push(normalized); + } + } + return domains; +} + +/** Parses `text` per `format` into a raw (unsorted, possibly duplicated) domain list. */ +function parseSource(text: string, format: SourceFormat): string[] { + return format === 'ublacklist' + ? parseUblacklistFormat(text) + : parseDomainListFormat(text); +} + +/** Lowercases, deduplicates, and lexicographically sorts a domain list for deterministic output. */ +function dedupeSort(domains: readonly string[]): string[] { + return Array.from(new Set(domains.map((d) => d.toLowerCase()))).sort(); +} + +/** One sentinel-delimited region located inside the credibility txt file. */ +interface Region { + /** Line index of the "# BEGIN AUTOGEN " marker. */ + beginIndex: number; + /** Line index of the "# END AUTOGEN " marker. */ + endIndex: number; + /** Domain lines currently between the markers (excludes credit comment lines). */ + domains: string[]; +} + +/** + * Locates the sentinel region for `sourceId` inside `lines` (the credibility + * file split on newlines). Returns the marker line indices plus the domain + * lines currently inside the region (any line starting with `#` between the + * markers is a courtesy-credit comment, not a domain, and is excluded). + * Throws if the markers are missing or malformed, since a missing sentinel + * means the file was hand-edited out of sync with this script's expectations. + */ +function findRegion(lines: readonly string[], sourceId: string): Region { + const beginMarker = `# BEGIN AUTOGEN ${sourceId}`; + const endMarker = `# END AUTOGEN ${sourceId}`; + const beginIndex = lines.indexOf(beginMarker); + const endIndex = lines.indexOf(endMarker); + if (beginIndex === -1 || endIndex === -1 || endIndex < beginIndex) { + throw new Error( + `could not locate sentinel region for "${sourceId}" (expected "${beginMarker}" / "${endMarker}")`, + ); + } + const domains = lines + .slice(beginIndex + 1, endIndex) + .map((l) => l.trim()) + .filter((l) => l.length > 0 && !l.startsWith('#')); + return { beginIndex, endIndex, domains }; +} + +/** + * Rewrites the sentinel region for `source` inside `lines`, replacing + * everything between its BEGIN/END markers with the courtesy credit line + * followed by the sorted `domains`. The markers themselves, and every line + * outside them, are left untouched. Returns a new line array. + */ +function replaceRegion( + lines: readonly string[], + source: Source, + domains: readonly string[], +): string[] { + const { beginIndex, endIndex } = findRegion(lines, source.id); + const replacement = [source.credit, ...domains]; + return [ + ...lines.slice(0, beginIndex + 1), + ...replacement, + ...lines.slice(endIndex), + ]; +} + +/** + * Parses the credibility file's `# drop` section into a domain list, mirroring + * the section-header handling in src-tauri/src/websearch/credibility.rs: a + * line reading exactly `# drop`, `# penalize`, or `# boost` switches section; + * any other `#` line (including a BEGIN/END sentinel) is a comment; a blank + * line is skipped; anything else inside the `# drop` section is a domain. + */ +function extractDropSection(fileText: string): string[] { + const domains: string[] = []; + let inDrop = false; + for (const rawLine of fileText.split('\n')) { + const line = rawLine.trim(); + if (line.length === 0) { + continue; + } + if (line.startsWith('#')) { + const header = line.slice(1).trim(); + if (header === 'drop' || header === 'penalize' || header === 'boost') { + inDrop = header === 'drop'; + } + continue; + } + if (inDrop) { + domains.push(line.toLowerCase()); + } + } + return domains; +} + +/** + * Resolves `domain` with a bounded timeout to classify it as still live, + * possibly dead, or inconclusive. A successful lookup is "resolves"; an + * ENOTFOUND/ENODATA error (the DNS server affirmatively has no record) is + * "possibly dead"; anything else (timeout, network failure, other resolver + * errors) is "inconclusive" because it does not prove the domain is gone. + */ +async function checkLiveness(domain: string): Promise { + let timeoutHandle: ReturnType | undefined; + const timeout = new Promise<'timeout'>((resolve) => { + timeoutHandle = setTimeout(() => resolve('timeout'), DNS_TIMEOUT_MS); + }); + try { + // Held separately from the race so a late rejection (arriving after the + // timeout branch has already won) still has a handler attached and does + // not surface as an unhandledRejection once main() has moved on. + const lookup = dns.lookup(domain); + lookup.catch(() => {}); + const outcome = await Promise.race([ + lookup.then(() => 'resolved' as const), + timeout, + ]); + if (outcome === 'timeout') { + return 'inconclusive'; + } + return 'resolves'; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOTFOUND' || code === 'ENODATA') { + return 'possibly dead'; + } + return 'inconclusive'; + } finally { + clearTimeout(timeoutHandle); + } +} + +/** One row of the drop-list liveness report. */ +interface LivenessResult { + domain: string; + verdict: LivenessVerdict; +} + +/** Runs `checkLiveness` over every drop-section domain, in parallel. */ +async function checkDropListLiveness( + domains: readonly string[], +): Promise { + return Promise.all( + domains.map(async (domain) => ({ + domain, + verdict: await checkLiveness(domain), + })), + ); +} + +/** Writes `content` to `path` atomically: write to a sibling temp file, then rename over the target. */ +async function writeFileAtomic(path: string, content: string): Promise { + const tmpDir = await mkdtemp(join(dirname(path), '.refresh-tmp-')); + const tmpPath = join(tmpDir, 'credibility_domains.txt'); + try { + await writeFile(tmpPath, content, 'utf8'); + await rename(tmpPath, path); + } finally { + await rm(tmpDir, { recursive: true, force: true }); + } +} + +/** Per-region before/after domain counts and set diff, for the summary. */ +interface RegionReport { + sourceId: string; + previousCount: number; + newCount: number; + added: string[]; + removed: string[]; + warnings: string[]; +} + +/** Diffs `previous` against `next`, returning domains only in one side. */ +function diffDomains( + previous: readonly string[], + next: readonly string[], +): { added: string[]; removed: string[] } { + const previousSet = new Set(previous); + const nextSet = new Set(next); + return { + added: next.filter((d) => !previousSet.has(d)).sort(), + removed: previous.filter((d) => !nextSet.has(d)).sort(), + }; +} + +/** + * Renders the markdown maintenance summary: per-section counts (drop / + * penalize / boost), per-region before/after counts with added/removed + * domains, any size-anomaly warnings, and the drop-list liveness report. + */ +function renderSummary( + sectionCounts: { drop: number; penalize: number; boost: number }, + regionReports: readonly RegionReport[], + liveness: readonly LivenessResult[], +): string { + const lines: string[] = []; + lines.push('# Credibility list refresh summary'); + lines.push(''); + lines.push('## Section counts'); + lines.push(`- drop: ${sectionCounts.drop}`); + lines.push(`- penalize: ${sectionCounts.penalize}`); + lines.push(`- boost: ${sectionCounts.boost}`); + lines.push(''); + lines.push('## Autogen regions'); + for (const report of regionReports) { + lines.push(`### ${report.sourceId}`); + lines.push( + `- domains: ${report.previousCount} -> ${report.newCount} (added ${report.added.length}, removed ${report.removed.length})`, + ); + for (const warning of report.warnings) { + lines.push(`- warning: ${warning}`); + } + if (report.added.length > 0) { + lines.push('
Added domains'); + lines.push(''); + lines.push(report.added.map((d) => `- ${d}`).join('\n')); + lines.push(''); + lines.push('
'); + } + if (report.removed.length > 0) { + lines.push('
Removed domains'); + lines.push(''); + lines.push(report.removed.map((d) => `- ${d}`).join('\n')); + lines.push(''); + lines.push('
'); + } + lines.push(''); + } + lines.push('## Drop-list liveness (report only, list never modified)'); + const possiblyDead = liveness.filter((r) => r.verdict === 'possibly dead'); + const inconclusive = liveness.filter((r) => r.verdict === 'inconclusive'); + lines.push( + `- ${liveness.length} domains checked: ${liveness.length - possiblyDead.length - inconclusive.length} resolve, ${possiblyDead.length} possibly dead, ${inconclusive.length} inconclusive`, + ); + if (possiblyDead.length > 0) { + lines.push( + '- possibly dead: ' + possiblyDead.map((r) => r.domain).join(', '), + ); + } + if (inconclusive.length > 0) { + lines.push( + '- inconclusive: ' + inconclusive.map((r) => r.domain).join(', '), + ); + } + lines.push(''); + return lines.join('\n'); +} + +/** Reads `--summary ` from argv, if present. */ +function parseArgs(argv: readonly string[]): { summaryPath: string | null } { + const idx = argv.indexOf('--summary'); + if (idx === -1 || idx + 1 >= argv.length) { + return { summaryPath: null }; + } + return { summaryPath: argv[idx + 1] ?? null }; +} + +/** + * Entry point: fetches every upstream source, parses and validates it, + * rewrites only the matching sentinel regions of the credibility file, runs + * the drop-list liveness check, and writes the markdown summary (to stdout + * always, and to `--summary ` if given). Exits nonzero without touching + * the credibility file if any region parses to fewer than MIN_REGION_DOMAINS + * domains, since that indicates upstream truncation or hijack. + */ +async function main(): Promise { + const { summaryPath } = parseArgs(process.argv.slice(2)); + const originalText = await readFile(CREDIBILITY_FILE, 'utf8'); + const originalLines = originalText.split('\n'); + + const regionReports: RegionReport[] = []; + let updatedLines = originalLines; + + for (const source of SOURCES) { + const previousRegion = findRegion(originalLines, source.id); + const fetched = await fetchWithCap(source.url, MAX_RESPONSE_BYTES); + const parsed = parseSource(fetched, source.format); + const domains = dedupeSort(parsed); + if (domains.length < MIN_REGION_DOMAINS) { + throw new Error( + `source "${source.id}" parsed to only ${domains.length} domains (minimum ${MIN_REGION_DOMAINS}); ` + + 'refusing to write, this looks like upstream truncation or a hijacked response', + ); + } + const warnings: string[] = []; + if (domains.length > WARN_REGION_MAX_DOMAINS) { + warnings.push( + `region has ${domains.length} domains, above the ${WARN_REGION_MAX_DOMAINS} sanity threshold`, + ); + } + if ( + previousRegion.domains.length > 0 && + domains.length < previousRegion.domains.length / 2 + ) { + warnings.push( + `region shrank from ${previousRegion.domains.length} to ${domains.length} domains (more than half)`, + ); + } + const { added, removed } = diffDomains(previousRegion.domains, domains); + regionReports.push({ + sourceId: source.id, + previousCount: previousRegion.domains.length, + newCount: domains.length, + added, + removed, + warnings, + }); + updatedLines = replaceRegion(updatedLines, source, domains); + } + + const updatedText = updatedLines.join('\n'); + const dropDomains = extractDropSection(updatedText); + const penalizeSectionCount = countSectionDomains(updatedText, 'penalize'); + const boostSectionCount = countSectionDomains(updatedText, 'boost'); + + const liveness = await checkDropListLiveness(dropDomains); + + const summary = renderSummary( + { + drop: dropDomains.length, + penalize: penalizeSectionCount, + boost: boostSectionCount, + }, + regionReports, + liveness, + ); + + await writeFileAtomic(CREDIBILITY_FILE, updatedText); + + process.stdout.write(summary + '\n'); + if (summaryPath) { + await writeFile(resolve(summaryPath), summary, 'utf8'); + } +} + +/** + * Counts domains in a named section (`drop` / `penalize` / `boost`), the same + * way `extractDropSection` does for `drop`, generalized to any of the three. + */ +function countSectionDomains(fileText: string, section: string): number { + let count = 0; + let inSection = false; + for (const rawLine of fileText.split('\n')) { + const line = rawLine.trim(); + if (line.length === 0) { + continue; + } + if (line.startsWith('#')) { + const header = line.slice(1).trim(); + if (header === 'drop' || header === 'penalize' || header === 'boost') { + inSection = header === section; + } + continue; + } + if (inSection) { + count += 1; + } + } + return count; +} + +main().catch(async (err) => { + const message = err instanceof Error ? err.message : String(err); + process.stderr.write(`refresh-credibility-list: ${message}\n`); + const { summaryPath } = parseArgs(process.argv.slice(2)); + const failureSummary = `# Credibility list refresh summary\n\nRefresh failed, credibility_domains.txt was not modified:\n\n\`\`\`\n${message}\n\`\`\`\n`; + process.stdout.write(failureSummary); + if (summaryPath) { + await writeFile(resolve(summaryPath), failureSummary, 'utf8').catch( + () => undefined, + ); + } + process.exitCode = 1; +}); diff --git a/src-tauri/src/websearch/credibility_domains.txt b/src-tauri/src/websearch/credibility_domains.txt index 7bbb035f..f20d589e 100644 --- a/src-tauri/src/websearch/credibility_domains.txt +++ b/src-tauri/src/websearch/credibility_domains.txt @@ -7,15 +7,17 @@ # Lines beginning with # other than the three section headers are courtesy source # credits and are ignored by the parser. Every domain traces to a CC0/Unlicense # source or individual editorial verification; no attribution is required. +# Regions between "# BEGIN AUTOGEN " and "# END AUTOGEN " are +# rewritten verbatim by scripts/refresh-credibility-list.ts; do not hand-edit domains +# inside them, hand-edit the upstream list instead. Everything outside those regions +# is hand-curated and left untouched by the script. # drop # cluster: editorial judgment -- impostor/hoax "news" domains, individually verified active 2026-07-10 via en.wikipedia.org/wiki/List_of_fake_news_websites (cross-cited PolitiFact/NewsGuard/Snopes/Lead Stories); not a bulk copy of that page actionnews3.com -breaking13news.com dailybuzzlive.com news4ktla.com -news4local.com now8news.com channel22news.com channel24news.com @@ -32,18 +34,19 @@ current-affairs.org # cluster: editorial judgment -- 2026-07-14 gold-price smoke (giá vàng hôm nay): SEO scrapes / # free-blog hosts that assembled stale "SJC 80 triệu" / 2020 USD quotes ahead of real VN hubs blogdanica.com -globaleasyforex.com ponselharian.com kulturellen.net # penalize -# cluster: quenhus/uBlock-Origin-dev-filter (dist/duckduckgo/seo_spam.txt) -- Unlicense +# BEGIN AUTOGEN quenhus-seo-spam +# source: quenhus/uBlock-Origin-dev-filter (dist/other_format/domains/seo_spam.txt), Unlicense 900913.ru actingcollegeses.com answerforyou.net azazworld.com bong-faq.com +brianlovin.com britguidenewyork.net code-discuss.com codertw.com @@ -77,10 +80,13 @@ thesassway.com topcode.in unbate.com worldgrowthtoday.com +# END AUTOGEN quenhus-seo-spam -# cluster: quenhus/uBlock-Origin-dev-filter (dist/duckduckgo/wikipedia_copycats.txt) -- Unlicense +# BEGIN AUTOGEN quenhus-wikipedia-copycats +# source: quenhus/uBlock-Origin-dev-filter (dist/other_format/domains/wikipedia_copycats.txt), Unlicense 360wiki.ru accordeonmuseum.nl +algebra.com buildwiki.ru cyclowiki.org datewiki.ru @@ -98,7 +104,9 @@ hmong.ru livepcwiki.ru mediawiki.feverous.co.uk ru-wiki.ru +scholarship.edu.vn second.wiki +secret-bases.co.uk static.hlt.bme.hu sv.abcdef.wiki th.hmong.wiki @@ -122,51 +130,77 @@ wikiwand.com wikizero.com wiwa.wiki zxc.wiki +# END AUTOGEN quenhus-wikipedia-copycats -# cluster: arosh/ublacklist-stackoverflow-translation (uBlacklist.txt) -- CC0-1.0 +# BEGIN AUTOGEN arosh-stackoverflow-copycats +# source: arosh/ublacklist-stackoverflow-translation (uBlacklist.txt), CC0-1.0 16892.net 1r1g.com 55276.net +5axxw.com 711web.com 9ishenzhen.com 9to5answer.com ajaxhispano.com answer-id.com +answerlib.com +anycodings.com +appsloveworld.com arip-photo.org ask-dev.ru askcodez.com +binarydevelop.com bitcoden.com cainiaojiaocheng.com code-examples.net +codebaoku.com codegrepr.com codeguides.site +codenong.com +coder.work coderoad.ru codeutility.org copyprogramming.com daplus.net de-vraag.com +debugcn.com +debugko.com +desenv-web-rp.com developreference.com digitrain.ru doraprojects.net dovov.com +edureka.co errorsfixing.com exchangetuts.com +fixes.pub fluffyfables.com flutterhq.com fmihm.org fullstackuser.com +generacodice.com +intellipaat.com iquestion.pro isolution.pro +it-swarm-fr.com +it-swarm-ja.com +it-swarm-ja.tech +it-swarm.jp.net itecnote.com itecnotes.com itectec.com iteramos.com +javaer101.com +javafixing.com jike.in jonic.cn jtuto.com +kutombawewe.net kzen.dev +linuxfixes.com living-sun.com mediatagtw.com +mejorcodigo.com microeducate.tech mlink.in narkive.jp @@ -196,9 +230,11 @@ qastack.net.bd qastack.ru qastack.vn qi-u.com +querythreads.com question-it.com questu.ru routinepanic.com +semicolonworld.com shenghuobao.net shenzhenjia.cn shenzhenjia.net @@ -207,6 +243,7 @@ softwareuser.asklobster.com solveforum.com splunktool.com sqlite.in +stackfinder.ru stackguides.com stackoom.com stackovergo.com @@ -221,45 +258,21 @@ vigge.cn vigge.net vigges.net voidcc.com -web-dev-qa-db-fra.com -webdevask.com -webdevdesigner.com -wujigu.com -5axxw.com -answerlib.com -anycodings.com -appsloveworld.com -binarydevelop.com -codebaoku.com -codenong.com -coder.work -debugcn.com -debugko.com -desenv-web-rp.com -fixes.pub -generacodice.com -it-swarm-fr.com -it-swarm-ja.com -it-swarm-ja.tech -it-swarm.jp.net -javaer101.com -javafixing.com -kutombawewe.net -linuxfixes.com -mejorcodigo.com -querythreads.com -semicolonworld.com -stackfinder.ru wake-up-neo.net web-dev-qa-db-fr.com +web-dev-qa-db-fra.com web-dev-qa-db-ja.com web-dev-qa-db-pt.com web-dev-qa.com +webdevask.com +webdevdesigner.com +wujigu.com xstack.ru xstack.us yaoply.com zaizhele.cn zaizhele.net +# END AUTOGEN arosh-stackoverflow-copycats # cluster: editorial judgment -- thin finance/ranking-content aggregators (user-observed offenders; only 2 independently confirmed, category needs expansion -- see report) wealthrank.in