diff --git a/scripts/seed-regulatory-actions.mjs b/scripts/seed-regulatory-actions.mjs new file mode 100644 index 0000000000..844105b741 --- /dev/null +++ b/scripts/seed-regulatory-actions.mjs @@ -0,0 +1,269 @@ +#!/usr/bin/env node +// @ts-check + +import { pathToFileURL } from 'node:url'; +import { CHROME_UA } from './_seed-utils.mjs'; + +const FEED_TIMEOUT_MS = 15_000; +const XML_ACCEPT = 'application/atom+xml, application/rss+xml, application/xml, text/xml, */*'; +const SEC_USER_AGENT = 'WorldMonitor/2.0 (monitor@worldmonitor.app)'; +const DEFAULT_FETCH = (...args) => globalThis.fetch(...args); + +const REGULATORY_FEEDS = [ + { agency: 'SEC', url: 'https://www.sec.gov/news/pressreleases.rss', userAgent: SEC_USER_AGENT }, + { agency: 'CFTC', url: 'https://www.cftc.gov/RSS/RSSENF/rssenf.xml' }, + { agency: 'CFTC', url: 'https://www.cftc.gov/RSS/RSSGP/rssgp.xml' }, + { agency: 'Federal Reserve', url: 'https://www.federalreserve.gov/feeds/press_all.xml' }, + { agency: 'FDIC', url: 'https://public.govdelivery.com/topics/USFDIC_26/feed.rss' }, + // FINRA still publishes this RSS endpoint over plain HTTP; HTTPS requests fail + // from both Node fetch and curl in validation, so keep the official feed URL + // and periodically recheck whether HTTPS starts working. + { agency: 'FINRA', url: 'http://feeds.finra.org/FINRANotices' }, +]; + +function decodeEntities(input) { + if (!input) return ''; + const named = input + .replace(/&/gi, '&') + .replace(/</gi, '<') + .replace(/>/gi, '>') + .replace(/"/gi, '"') + .replace(/'/gi, "'") + .replace(/ /gi, ' '); + + return named + .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code))) + .replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCodePoint(parseInt(code, 16))); +} + +function stripHtml(input) { + const unwrapped = String(input || '').replace(//g, '$1'); + const decoded = decodeEntities(unwrapped); + return decoded.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim(); +} + +function getTagValue(block, tagName) { + const match = block.match(new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>`, 'i')); + return stripHtml(match?.[1] || ''); +} + +function extractAtomLink(block) { + const linkTags = [...block.matchAll(/]*)\/?>/gi)]; + if (linkTags.length === 0) return ''; + + for (const [, attrs] of linkTags) { + const href = attrs.match(/\bhref=["']([^"']+)["']/i)?.[1]; + const rel = attrs.match(/\brel=["']([^"']+)["']/i)?.[1]?.toLowerCase() || ''; + if (href && (!rel || rel === 'alternate')) return decodeEntities(href.trim()); + } + + for (const [, attrs] of linkTags) { + const href = attrs.match(/\bhref=["']([^"']+)["']/i)?.[1]; + if (href) return decodeEntities(href.trim()); + } + + return ''; +} + +function resolveFeedLink(link, feedUrl) { + if (!link) return ''; + try { + return new URL(link).href; + } catch {} + try { + return new URL(link, feedUrl).href; + } catch { + return ''; + } +} + +function canonicalizeLink(link, feedUrl = '') { + const resolved = resolveFeedLink(link, feedUrl); + if (!resolved) return ''; + try { + const url = new URL(resolved); + url.hash = ''; + return url.href; + } catch { + return ''; + } +} + +function toIsoDate(rawDate) { + const value = stripHtml(rawDate); + if (!value) return ''; + const ts = Date.parse(value); + return Number.isFinite(ts) ? new Date(ts).toISOString() : ''; +} + +function slugifyTitle(title) { + return stripHtml(title) + .normalize('NFKD') + .replace(/[\u0300-\u036f]/g, '') + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 80); +} + +function yyyymmdd(isoDate) { + return String(isoDate || '').slice(0, 10).replace(/-/g, ''); +} + +function hhmmss(isoDate) { + return String(isoDate || '').slice(11, 19).replace(/:/g, ''); +} + +function buildActionId(agency, title, publishedAt) { + const agencySlug = slugifyTitle(agency) || 'agency'; + const titleSlug = slugifyTitle(title) || 'untitled'; + const datePart = yyyymmdd(publishedAt) || 'undated'; + const timePart = hhmmss(publishedAt) || '000000'; + return `${agencySlug}-${titleSlug}-${datePart}-${timePart}`; +} + +function parseRssItems(xml, feedUrl) { + const items = []; + const itemRegex = /]*>([\s\S]*?)<\/item>/gi; + let match; + while ((match = itemRegex.exec(xml)) !== null) { + const block = match[1]; + const title = getTagValue(block, 'title'); + const description = getTagValue(block, 'description'); + const link = canonicalizeLink(getTagValue(block, 'link'), feedUrl); + const publishedAt = toIsoDate(getTagValue(block, 'pubDate') || getTagValue(block, 'updated')); + items.push({ title, description, link, publishedAt }); + } + return items; +} + +function parseAtomEntries(xml, feedUrl) { + const entries = []; + const entryRegex = /]*>([\s\S]*?)<\/entry>/gi; + let match; + while ((match = entryRegex.exec(xml)) !== null) { + const block = match[1]; + const title = getTagValue(block, 'title'); + const description = getTagValue(block, 'summary') || getTagValue(block, 'content'); + const link = canonicalizeLink(extractAtomLink(block), feedUrl); + const publishedAt = toIsoDate( + getTagValue(block, 'updated') || getTagValue(block, 'published') || getTagValue(block, 'pubDate') + ); + entries.push({ title, description, link, publishedAt }); + } + return entries; +} + +function parseFeed(xml, feedUrl) { + if (/ item.title && item.link && item.publishedAt) + .map((item) => ({ + id: buildActionId(agency, item.title, item.publishedAt), + agency, + title: item.title, + description: item.description || '', + link: item.link, + publishedAt: item.publishedAt, + })); +} + +function dedupeAndSortActions(actions) { + const seen = new Set(); + const deduped = []; + for (const action of actions) { + const key = canonicalizeLink(action.link); + if (!key || seen.has(key)) continue; + seen.add(key); + deduped.push({ ...action, link: key }); + } + + deduped.sort((a, b) => Date.parse(b.publishedAt) - Date.parse(a.publishedAt)); + return deduped; +} + +async function fetchFeed(feed, fetchImpl = DEFAULT_FETCH) { + const headers = { + Accept: XML_ACCEPT, + 'User-Agent': feed.userAgent || CHROME_UA, + }; + + const response = await fetchImpl(feed.url, { + headers, + signal: AbortSignal.timeout(FEED_TIMEOUT_MS), + }); + + if (!response.ok) { + throw new Error(`${feed.agency}: HTTP ${response.status}`); + } + + const xml = await response.text(); + const parsed = parseFeed(xml, feed.url); + return normalizeFeedItems(parsed, feed.agency); +} + +async function fetchAllFeeds(fetchImpl = DEFAULT_FETCH, feeds = REGULATORY_FEEDS) { + const results = await Promise.allSettled(feeds.map((feed) => fetchFeed(feed, fetchImpl))); + const actions = []; + let successCount = 0; + + for (let index = 0; index < results.length; index += 1) { + const result = results[index]; + const feed = feeds[index]; + if (result.status === 'fulfilled') { + successCount += 1; + actions.push(...result.value); + continue; + } + console.error(`[regulatory] ${feed.agency}: ${result.reason?.message || result.reason}`); + } + + if (successCount === 0) { + throw new Error('All regulatory feeds failed'); + } + + return dedupeAndSortActions(actions); +} + +async function main(fetchImpl = DEFAULT_FETCH) { + const actions = await fetchAllFeeds(fetchImpl); + process.stdout.write(`${JSON.stringify(actions, null, 2)}\n`); + return actions; +} + +const isDirectRun = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; + +if (isDirectRun) { + main().catch((err) => { + console.error(`FETCH FAILED: ${err.message || err}`); + process.exit(1); + }); +} + +export { + CHROME_UA, + FEED_TIMEOUT_MS, + REGULATORY_FEEDS, + SEC_USER_AGENT, + buildActionId, + canonicalizeLink, + decodeEntities, + dedupeAndSortActions, + extractAtomLink, + fetchAllFeeds, + fetchFeed, + getTagValue, + main, + normalizeFeedItems, + parseAtomEntries, + parseFeed, + parseRssItems, + resolveFeedLink, + slugifyTitle, + stripHtml, + toIsoDate, +}; diff --git a/tests/regulatory-seed-unit.test.mjs b/tests/regulatory-seed-unit.test.mjs new file mode 100644 index 0000000000..28dee14b2c --- /dev/null +++ b/tests/regulatory-seed-unit.test.mjs @@ -0,0 +1,213 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import vm from 'node:vm'; + +function normalize(value) { + return JSON.parse(JSON.stringify(value)); +} + +const seedSrc = readFileSync('scripts/seed-regulatory-actions.mjs', 'utf8'); + +const pureSrc = seedSrc + .replace(/^import\s.*$/gm, '') + .replace(/const isDirectRun[\s\S]*?}\n\nexport\s*{[\s\S]*?};?\s*$/m, ''); + +const ctx = vm.createContext({ + console, + Date, + Math, + Number, + Array, + Set, + String, + RegExp, + URL, + URLSearchParams, + AbortSignal, + CHROME_UA: 'Mozilla/5.0 (test)', +}); + +vm.runInContext(pureSrc, ctx); + +const { + decodeEntities, + stripHtml, + extractAtomLink, + parseRssItems, + parseAtomEntries, + parseFeed, + normalizeFeedItems, + dedupeAndSortActions, + fetchAllFeeds, +} = ctx; + +describe('decodeEntities', () => { + it('decodes named and numeric entities', () => { + assert.equal(decodeEntities('Tom & Jerry & &'), 'Tom & Jerry & &'); + }); +}); + +describe('stripHtml', () => { + it('removes tags and CDATA while preserving text', () => { + assert.equal(stripHtml('world]]>'), 'Hello world'); + }); + + it('strips entity-escaped HTML tags (FINRA-style descriptions)', () => { + assert.equal(stripHtml('<h2>Summary</h2><p>FINRA amends Rule 4210.</p>'), 'Summary FINRA amends Rule 4210.'); + }); +}); + +describe('parseRssItems', () => { + it('extracts RSS items with description, normalized links, and pubDate', () => { + const xml = ` + + + <![CDATA[SEC & Co. Charges <b>Issuer</b>]]> + fraud & disclosure failures]]> + /news/press-release/2026-10 + Mon, 30 Mar 2026 18:00:00 GMT + + `; + + assert.deepEqual(normalize(parseRssItems(xml, 'https://www.sec.gov/news/pressreleases.rss')), [{ + title: 'SEC & Co. Charges Issuer', + description: 'Alleges fraud & disclosure failures', + link: 'https://www.sec.gov/news/press-release/2026-10', + publishedAt: '2026-03-30T18:00:00.000Z', + }]); + }); +}); + +describe('extractAtomLink + parseAtomEntries', () => { + it('prefers alternate href and extracts summary/content with normalized publishedAt', () => { + const xml = ` + + + Fed issues notice + policy summary]]> + + + 2026-03-29T12:30:00Z + + `; + + assert.equal( + extractAtomLink(''), + '/press/notice-a' + ); + + assert.deepEqual(normalize(parseAtomEntries(xml, 'https://www.federalreserve.gov/feeds/press_all.xml')), [{ + title: 'Fed issues notice', + description: 'Detailed policy summary', + link: 'https://www.federalreserve.gov/press/notice-a', + publishedAt: '2026-03-29T12:30:00.000Z', + }]); + + const contentXml = ` + + + FDIC update + Formal administrative note

]]> + + 2026-03-28T09:15:00Z + + `; + + assert.deepEqual(normalize(parseAtomEntries(contentXml, 'https://www.fdic.gov/feed')), [{ + title: 'FDIC update', + description: 'Formal administrative note', + link: 'https://fdic.example.test/a', + publishedAt: '2026-03-28T09:15:00.000Z', + }]); + }); +}); + +describe('parseFeed', () => { + it('detects Atom feeds automatically', () => { + const atom = 'A2026-03-28T00:00:00Z'; + const parsed = normalize(parseFeed(atom, 'https://example.test/feed')); + assert.equal(parsed.length, 1); + assert.equal(parsed[0].link, 'https://example.test/a'); + }); +}); + +describe('normalizeFeedItems', () => { + it('skips incomplete entries and generates deterministic ids', () => { + const normalized = normalize(normalizeFeedItems([ + { title: 'SEC Charges XYZ Corp', link: 'https://example.test/sec', publishedAt: '2026-03-29T14:00:00.000Z' }, + { title: 'SEC Summary', description: 'extra context', link: 'https://example.test/sec-2', publishedAt: '2026-03-29T14:30:00.000Z' }, + { title: '', link: 'https://example.test/missing', publishedAt: '2026-03-29T14:00:00.000Z' }, + ], 'SEC')); + + assert.equal(normalized.length, 2); + assert.equal(normalized[0].id, 'sec-sec-charges-xyz-corp-20260329-140000'); + assert.equal(normalized[0].description, ''); + assert.equal(normalized[1].description, 'extra context'); + }); +}); + +describe('dedupeAndSortActions', () => { + it('deduplicates by canonical link and sorts newest first', () => { + const actions = normalize(dedupeAndSortActions([ + { + id: 'older', + agency: 'SEC', + title: 'Older', + link: 'https://example.test/path#frag', + publishedAt: '2026-03-28T10:00:00.000Z', + }, + { + id: 'newer', + agency: 'FDIC', + title: 'Newer', + link: 'https://example.test/new', + publishedAt: '2026-03-30T10:00:00.000Z', + }, + { + id: 'duplicate', + agency: 'SEC', + title: 'Duplicate', + link: 'https://example.test/path', + publishedAt: '2026-03-29T10:00:00.000Z', + }, + ])); + + assert.deepEqual(actions.map((item) => item.id), ['newer', 'older']); + assert.equal(actions[1].link, 'https://example.test/path'); + }); +}); + +describe('fetchAllFeeds', () => { + const feeds = [ + { agency: 'SEC', url: 'https://feeds.test/sec', userAgent: 'Custom-SEC-UA' }, + { agency: 'FDIC', url: 'https://feeds.test/fdic' }, + ]; + + it('returns normalized aggregate when at least one feed succeeds', async () => { + const requests = []; + const fetchStub = async (url, options) => { + requests.push({ url, options }); + if (url.endsWith('/sec')) { + return { + ok: true, + text: async () => `SEC Charges Bankhttps://sec.test/aMon, 30 Mar 2026 18:00:00 GMT`, + }; + } + throw new Error('FDIC timeout'); + }; + + const result = normalize(await fetchAllFeeds(fetchStub, feeds)); + assert.equal(result.length, 1); + assert.equal(result[0].agency, 'SEC'); + assert.equal(requests[0].options.headers['User-Agent'], 'Custom-SEC-UA'); + assert.equal(requests[1].options.headers['User-Agent'], ctx.CHROME_UA); + }); + + it('throws when all feeds fail', async () => { + await assert.rejects( + fetchAllFeeds(async () => { throw new Error('nope'); }, feeds), + /All regulatory feeds failed/ + ); + }); +});