From cc5903d007cdd2d5299837859eed1b3bac2546bc Mon Sep 17 00:00:00 2001 From: lspassos1 Date: Tue, 31 Mar 2026 00:34:02 +0100 Subject: [PATCH 01/12] feat(regulatory): add regulatory RSS fetch seeder Add a standalone seeder that fetches and normalizes SEC, CFTC, Federal Reserve, FDIC, and FINRA regulatory feeds without introducing new dependencies. The script stays import-safe, tolerates partial feed failure, and emits JSON for the fetch/parse-only phase of the pipeline. Unit tests cover RSS/Atom parsing, deduplication, ordering, and degraded-feed behavior. Refs #2492 Refs #2493 Refs #2494 Refs #2495 --- scripts/seed-regulatory-actions.mjs | 257 ++++++++++++++++++++++++++++ tests/regulatory-seed-unit.test.mjs | 185 ++++++++++++++++++++ 2 files changed, 442 insertions(+) create mode 100644 scripts/seed-regulatory-actions.mjs create mode 100644 tests/regulatory-seed-unit.test.mjs diff --git a/scripts/seed-regulatory-actions.mjs b/scripts/seed-regulatory-actions.mjs new file mode 100644 index 000000000..569de5366 --- /dev/null +++ b/scripts/seed-regulatory-actions.mjs @@ -0,0 +1,257 @@ +#!/usr/bin/env node + +import { pathToFileURL } from 'node:url'; +import { CHROME_UA } from './_seed-utils.mjs'; + +const FEED_TIMEOUT_MS = 15_000; +const XML_ACCEPT = 'application/atom+xml, application/rss+xml, application/xml, text/xml, */*'; +const SEC_USER_AGENT = 'WorldMonitor/2.0 (monitor@worldmonitor.app)'; + +const REGULATORY_FEEDS = [ + { agency: 'SEC', url: 'https://www.sec.gov/news/pressreleases.rss', userAgent: SEC_USER_AGENT }, + { agency: 'CFTC', url: 'https://www.cftc.gov/RSS/RSSENF/rssenf.xml' }, + { agency: 'Federal Reserve', url: 'https://www.federalreserve.gov/feeds/press_all.xml' }, + { agency: 'FDIC', url: 'https://public.govdelivery.com/topics/USFDIC_26/feed.rss' }, + { agency: 'FINRA', url: 'http://feeds.finra.org/FINRANotices' }, +]; + +function decodeEntities(input) { + if (!input) return ''; + const named = input + .replace(/&/gi, '&') + .replace(/</gi, '<') + .replace(/>/gi, '>') + .replace(/"/gi, '"') + .replace(/'/gi, "'") + .replace(/ /gi, ' '); + + return named + .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code))) + .replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCodePoint(parseInt(code, 16))); +} + +function stripHtml(input) { + return decodeEntities( + String(input || '') + .replace(//g, '$1') + .replace(/<[^>]+>/g, ' ') + ).replace(/\s+/g, ' ').trim(); +} + +function getTagValue(block, tagName) { + const match = block.match(new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>`, 'i')); + return stripHtml(match?.[1] || ''); +} + +function extractAtomLink(block) { + const linkTags = [...block.matchAll(/]*)\/?>/gi)]; + if (linkTags.length === 0) return ''; + + for (const [, attrs] of linkTags) { + const href = attrs.match(/\bhref=["']([^"']+)["']/i)?.[1]; + const rel = attrs.match(/\brel=["']([^"']+)["']/i)?.[1]?.toLowerCase() || ''; + if (href && (!rel || rel === 'alternate')) return decodeEntities(href.trim()); + } + + for (const [, attrs] of linkTags) { + const href = attrs.match(/\bhref=["']([^"']+)["']/i)?.[1]; + if (href) return decodeEntities(href.trim()); + } + + return ''; +} + +function resolveFeedLink(link, feedUrl) { + if (!link) return ''; + try { + return new URL(link).href; + } catch {} + try { + return new URL(link, feedUrl).href; + } catch { + return ''; + } +} + +function canonicalizeLink(link, feedUrl = '') { + const resolved = resolveFeedLink(link, feedUrl); + if (!resolved) return ''; + try { + const url = new URL(resolved); + url.hash = ''; + return url.href; + } catch { + return ''; + } +} + +function toIsoDate(rawDate) { + const value = stripHtml(rawDate); + if (!value) return ''; + const ts = Date.parse(value); + return Number.isFinite(ts) ? new Date(ts).toISOString() : ''; +} + +function slugifyTitle(title) { + return stripHtml(title) + .normalize('NFKD') + .replace(/[\u0300-\u036f]/g, '') + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 80); +} + +function yyyymmdd(isoDate) { + return String(isoDate || '').slice(0, 10).replace(/-/g, ''); +} + +function buildActionId(agency, title, publishedAt) { + const agencySlug = slugifyTitle(agency) || 'agency'; + const titleSlug = slugifyTitle(title) || 'untitled'; + const datePart = yyyymmdd(publishedAt) || 'undated'; + return `${agencySlug}-${titleSlug}-${datePart}`; +} + +function parseRssItems(xml, feedUrl) { + const items = []; + const itemRegex = /]*>([\s\S]*?)<\/item>/gi; + let match; + while ((match = itemRegex.exec(xml)) !== null) { + const block = match[1]; + const title = getTagValue(block, 'title'); + const link = canonicalizeLink(getTagValue(block, 'link'), feedUrl); + const publishedAt = toIsoDate(getTagValue(block, 'pubDate') || getTagValue(block, 'updated')); + items.push({ title, link, publishedAt }); + } + return items; +} + +function parseAtomEntries(xml, feedUrl) { + const entries = []; + const entryRegex = /]*>([\s\S]*?)<\/entry>/gi; + let match; + while ((match = entryRegex.exec(xml)) !== null) { + const block = match[1]; + const title = getTagValue(block, 'title'); + const link = canonicalizeLink(extractAtomLink(block), feedUrl); + const publishedAt = toIsoDate( + getTagValue(block, 'updated') || getTagValue(block, 'published') || getTagValue(block, 'pubDate') + ); + entries.push({ title, link, publishedAt }); + } + return entries; +} + +function parseFeed(xml, feedUrl) { + if (/ item.title && item.link && item.publishedAt) + .map((item) => ({ + id: buildActionId(agency, item.title, item.publishedAt), + agency, + title: item.title, + link: item.link, + publishedAt: item.publishedAt, + })); +} + +function dedupeAndSortActions(actions) { + const seen = new Set(); + const deduped = []; + for (const action of actions) { + const key = canonicalizeLink(action.link); + if (!key || seen.has(key)) continue; + seen.add(key); + deduped.push({ ...action, link: key }); + } + + deduped.sort((a, b) => Date.parse(b.publishedAt) - Date.parse(a.publishedAt)); + return deduped; +} + +async function fetchFeed(feed, fetchImpl = globalThis.fetch) { + const headers = { + Accept: XML_ACCEPT, + 'User-Agent': feed.userAgent || CHROME_UA, + }; + + const response = await fetchImpl(feed.url, { + headers, + signal: AbortSignal.timeout(FEED_TIMEOUT_MS), + }); + + if (!response.ok) { + throw new Error(`${feed.agency}: HTTP ${response.status}`); + } + + const xml = await response.text(); + const parsed = parseFeed(xml, feed.url); + return normalizeFeedItems(parsed, feed.agency); +} + +async function fetchAllFeeds(fetchImpl = globalThis.fetch, feeds = REGULATORY_FEEDS) { + const results = await Promise.allSettled(feeds.map((feed) => fetchFeed(feed, fetchImpl))); + const actions = []; + let successCount = 0; + + for (let index = 0; index < results.length; index += 1) { + const result = results[index]; + const feed = feeds[index]; + if (result.status === 'fulfilled') { + successCount += 1; + actions.push(...result.value); + continue; + } + console.error(`[regulatory] ${feed.agency}: ${result.reason?.message || result.reason}`); + } + + if (successCount === 0) { + throw new Error('All regulatory feeds failed'); + } + + return dedupeAndSortActions(actions); +} + +async function main(fetchImpl = globalThis.fetch) { + const actions = await fetchAllFeeds(fetchImpl); + process.stdout.write(`${JSON.stringify(actions, null, 2)}\n`); + return actions; +} + +const isDirectRun = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; + +if (isDirectRun) { + main().catch((err) => { + console.error(`FETCH FAILED: ${err.message || err}`); + process.exit(1); + }); +} + +export { + CHROME_UA, + FEED_TIMEOUT_MS, + REGULATORY_FEEDS, + SEC_USER_AGENT, + buildActionId, + canonicalizeLink, + decodeEntities, + dedupeAndSortActions, + extractAtomLink, + fetchAllFeeds, + fetchFeed, + getTagValue, + main, + normalizeFeedItems, + parseAtomEntries, + parseFeed, + parseRssItems, + resolveFeedLink, + slugifyTitle, + stripHtml, + toIsoDate, +}; diff --git a/tests/regulatory-seed-unit.test.mjs b/tests/regulatory-seed-unit.test.mjs new file mode 100644 index 000000000..852d1ddb3 --- /dev/null +++ b/tests/regulatory-seed-unit.test.mjs @@ -0,0 +1,185 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import vm from 'node:vm'; + +function normalize(value) { + return JSON.parse(JSON.stringify(value)); +} + +const seedSrc = readFileSync('scripts/seed-regulatory-actions.mjs', 'utf8'); + +const pureSrc = seedSrc + .replace(/^import\s.*$/gm, '') + .replace(/const isDirectRun[\s\S]*?}\n\nexport\s*{[\s\S]*?};?\s*$/m, ''); + +const ctx = vm.createContext({ + console, + Date, + Math, + Number, + Array, + Set, + String, + RegExp, + URL, + URLSearchParams, + AbortSignal, + CHROME_UA: 'Mozilla/5.0 (test)', +}); + +vm.runInContext(pureSrc, ctx); + +const { + decodeEntities, + stripHtml, + extractAtomLink, + parseRssItems, + parseAtomEntries, + parseFeed, + normalizeFeedItems, + dedupeAndSortActions, + fetchAllFeeds, +} = ctx; + +describe('decodeEntities', () => { + it('decodes named and numeric entities', () => { + assert.equal(decodeEntities('Tom & Jerry & &'), 'Tom & Jerry & &'); + }); +}); + +describe('stripHtml', () => { + it('removes tags and CDATA while preserving text', () => { + assert.equal(stripHtml('world]]>'), 'Hello world'); + }); +}); + +describe('parseRssItems', () => { + it('extracts RSS items with normalized links and pubDate', () => { + const xml = ` + + + <![CDATA[SEC & Co. Charges <b>Issuer</b>]]> + /news/press-release/2026-10 + Mon, 30 Mar 2026 18:00:00 GMT + + `; + + assert.deepEqual(normalize(parseRssItems(xml, 'https://www.sec.gov/news/pressreleases.rss')), [{ + title: 'SEC & Co. Charges Issuer', + link: 'https://www.sec.gov/news/press-release/2026-10', + publishedAt: '2026-03-30T18:00:00.000Z', + }]); + }); +}); + +describe('extractAtomLink + parseAtomEntries', () => { + it('prefers alternate href and normalizes publishedAt from updated', () => { + const xml = ` + + + Fed issues notice + + + 2026-03-29T12:30:00Z + + `; + + assert.equal( + extractAtomLink(''), + '/press/notice-a' + ); + + assert.deepEqual(normalize(parseAtomEntries(xml, 'https://www.federalreserve.gov/feeds/press_all.xml')), [{ + title: 'Fed issues notice', + link: 'https://www.federalreserve.gov/press/notice-a', + publishedAt: '2026-03-29T12:30:00.000Z', + }]); + }); +}); + +describe('parseFeed', () => { + it('detects Atom feeds automatically', () => { + const atom = 'A2026-03-28T00:00:00Z'; + const parsed = normalize(parseFeed(atom, 'https://example.test/feed')); + assert.equal(parsed.length, 1); + assert.equal(parsed[0].link, 'https://example.test/a'); + }); +}); + +describe('normalizeFeedItems', () => { + it('skips incomplete entries and generates deterministic ids', () => { + const normalized = normalize(normalizeFeedItems([ + { title: 'SEC Charges XYZ Corp', link: 'https://example.test/sec', publishedAt: '2026-03-29T14:00:00.000Z' }, + { title: '', link: 'https://example.test/missing', publishedAt: '2026-03-29T14:00:00.000Z' }, + ], 'SEC')); + + assert.equal(normalized.length, 1); + assert.equal(normalized[0].id, 'sec-sec-charges-xyz-corp-20260329'); + }); +}); + +describe('dedupeAndSortActions', () => { + it('deduplicates by canonical link and sorts newest first', () => { + const actions = normalize(dedupeAndSortActions([ + { + id: 'older', + agency: 'SEC', + title: 'Older', + link: 'https://example.test/path#frag', + publishedAt: '2026-03-28T10:00:00.000Z', + }, + { + id: 'newer', + agency: 'FDIC', + title: 'Newer', + link: 'https://example.test/new', + publishedAt: '2026-03-30T10:00:00.000Z', + }, + { + id: 'duplicate', + agency: 'SEC', + title: 'Duplicate', + link: 'https://example.test/path', + publishedAt: '2026-03-29T10:00:00.000Z', + }, + ])); + + assert.deepEqual(actions.map((item) => item.id), ['newer', 'older']); + assert.equal(actions[1].link, 'https://example.test/path'); + }); +}); + +describe('fetchAllFeeds', () => { + const feeds = [ + { agency: 'SEC', url: 'https://feeds.test/sec', userAgent: 'Custom-SEC-UA' }, + { agency: 'FDIC', url: 'https://feeds.test/fdic' }, + ]; + + it('returns normalized aggregate when at least one feed succeeds', async () => { + const requests = []; + const fetchStub = async (url, options) => { + requests.push({ url, options }); + if (url.endsWith('/sec')) { + return { + ok: true, + text: async () => `SEC Charges Bankhttps://sec.test/aMon, 30 Mar 2026 18:00:00 GMT`, + }; + } + throw new Error('FDIC timeout'); + }; + + const result = normalize(await fetchAllFeeds(fetchStub, feeds)); + assert.equal(result.length, 1); + assert.equal(result[0].agency, 'SEC'); + assert.equal(requests[0].options.headers['User-Agent'], 'Custom-SEC-UA'); + assert.equal(requests[1].options.headers['User-Agent'], ctx.CHROME_UA); + }); + + it('throws when all feeds fail', async () => { + await assert.rejects( + fetchAllFeeds(async () => { throw new Error('nope'); }, feeds), + /All regulatory feeds failed/ + ); + }); +}); From f3547f4c721e35d1c87d626d5d892bf27b028022 Mon Sep 17 00:00:00 2001 From: lspassos1 Date: Tue, 31 Mar 2026 00:39:16 +0100 Subject: [PATCH 02/12] feat(regulatory): classify and publish regulatory actions Build on the standalone RSS fetcher by adding keyword-based tier classification, aggregate payload counts, and runSeed integration for regulatory:actions:v1. The updated tests cover matched keywords, payload stats, and the runSeed wiring needed for Redis publication. Refs #2493 Depends on #2564 --- scripts/seed-regulatory-actions.mjs | 79 ++++++++++++++++++- tests/regulatory-seed-unit.test.mjs | 117 ++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+), 4 deletions(-) diff --git a/scripts/seed-regulatory-actions.mjs b/scripts/seed-regulatory-actions.mjs index 569de5366..cc55226eb 100644 --- a/scripts/seed-regulatory-actions.mjs +++ b/scripts/seed-regulatory-actions.mjs @@ -1,11 +1,26 @@ #!/usr/bin/env node import { pathToFileURL } from 'node:url'; -import { CHROME_UA } from './_seed-utils.mjs'; +import { CHROME_UA, loadEnvFile, runSeed } from './_seed-utils.mjs'; +loadEnvFile(import.meta.url); + +const CANONICAL_KEY = 'regulatory:actions:v1'; const FEED_TIMEOUT_MS = 15_000; +const TTL_SECONDS = 7200; const XML_ACCEPT = 'application/atom+xml, application/rss+xml, application/xml, text/xml, */*'; const SEC_USER_AGENT = 'WorldMonitor/2.0 (monitor@worldmonitor.app)'; +const HIGH_KEYWORDS = [ + 'enforcement', 'charges', 'charged', 'fraud', 'failure', 'failed bank', + 'emergency', 'halt', 'suspension', 'suspended', 'cease', 'desist', + 'penalty', 'fine', 'fined', 'settlement', 'indictment', 'manipulation', + 'ban', 'revocation', 'insolvency', +]; +const MEDIUM_KEYWORDS = [ + 'proposed rule', 'final rule', 'rulemaking', 'guidance', 'warning', + 'notice', 'advisory', 'review', 'examination', 'investigation', + 'stress test', 'capital requirement', 'disclosure requirement', +]; const REGULATORY_FEEDS = [ { agency: 'SEC', url: 'https://www.sec.gov/news/pressreleases.rss', userAgent: SEC_USER_AGENT }, @@ -217,10 +232,58 @@ async function fetchAllFeeds(fetchImpl = globalThis.fetch, feeds = REGULATORY_FE return dedupeAndSortActions(actions); } -async function main(fetchImpl = globalThis.fetch) { +function escapeRegex(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function findMatchedKeywords(title, keywords) { + const lowerTitle = stripHtml(title).toLowerCase(); + return keywords.filter((keyword) => { + const pattern = `\\b${escapeRegex(keyword.toLowerCase()).replace(/\s+/g, '\\s+')}\\b`; + return new RegExp(pattern, 'i').test(lowerTitle); + }); +} + +function classifyAction(action) { + const highMatches = findMatchedKeywords(action.title, HIGH_KEYWORDS); + if (highMatches.length > 0) { + return { ...action, tier: 'high', matchedKeywords: [...new Set(highMatches)] }; + } + + const mediumMatches = findMatchedKeywords(action.title, MEDIUM_KEYWORDS); + if (mediumMatches.length > 0) { + return { ...action, tier: 'medium', matchedKeywords: [...new Set(mediumMatches)] }; + } + + return { ...action, tier: 'low', matchedKeywords: [] }; +} + +function buildSeedPayload(actions, fetchedAt = Date.now()) { + const classified = actions.map(classifyAction); + const highCount = classified.filter((action) => action.tier === 'high').length; + const mediumCount = classified.filter((action) => action.tier === 'medium').length; + + return { + actions: classified, + fetchedAt, + recordCount: classified.length, + highCount, + mediumCount, + }; +} + +async function fetchRegulatoryActionPayload(fetchImpl = globalThis.fetch) { const actions = await fetchAllFeeds(fetchImpl); - process.stdout.write(`${JSON.stringify(actions, null, 2)}\n`); - return actions; + return buildSeedPayload(actions, Date.now()); +} + +async function main(fetchImpl = globalThis.fetch, runSeedImpl = runSeed) { + return runSeedImpl('regulatory', 'actions', CANONICAL_KEY, () => fetchRegulatoryActionPayload(fetchImpl), { + ttlSeconds: TTL_SECONDS, + validateFn: (data) => Array.isArray(data?.actions), + recordCount: (data) => data?.recordCount || 0, + sourceVersion: 'regulatory-rss-v1', + }); } const isDirectRun = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; @@ -233,17 +296,25 @@ if (isDirectRun) { } export { + CANONICAL_KEY, CHROME_UA, FEED_TIMEOUT_MS, + HIGH_KEYWORDS, + MEDIUM_KEYWORDS, REGULATORY_FEEDS, SEC_USER_AGENT, + TTL_SECONDS, buildActionId, + buildSeedPayload, canonicalizeLink, + classifyAction, decodeEntities, dedupeAndSortActions, extractAtomLink, fetchAllFeeds, fetchFeed, + fetchRegulatoryActionPayload, + findMatchedKeywords, getTagValue, main, normalizeFeedItems, diff --git a/tests/regulatory-seed-unit.test.mjs b/tests/regulatory-seed-unit.test.mjs index 852d1ddb3..6a5b998d2 100644 --- a/tests/regulatory-seed-unit.test.mjs +++ b/tests/regulatory-seed-unit.test.mjs @@ -11,6 +11,7 @@ const seedSrc = readFileSync('scripts/seed-regulatory-actions.mjs', 'utf8'); const pureSrc = seedSrc .replace(/^import\s.*$/gm, '') + .replace(/loadEnvFile\([^)]+\);\n/, '') .replace(/const isDirectRun[\s\S]*?}\n\nexport\s*{[\s\S]*?};?\s*$/m, ''); const ctx = vm.createContext({ @@ -26,6 +27,8 @@ const ctx = vm.createContext({ URLSearchParams, AbortSignal, CHROME_UA: 'Mozilla/5.0 (test)', + loadEnvFile: () => {}, + runSeed: async () => {}, }); vm.runInContext(pureSrc, ctx); @@ -40,6 +43,10 @@ const { normalizeFeedItems, dedupeAndSortActions, fetchAllFeeds, + classifyAction, + buildSeedPayload, + fetchRegulatoryActionPayload, + main, } = ctx; describe('decodeEntities', () => { @@ -183,3 +190,113 @@ describe('fetchAllFeeds', () => { ); }); }); + +describe('classifyAction', () => { + it('marks high priority actions and captures matched keywords', () => { + const action = normalize(classifyAction({ + id: 'sec-a', + agency: 'SEC', + title: 'SEC Charges Bank for Accounting Fraud', + link: 'https://example.test/sec-a', + publishedAt: '2026-03-30T18:00:00.000Z', + })); + + assert.equal(action.tier, 'high'); + assert.deepEqual(action.matchedKeywords, ['charges', 'fraud']); + }); + + it('falls back to medium and then low', () => { + const medium = normalize(classifyAction({ + id: 'fed-a', + agency: 'Federal Reserve', + title: 'Federal Reserve Issues Capital Requirement Guidance', + link: 'https://example.test/fed-a', + publishedAt: '2026-03-30T18:00:00.000Z', + })); + const low = normalize(classifyAction({ + id: 'finra-a', + agency: 'FINRA', + title: 'FINRA Publishes Monthly Highlights', + link: 'https://example.test/finra-a', + publishedAt: '2026-03-30T18:00:00.000Z', + })); + + assert.equal(medium.tier, 'medium'); + assert.deepEqual(medium.matchedKeywords, ['guidance', 'capital requirement']); + assert.equal(low.tier, 'low'); + assert.deepEqual(low.matchedKeywords, []); + }); +}); + +describe('buildSeedPayload', () => { + it('adds fetchedAt and aggregate counts', () => { + const payload = normalize(buildSeedPayload([ + { + id: 'sec-a', + agency: 'SEC', + title: 'SEC Charges Bank for Fraud', + link: 'https://example.test/sec-a', + publishedAt: '2026-03-30T18:00:00.000Z', + }, + { + id: 'fed-a', + agency: 'Federal Reserve', + title: 'Federal Reserve Issues Guidance', + link: 'https://example.test/fed-a', + publishedAt: '2026-03-29T18:00:00.000Z', + }, + { + id: 'finra-a', + agency: 'FINRA', + title: 'FINRA Monthly Bulletin', + link: 'https://example.test/finra-a', + publishedAt: '2026-03-28T18:00:00.000Z', + }, + ], 1711718400000)); + + assert.equal(payload.fetchedAt, 1711718400000); + assert.equal(payload.recordCount, 3); + assert.equal(payload.highCount, 1); + assert.equal(payload.mediumCount, 1); + assert.equal(payload.actions[2].tier, 'low'); + }); +}); + +describe('fetchRegulatoryActionPayload', () => { + it('returns classified payload from fetched actions', async () => { + const payload = normalize(await fetchRegulatoryActionPayload(async (url) => ({ + ok: true, + text: async () => `FDIC Publishes Enforcement Orders${url}/itemMon, 30 Mar 2026 18:00:00 GMT`, + }))); + + assert.equal(payload.actions.length, 5); + assert.equal(payload.recordCount, 5); + assert.ok(typeof payload.fetchedAt === 'number'); + assert.equal(payload.actions[0].tier, 'high'); + assert.deepEqual(payload.actions[0].matchedKeywords, ['enforcement']); + }); +}); + +describe('main', () => { + it('wires runSeed with the regulatory key, TTL, and validateFn', async () => { + const calls = []; + const runSeedStub = async (domain, resource, canonicalKey, fetchFn, opts) => { + calls.push({ domain, resource, canonicalKey, opts, payload: await fetchFn() }); + return 'ok'; + }; + const fetchStub = async (url) => ({ + ok: true, + text: async () => `CFTC Issues Advisory${url}/itemMon, 30 Mar 2026 18:00:00 GMT`, + }); + + const result = await main(fetchStub, runSeedStub); + assert.equal(result, 'ok'); + assert.equal(calls.length, 1); + assert.equal(calls[0].domain, 'regulatory'); + assert.equal(calls[0].resource, 'actions'); + assert.equal(calls[0].canonicalKey, 'regulatory:actions:v1'); + assert.equal(calls[0].opts.ttlSeconds, 7200); + assert.equal(calls[0].opts.validateFn({ actions: [] }), true); + assert.equal(calls[0].payload.recordCount, 5); + }); +}); From ce40b921caa5c2a8dca7353a4cd8453b53545135 Mon Sep 17 00:00:00 2001 From: lspassos1 Date: Tue, 31 Mar 2026 00:42:39 +0100 Subject: [PATCH 03/12] feat(intelligence): add regulatory action cross-source signal Add regulatory:actions:v1 as a new cross-source input, map regulatory actions into the policy category, and emit CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION signals for recent high/medium items. The new test covers severity scoring and composite escalation when policy, financial, and economic signals co-fire in Global Markets. Refs #2494 Depends on #2567 --- scripts/seed-cross-source-signals.mjs | 28 +++ .../cross-source-signals-regulatory.test.mjs | 162 ++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 tests/cross-source-signals-regulatory.test.mjs diff --git a/scripts/seed-cross-source-signals.mjs b/scripts/seed-cross-source-signals.mjs index c4579d507..d00a87b50 100644 --- a/scripts/seed-cross-source-signals.mjs +++ b/scripts/seed-cross-source-signals.mjs @@ -31,6 +31,7 @@ const SOURCE_KEYS = [ 'gdelt:intel:tone:maritime', 'weather:alerts:v1', 'risk:scores:sebuf:stale:v1', + 'regulatory:actions:v1', ]; // ── Theater classification helpers ──────────────────────────────────────────── @@ -109,6 +110,7 @@ const TYPE_CATEGORY = { CROSS_SOURCE_SIGNAL_TYPE_WEATHER_EXTREME: 'natural', CROSS_SOURCE_SIGNAL_TYPE_MEDIA_TONE_DETERIORATION: 'information', CROSS_SOURCE_SIGNAL_TYPE_RISK_SCORE_SPIKE: 'intelligence', + CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION: 'policy', }; // Base severity weights for each signal type @@ -139,6 +141,7 @@ const BASE_WEIGHT = { CROSS_SOURCE_SIGNAL_TYPE_FORECAST_DETERIORATION: 1.5, // predictive — lower confidence CROSS_SOURCE_SIGNAL_TYPE_WEATHER_EXTREME: 1.5, // environmental — regional CROSS_SOURCE_SIGNAL_TYPE_MEDIA_TONE_DETERIORATION: 1.5, // sentiment — lagging + CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION: 2.0, // policy action — direct market impact }; function scoreTier(score) { @@ -713,6 +716,30 @@ function extractRiskScoreSpike(d) { }); } +function extractRegulatoryAction(d) { + const payload = d['regulatory:actions:v1']; + if (!payload) return []; + const cutoff = Date.now() - 48 * 3600 * 1000; + const recent = (payload.actions || []) + .filter((action) => new Date(action.publishedAt).getTime() > cutoff && action.tier !== 'low'); + if (recent.length === 0) return []; + return recent.slice(0, 3).map((action) => { + const tierMult = action.tier === 'high' ? 1.5 : 1.0; + const score = BASE_WEIGHT.CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION * tierMult; + return { + id: `regulatory:${action.id}`, + type: 'CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION', + theater: 'Global Markets', + summary: `${action.agency}: ${action.title}`, + severity: scoreTier(score), + severityScore: score, + detectedAt: new Date(action.publishedAt).getTime(), + contributingTypes: [], + signalCount: 0, + }; + }); +} + // ── Composite escalation detector ───────────────────────────────────────────── // Fires when >=3 signals from DIFFERENT categories share the same theater. function detectCompositeEscalation(signals) { @@ -790,6 +817,7 @@ async function aggregateCrossSourceSignals() { extractWeatherExtreme, extractMediaToneDeterioration, extractRiskScoreSpike, + extractRegulatoryAction, ]; for (const extractor of extractors) { diff --git a/tests/cross-source-signals-regulatory.test.mjs b/tests/cross-source-signals-regulatory.test.mjs new file mode 100644 index 000000000..654ce7c5e --- /dev/null +++ b/tests/cross-source-signals-regulatory.test.mjs @@ -0,0 +1,162 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import vm from 'node:vm'; + +function normalize(value) { + return JSON.parse(JSON.stringify(value)); +} + +const seedSrc = readFileSync('scripts/seed-cross-source-signals.mjs', 'utf8'); + +const pureSrc = seedSrc + .replace(/^import\s.*$/gm, '') + .replace(/loadEnvFile\([^)]+\);\n/, '') + .replace(/async function readAllSourceKeys[\s\S]*?\n}\n\n\/\/ ── Signal extractors/m, '// readAllSourceKeys removed for unit test\n\n// ── Signal extractors') + .replace(/runSeed\('intelligence'[\s\S]*$/m, ''); + +const ctx = vm.createContext({ console, Date, Math, Number, Array, Map, Set, String, RegExp }); +vm.runInContext(`${pureSrc}\n;globalThis.__exports = { SOURCE_KEYS, TYPE_CATEGORY, BASE_WEIGHT, scoreTier, extractRegulatoryAction, detectCompositeEscalation };`, ctx); + +const { + SOURCE_KEYS, + TYPE_CATEGORY, + BASE_WEIGHT, + scoreTier, + extractRegulatoryAction, + detectCompositeEscalation, +} = ctx.__exports; + +describe('source registration', () => { + it('adds the regulatory seed key to SOURCE_KEYS', () => { + assert.ok(SOURCE_KEYS.includes('regulatory:actions:v1')); + }); + + it('maps regulatory actions to the policy category and a 2.0 base weight', () => { + assert.equal(TYPE_CATEGORY.CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION, 'policy'); + assert.equal(BASE_WEIGHT.CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION, 2.0); + }); + + it('registers the extractor in the extractor list', () => { + assert.match(seedSrc, /extractRegulatoryAction,\n\s*];/); + }); +}); + +describe('extractRegulatoryAction', () => { + it('returns an empty array when the source key is missing', () => { + assert.deepEqual(normalize(extractRegulatoryAction({})), []); + }); + + it('emits high and medium signals, ignores low, and limits output to 3', () => { + const now = Date.now(); + const payload = { + 'regulatory:actions:v1': { + actions: [ + { + id: 'sec-a', + agency: 'SEC', + title: 'SEC Charges Issuer', + publishedAt: new Date(now - 1 * 3600 * 1000).toISOString(), + tier: 'high', + }, + { + id: 'fdic-a', + agency: 'FDIC', + title: 'FDIC Guidance Update', + publishedAt: new Date(now - 2 * 3600 * 1000).toISOString(), + tier: 'medium', + }, + { + id: 'finra-low', + agency: 'FINRA', + title: 'FINRA Monthly Bulletin', + publishedAt: new Date(now - 3 * 3600 * 1000).toISOString(), + tier: 'low', + }, + { + id: 'cftc-b', + agency: 'CFTC', + title: 'CFTC Advisory Notice', + publishedAt: new Date(now - 4 * 3600 * 1000).toISOString(), + tier: 'medium', + }, + { + id: 'fed-old', + agency: 'Federal Reserve', + title: 'Old Enforcement Notice', + publishedAt: new Date(now - 72 * 3600 * 1000).toISOString(), + tier: 'high', + }, + { + id: 'sec-c', + agency: 'SEC', + title: 'SEC Settlement', + publishedAt: new Date(now - 5 * 3600 * 1000).toISOString(), + tier: 'high', + }, + ], + }, + }; + + const signals = normalize(extractRegulatoryAction(payload)); + assert.equal(signals.length, 3); + assert.deepEqual(signals.map((signal) => signal.id), [ + 'regulatory:sec-a', + 'regulatory:fdic-a', + 'regulatory:cftc-b', + ]); + assert.equal(signals[0].severityScore, 3.0); + assert.equal(signals[0].severity, 'CROSS_SOURCE_SIGNAL_SEVERITY_HIGH'); + assert.equal(signals[1].severityScore, 2.0); + assert.equal(signals[1].severity, 'CROSS_SOURCE_SIGNAL_SEVERITY_MEDIUM'); + assert.equal(signals[0].theater, 'Global Markets'); + assert.equal(signals[0].summary, 'SEC: SEC Charges Issuer'); + }); +}); + +describe('detectCompositeEscalation', () => { + it('fires when policy, financial, and economic signals co-fire in Global Markets', () => { + const composite = normalize(detectCompositeEscalation([ + { + id: 'regulatory:sec-a', + type: 'CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION', + theater: 'Global Markets', + summary: 'SEC: SEC Charges Issuer', + severity: scoreTier(3.0), + severityScore: 3.0, + detectedAt: Date.now(), + contributingTypes: [], + signalCount: 0, + }, + { + id: 'vix:global', + type: 'CROSS_SOURCE_SIGNAL_TYPE_VIX_SPIKE', + theater: 'Global Markets', + summary: 'VIX elevated', + severity: scoreTier(2.0), + severityScore: 2.0, + detectedAt: Date.now(), + contributingTypes: [], + signalCount: 0, + }, + { + id: 'commodity:oil', + type: 'CROSS_SOURCE_SIGNAL_TYPE_COMMODITY_SHOCK', + theater: 'Global Markets', + summary: 'Oil shock', + severity: scoreTier(2.0), + severityScore: 2.0, + detectedAt: Date.now(), + contributingTypes: [], + signalCount: 0, + }, + ])); + + assert.equal(composite.length, 1); + assert.equal(composite[0].type, 'CROSS_SOURCE_SIGNAL_TYPE_COMPOSITE_ESCALATION'); + assert.equal(composite[0].theater, 'Global Markets'); + assert.ok(composite[0].contributingTypes.includes('regulatory action')); + assert.ok(composite[0].contributingTypes.includes('vix spike')); + assert.ok(composite[0].contributingTypes.includes('commodity shock')); + }); +}); From b7b399f164a4800c3ebd508060f1d03058dfc835 Mon Sep 17 00:00:00 2001 From: lspassos1 Date: Tue, 31 Mar 2026 00:46:26 +0100 Subject: [PATCH 04/12] chore(railway): sync regulatory seed cron schedule Extend the Railway seed-service sync script to enforce the expected cronSchedule for seed-regulatory-actions, while continuing to validate watch patterns and start commands. Add a focused test for the new cronSchedule path and fail fast when the required seed-regulatory-actions service is missing. Refs #2495 Depends on #2568 --- scripts/railway-set-watch-paths.mjs | 55 +++++++++++++++++++------- tests/railway-set-watch-paths.test.mjs | 26 ++++++++++++ 2 files changed, 66 insertions(+), 15 deletions(-) create mode 100644 tests/railway-set-watch-paths.test.mjs diff --git a/scripts/railway-set-watch-paths.mjs b/scripts/railway-set-watch-paths.mjs index c5b2d38e0..f5af0967b 100644 --- a/scripts/railway-set-watch-paths.mjs +++ b/scripts/railway-set-watch-paths.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node /** - * Sets watchPatterns and validates startCommand on all Railway seed services. + * Sets watchPatterns, validates startCommand, and syncs cronSchedule on Railway seed services. * * All seed services use rootDirectory="scripts", so the correct startCommand * is `node seed-.mjs` (NOT `node scripts/seed-.mjs` — that path @@ -22,6 +22,10 @@ const DRY_RUN = process.argv.includes('--dry-run'); const PROJECT_ID = '29419572-0b0d-437f-8e71-4fa68daf514f'; const ENV_ID = '91a05726-0b83-4d44-a33e-6aec94e58780'; const API = 'https://backboard.railway.app/graphql/v2'; +const REQUIRED_SEED_SERVICES = new Set(['seed-regulatory-actions']); +const EXPECTED_CRON_SCHEDULES = new Map([ + ['seed-regulatory-actions', '0 */2 * * *'], +]); // Seeds that use loadSharedConfig (depend on scripts/shared/*.json) const USES_SHARED_CONFIG = new Set([ @@ -50,6 +54,21 @@ async function gql(token, query, variables = {}) { return json.data; } +function buildExpectedPatterns(serviceName) { + const scriptFile = `scripts/${serviceName}.mjs`; + const patterns = [scriptFile, 'scripts/_seed-utils.mjs', 'scripts/package.json']; + + if (USES_SHARED_CONFIG.has(serviceName)) { + patterns.push('scripts/shared/**', 'shared/**'); + } + + if (serviceName === 'seed-iran-events') { + patterns.push('scripts/data/iran-events-latest.json'); + } + + return patterns; +} + async function main() { const token = getToken(); @@ -66,15 +85,22 @@ async function main() { .map(e => e.node) .filter(s => s.name.startsWith('seed-')); + const missingRequiredServices = [...REQUIRED_SEED_SERVICES].filter( + (name) => !services.some((service) => service.name === name) + ); + if (missingRequiredServices.length > 0) { + throw new Error(`Missing required seed service(s): ${missingRequiredServices.join(', ')}`); + } + console.log(`Found ${services.length} seed services\n`); - // 2. Check each service's watchPatterns and startCommand + // 2. Check each service's watchPatterns, startCommand, and cronSchedule for (const svc of services) { const { service } = await gql(token, ` query ($id: String!, $envId: String!) { service(id: $id) { serviceInstances(first: 1, environmentId: $envId) { - edges { node { watchPatterns startCommand } } + edges { node { watchPatterns startCommand cronSchedule } } } } } @@ -83,26 +109,20 @@ async function main() { const instance = service.serviceInstances.edges[0]?.node || {}; const currentPatterns = instance.watchPatterns || []; const currentStartCmd = instance.startCommand || ''; + const currentCronSchedule = instance.cronSchedule || ''; // rootDirectory="scripts" so startCommand must NOT include the scripts/ prefix const expectedStartCmd = `node ${svc.name}.mjs`; const startCmdOk = currentStartCmd === expectedStartCmd; + const expectedCronSchedule = EXPECTED_CRON_SCHEDULES.get(svc.name) || ''; + const hasExpectedCronSchedule = EXPECTED_CRON_SCHEDULES.has(svc.name); + const cronScheduleOk = !hasExpectedCronSchedule || currentCronSchedule === expectedCronSchedule; // Build expected watch patterns (relative to git repo root) - const scriptFile = `scripts/${svc.name}.mjs`; - const patterns = [scriptFile, 'scripts/_seed-utils.mjs', 'scripts/package.json']; - - if (USES_SHARED_CONFIG.has(svc.name)) { - patterns.push('scripts/shared/**', 'shared/**'); - } - - if (svc.name === 'seed-iran-events') { - patterns.push('scripts/data/iran-events-latest.json'); - } - + const patterns = buildExpectedPatterns(svc.name); const patternsOk = JSON.stringify(currentPatterns.sort()) === JSON.stringify([...patterns].sort()); - if (patternsOk && startCmdOk) { + if (patternsOk && startCmdOk && cronScheduleOk) { console.log(` ${svc.name}: already correct`); continue; } @@ -116,6 +136,10 @@ async function main() { console.log(` watchPatterns current: ${currentPatterns.length ? currentPatterns.join(', ') : '(none)'}`); console.log(` watchPatterns setting: ${patterns.join(', ')}`); } + if (hasExpectedCronSchedule && !cronScheduleOk) { + console.log(` cronSchedule current: ${currentCronSchedule || '(none)'}`); + console.log(` cronSchedule expected: ${expectedCronSchedule}`); + } if (DRY_RUN) { console.log(` [DRY RUN] skipped\n`); @@ -126,6 +150,7 @@ async function main() { const input = {}; if (!patternsOk) input.watchPatterns = patterns; if (!startCmdOk) input.startCommand = expectedStartCmd; + if (hasExpectedCronSchedule && !cronScheduleOk) input.cronSchedule = expectedCronSchedule; await gql(token, ` mutation ($serviceId: String!, $environmentId: String!, $input: ServiceInstanceUpdateInput!) { diff --git a/tests/railway-set-watch-paths.test.mjs b/tests/railway-set-watch-paths.test.mjs new file mode 100644 index 000000000..528a362ff --- /dev/null +++ b/tests/railway-set-watch-paths.test.mjs @@ -0,0 +1,26 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; + +const scriptSrc = readFileSync('scripts/railway-set-watch-paths.mjs', 'utf8'); + +describe('railway-set-watch-paths regulatory cron sync', () => { + it('declares seed-regulatory-actions as a required seed service', () => { + assert.match(scriptSrc, /REQUIRED_SEED_SERVICES = new Set\(\['seed-regulatory-actions'\]\)/); + assert.match(scriptSrc, /Missing required seed service\(s\):/); + }); + + it('defines the expected 2-hour cron schedule for seed-regulatory-actions', () => { + assert.match(scriptSrc, /EXPECTED_CRON_SCHEDULES = new Map\(\[\s*\['seed-regulatory-actions', '0 \*\/2 \* \* \*'\]/s); + }); + + it('queries and updates cronSchedule via serviceInstanceUpdate', () => { + assert.match(scriptSrc, /node \{ watchPatterns startCommand cronSchedule \}/); + assert.match(scriptSrc, /input\.cronSchedule = expectedCronSchedule/); + }); + + it('continues to derive watch patterns from the seed name', () => { + assert.match(scriptSrc, /function buildExpectedPatterns\(serviceName\)/); + assert.match(scriptSrc, /const scriptFile = `scripts\/\$\{serviceName\}\.mjs`/); + }); +}); From 79b1c729a0ff7ec2a6dc1f20fe399b7892fdd5d8 Mon Sep 17 00:00:00 2001 From: lspassos1 Date: Tue, 31 Mar 2026 01:20:29 +0100 Subject: [PATCH 05/12] fix(regulatory): harden feed fetch defaults and action ids Use the repository-standard fetch wrapper in the seeder defaults, keep the documented FINRA HTTP exception in place, and include publish time in generated action ids to avoid same-day collisions. Validated with: node --test tests/regulatory-seed-unit.test.mjs; node scripts/seed-regulatory-actions.mjs | head -n 20 --- scripts/seed-regulatory-actions.mjs | 38 ++++++++++++++++++++--------- tests/regulatory-seed-unit.test.mjs | 2 +- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/scripts/seed-regulatory-actions.mjs b/scripts/seed-regulatory-actions.mjs index cc55226eb..7c6d5f65a 100644 --- a/scripts/seed-regulatory-actions.mjs +++ b/scripts/seed-regulatory-actions.mjs @@ -10,6 +10,7 @@ const FEED_TIMEOUT_MS = 15_000; const TTL_SECONDS = 7200; const XML_ACCEPT = 'application/atom+xml, application/rss+xml, application/xml, text/xml, */*'; const SEC_USER_AGENT = 'WorldMonitor/2.0 (monitor@worldmonitor.app)'; +const DEFAULT_FETCH = (...args) => globalThis.fetch(...args); const HIGH_KEYWORDS = [ 'enforcement', 'charges', 'charged', 'fraud', 'failure', 'failed bank', 'emergency', 'halt', 'suspension', 'suspended', 'cease', 'desist', @@ -27,6 +28,8 @@ const REGULATORY_FEEDS = [ { agency: 'CFTC', url: 'https://www.cftc.gov/RSS/RSSENF/rssenf.xml' }, { agency: 'Federal Reserve', url: 'https://www.federalreserve.gov/feeds/press_all.xml' }, { agency: 'FDIC', url: 'https://public.govdelivery.com/topics/USFDIC_26/feed.rss' }, + // FINRA still publishes this RSS endpoint over plain HTTP; HTTPS requests fail + // from both Node fetch and curl in validation, so keep the official feed URL. { agency: 'FINRA', url: 'http://feeds.finra.org/FINRANotices' }, ]; @@ -121,11 +124,16 @@ function yyyymmdd(isoDate) { return String(isoDate || '').slice(0, 10).replace(/-/g, ''); } +function hhmmss(isoDate) { + return String(isoDate || '').slice(11, 19).replace(/:/g, ''); +} + function buildActionId(agency, title, publishedAt) { const agencySlug = slugifyTitle(agency) || 'agency'; const titleSlug = slugifyTitle(title) || 'untitled'; const datePart = yyyymmdd(publishedAt) || 'undated'; - return `${agencySlug}-${titleSlug}-${datePart}`; + const timePart = hhmmss(publishedAt) || '000000'; + return `${agencySlug}-${titleSlug}-${datePart}-${timePart}`; } function parseRssItems(xml, feedUrl) { @@ -189,7 +197,7 @@ function dedupeAndSortActions(actions) { return deduped; } -async function fetchFeed(feed, fetchImpl = globalThis.fetch) { +async function fetchFeed(feed, fetchImpl = DEFAULT_FETCH) { const headers = { Accept: XML_ACCEPT, 'User-Agent': feed.userAgent || CHROME_UA, @@ -209,7 +217,7 @@ async function fetchFeed(feed, fetchImpl = globalThis.fetch) { return normalizeFeedItems(parsed, feed.agency); } -async function fetchAllFeeds(fetchImpl = globalThis.fetch, feeds = REGULATORY_FEEDS) { +async function fetchAllFeeds(fetchImpl = DEFAULT_FETCH, feeds = REGULATORY_FEEDS) { const results = await Promise.allSettled(feeds.map((feed) => fetchFeed(feed, fetchImpl))); const actions = []; let successCount = 0; @@ -232,25 +240,31 @@ async function fetchAllFeeds(fetchImpl = globalThis.fetch, feeds = REGULATORY_FE return dedupeAndSortActions(actions); } +<<<<<<< HEAD function escapeRegex(value) { return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } -function findMatchedKeywords(title, keywords) { +function compileKeywordPattern(keyword) { + const pattern = `\\b${escapeRegex(keyword.toLowerCase()).replace(/\s+/g, '\\s+')}\\b`; + return { keyword, regex: new RegExp(pattern, 'i') }; +} + +const HIGH_KEYWORD_PATTERNS = HIGH_KEYWORDS.map(compileKeywordPattern); +const MEDIUM_KEYWORD_PATTERNS = MEDIUM_KEYWORDS.map(compileKeywordPattern); + +function findMatchedKeywords(title, keywordPatterns) { const lowerTitle = stripHtml(title).toLowerCase(); - return keywords.filter((keyword) => { - const pattern = `\\b${escapeRegex(keyword.toLowerCase()).replace(/\s+/g, '\\s+')}\\b`; - return new RegExp(pattern, 'i').test(lowerTitle); - }); + return keywordPatterns.filter(({ regex }) => regex.test(lowerTitle)).map(({ keyword }) => keyword); } function classifyAction(action) { - const highMatches = findMatchedKeywords(action.title, HIGH_KEYWORDS); + const highMatches = findMatchedKeywords(action.title, HIGH_KEYWORD_PATTERNS); if (highMatches.length > 0) { return { ...action, tier: 'high', matchedKeywords: [...new Set(highMatches)] }; } - const mediumMatches = findMatchedKeywords(action.title, MEDIUM_KEYWORDS); + const mediumMatches = findMatchedKeywords(action.title, MEDIUM_KEYWORD_PATTERNS); if (mediumMatches.length > 0) { return { ...action, tier: 'medium', matchedKeywords: [...new Set(mediumMatches)] }; } @@ -272,12 +286,12 @@ function buildSeedPayload(actions, fetchedAt = Date.now()) { }; } -async function fetchRegulatoryActionPayload(fetchImpl = globalThis.fetch) { +async function fetchRegulatoryActionPayload(fetchImpl = DEFAULT_FETCH) { const actions = await fetchAllFeeds(fetchImpl); return buildSeedPayload(actions, Date.now()); } -async function main(fetchImpl = globalThis.fetch, runSeedImpl = runSeed) { +async function main(fetchImpl = DEFAULT_FETCH, runSeedImpl = runSeed) { return runSeedImpl('regulatory', 'actions', CANONICAL_KEY, () => fetchRegulatoryActionPayload(fetchImpl), { ttlSeconds: TTL_SECONDS, validateFn: (data) => Array.isArray(data?.actions), diff --git a/tests/regulatory-seed-unit.test.mjs b/tests/regulatory-seed-unit.test.mjs index 6a5b998d2..a52468336 100644 --- a/tests/regulatory-seed-unit.test.mjs +++ b/tests/regulatory-seed-unit.test.mjs @@ -122,7 +122,7 @@ describe('normalizeFeedItems', () => { ], 'SEC')); assert.equal(normalized.length, 1); - assert.equal(normalized[0].id, 'sec-sec-charges-xyz-corp-20260329'); + assert.equal(normalized[0].id, 'sec-sec-charges-xyz-corp-20260329-140000'); }); }); From 53cccd16325f5fe3aaa726936f03348da1ded95f Mon Sep 17 00:00:00 2001 From: lspassos1 Date: Tue, 31 Mar 2026 01:22:30 +0100 Subject: [PATCH 06/12] fix(regulatory): remove classifier conflict marker Clean up the leftover cherry-pick marker after carrying the shared seeder hardening changes onto this branch. Validated with: node --test tests/regulatory-seed-unit.test.mjs and a local fetchRegulatoryActionPayload smoke check. --- scripts/seed-regulatory-actions.mjs | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/seed-regulatory-actions.mjs b/scripts/seed-regulatory-actions.mjs index 7c6d5f65a..e0fad3227 100644 --- a/scripts/seed-regulatory-actions.mjs +++ b/scripts/seed-regulatory-actions.mjs @@ -240,7 +240,6 @@ async function fetchAllFeeds(fetchImpl = DEFAULT_FETCH, feeds = REGULATORY_FEEDS return dedupeAndSortActions(actions); } -<<<<<<< HEAD function escapeRegex(value) { return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } From 17a3b56fc82c9b00249941565807fc8660f3b9f3 Mon Sep 17 00:00:00 2001 From: lspassos1 Date: Tue, 31 Mar 2026 01:23:30 +0100 Subject: [PATCH 07/12] fix(intelligence): sort regulatory actions before slicing Explicitly sort regulatory actions by publishedAt inside the cross-source extractor before applying the 3-item limit, and cover the behavior with an out-of-order payload test. Validated with: node --test tests/regulatory-seed-unit.test.mjs and node --test tests/cross-source-signals-regulatory.test.mjs. --- scripts/seed-cross-source-signals.mjs | 3 +- .../cross-source-signals-regulatory.test.mjs | 28 +++++++++---------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/scripts/seed-cross-source-signals.mjs b/scripts/seed-cross-source-signals.mjs index d00a87b50..c16800f4a 100644 --- a/scripts/seed-cross-source-signals.mjs +++ b/scripts/seed-cross-source-signals.mjs @@ -721,7 +721,8 @@ function extractRegulatoryAction(d) { if (!payload) return []; const cutoff = Date.now() - 48 * 3600 * 1000; const recent = (payload.actions || []) - .filter((action) => new Date(action.publishedAt).getTime() > cutoff && action.tier !== 'low'); + .filter((action) => new Date(action.publishedAt).getTime() > cutoff && action.tier !== 'low') + .sort((a, b) => new Date(b.publishedAt).getTime() - new Date(a.publishedAt).getTime()); if (recent.length === 0) return []; return recent.slice(0, 3).map((action) => { const tierMult = action.tier === 'high' ? 1.5 : 1.0; diff --git a/tests/cross-source-signals-regulatory.test.mjs b/tests/cross-source-signals-regulatory.test.mjs index 654ce7c5e..f79dd49a8 100644 --- a/tests/cross-source-signals-regulatory.test.mjs +++ b/tests/cross-source-signals-regulatory.test.mjs @@ -52,13 +52,6 @@ describe('extractRegulatoryAction', () => { const payload = { 'regulatory:actions:v1': { actions: [ - { - id: 'sec-a', - agency: 'SEC', - title: 'SEC Charges Issuer', - publishedAt: new Date(now - 1 * 3600 * 1000).toISOString(), - tier: 'high', - }, { id: 'fdic-a', agency: 'FDIC', @@ -66,6 +59,20 @@ describe('extractRegulatoryAction', () => { publishedAt: new Date(now - 2 * 3600 * 1000).toISOString(), tier: 'medium', }, + { + id: 'sec-c', + agency: 'SEC', + title: 'SEC Settlement', + publishedAt: new Date(now - 5 * 3600 * 1000).toISOString(), + tier: 'high', + }, + { + id: 'sec-a', + agency: 'SEC', + title: 'SEC Charges Issuer', + publishedAt: new Date(now - 1 * 3600 * 1000).toISOString(), + tier: 'high', + }, { id: 'finra-low', agency: 'FINRA', @@ -87,13 +94,6 @@ describe('extractRegulatoryAction', () => { publishedAt: new Date(now - 72 * 3600 * 1000).toISOString(), tier: 'high', }, - { - id: 'sec-c', - agency: 'SEC', - title: 'SEC Settlement', - publishedAt: new Date(now - 5 * 3600 * 1000).toISOString(), - tier: 'high', - }, ], }, }; From 81f6667ae5ef53397196a41c7d889056705e6ad0 Mon Sep 17 00:00:00 2001 From: lspassos1 Date: Tue, 31 Mar 2026 01:24:19 +0100 Subject: [PATCH 08/12] fix(railway): avoid mutating watch patterns during diff Compare Railway watchPatterns using a copied array so the validation path stays side-effect free. Validated with: node --test tests/regulatory-seed-unit.test.mjs; node --test tests/cross-source-signals-regulatory.test.mjs; node --test tests/railway-set-watch-paths.test.mjs; node --test tests/bootstrap.test.mjs. --- scripts/railway-set-watch-paths.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/railway-set-watch-paths.mjs b/scripts/railway-set-watch-paths.mjs index f5af0967b..2866b7758 100644 --- a/scripts/railway-set-watch-paths.mjs +++ b/scripts/railway-set-watch-paths.mjs @@ -120,7 +120,7 @@ async function main() { // Build expected watch patterns (relative to git repo root) const patterns = buildExpectedPatterns(svc.name); - const patternsOk = JSON.stringify(currentPatterns.sort()) === JSON.stringify([...patterns].sort()); + const patternsOk = JSON.stringify([...currentPatterns].sort()) === JSON.stringify([...patterns].sort()); if (patternsOk && startCmdOk && cronScheduleOk) { console.log(` ${svc.name}: already correct`); From 10ed4ddb80673330cb599ed263797b8322fefc96 Mon Sep 17 00:00:00 2001 From: lspassos1 Date: Wed, 1 Apr 2026 22:41:48 +0100 Subject: [PATCH 09/12] feat(regulatory): capture feed descriptions in action records Extract RSS and Atom descriptions into the normalized action payload so later classifier work can use the same parsed feed output. Also adds @ts-check and documents the FINRA HTTP feed constraint. --- scripts/seed-regulatory-actions.mjs | 11 ++++++++--- tests/regulatory-seed-unit.test.mjs | 30 ++++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/scripts/seed-regulatory-actions.mjs b/scripts/seed-regulatory-actions.mjs index e0fad3227..529bf0202 100644 --- a/scripts/seed-regulatory-actions.mjs +++ b/scripts/seed-regulatory-actions.mjs @@ -1,4 +1,5 @@ #!/usr/bin/env node +// @ts-check import { pathToFileURL } from 'node:url'; import { CHROME_UA, loadEnvFile, runSeed } from './_seed-utils.mjs'; @@ -29,7 +30,8 @@ const REGULATORY_FEEDS = [ { agency: 'Federal Reserve', url: 'https://www.federalreserve.gov/feeds/press_all.xml' }, { agency: 'FDIC', url: 'https://public.govdelivery.com/topics/USFDIC_26/feed.rss' }, // FINRA still publishes this RSS endpoint over plain HTTP; HTTPS requests fail - // from both Node fetch and curl in validation, so keep the official feed URL. + // from both Node fetch and curl in validation, so keep the official feed URL + // and periodically recheck whether HTTPS starts working. { agency: 'FINRA', url: 'http://feeds.finra.org/FINRANotices' }, ]; @@ -143,9 +145,10 @@ function parseRssItems(xml, feedUrl) { while ((match = itemRegex.exec(xml)) !== null) { const block = match[1]; const title = getTagValue(block, 'title'); + const description = getTagValue(block, 'description'); const link = canonicalizeLink(getTagValue(block, 'link'), feedUrl); const publishedAt = toIsoDate(getTagValue(block, 'pubDate') || getTagValue(block, 'updated')); - items.push({ title, link, publishedAt }); + items.push({ title, description, link, publishedAt }); } return items; } @@ -157,11 +160,12 @@ function parseAtomEntries(xml, feedUrl) { while ((match = entryRegex.exec(xml)) !== null) { const block = match[1]; const title = getTagValue(block, 'title'); + const description = getTagValue(block, 'summary') || getTagValue(block, 'content'); const link = canonicalizeLink(extractAtomLink(block), feedUrl); const publishedAt = toIsoDate( getTagValue(block, 'updated') || getTagValue(block, 'published') || getTagValue(block, 'pubDate') ); - entries.push({ title, link, publishedAt }); + entries.push({ title, description, link, publishedAt }); } return entries; } @@ -178,6 +182,7 @@ function normalizeFeedItems(items, agency) { id: buildActionId(agency, item.title, item.publishedAt), agency, title: item.title, + description: item.description || '', link: item.link, publishedAt: item.publishedAt, })); diff --git a/tests/regulatory-seed-unit.test.mjs b/tests/regulatory-seed-unit.test.mjs index a52468336..b351ef307 100644 --- a/tests/regulatory-seed-unit.test.mjs +++ b/tests/regulatory-seed-unit.test.mjs @@ -62,11 +62,12 @@ describe('stripHtml', () => { }); describe('parseRssItems', () => { - it('extracts RSS items with normalized links and pubDate', () => { + it('extracts RSS items with description, normalized links, and pubDate', () => { const xml = ` <![CDATA[SEC & Co. Charges <b>Issuer</b>]]> + fraud & disclosure failures]]> /news/press-release/2026-10 Mon, 30 Mar 2026 18:00:00 GMT @@ -74,6 +75,7 @@ describe('parseRssItems', () => { assert.deepEqual(normalize(parseRssItems(xml, 'https://www.sec.gov/news/pressreleases.rss')), [{ title: 'SEC & Co. Charges Issuer', + description: 'Alleges fraud & disclosure failures', link: 'https://www.sec.gov/news/press-release/2026-10', publishedAt: '2026-03-30T18:00:00.000Z', }]); @@ -81,11 +83,12 @@ describe('parseRssItems', () => { }); describe('extractAtomLink + parseAtomEntries', () => { - it('prefers alternate href and normalizes publishedAt from updated', () => { + it('prefers alternate href and extracts summary/content with normalized publishedAt', () => { const xml = ` Fed issues notice + policy summary]]> 2026-03-29T12:30:00Z @@ -99,9 +102,27 @@ describe('extractAtomLink + parseAtomEntries', () => { assert.deepEqual(normalize(parseAtomEntries(xml, 'https://www.federalreserve.gov/feeds/press_all.xml')), [{ title: 'Fed issues notice', + description: 'Detailed policy summary', link: 'https://www.federalreserve.gov/press/notice-a', publishedAt: '2026-03-29T12:30:00.000Z', }]); + + const contentXml = ` + + + FDIC update + Formal administrative note

]]> + + 2026-03-28T09:15:00Z + + `; + + assert.deepEqual(normalize(parseAtomEntries(contentXml, 'https://www.fdic.gov/feed')), [{ + title: 'FDIC update', + description: 'Formal administrative note', + link: 'https://fdic.example.test/a', + publishedAt: '2026-03-28T09:15:00.000Z', + }]); }); }); @@ -118,11 +139,14 @@ describe('normalizeFeedItems', () => { it('skips incomplete entries and generates deterministic ids', () => { const normalized = normalize(normalizeFeedItems([ { title: 'SEC Charges XYZ Corp', link: 'https://example.test/sec', publishedAt: '2026-03-29T14:00:00.000Z' }, + { title: 'SEC Summary', description: 'extra context', link: 'https://example.test/sec-2', publishedAt: '2026-03-29T14:30:00.000Z' }, { title: '', link: 'https://example.test/missing', publishedAt: '2026-03-29T14:00:00.000Z' }, ], 'SEC')); - assert.equal(normalized.length, 1); + assert.equal(normalized.length, 2); assert.equal(normalized[0].id, 'sec-sec-charges-xyz-corp-20260329-140000'); + assert.equal(normalized[0].description, ''); + assert.equal(normalized[1].description, 'extra context'); }); }); From 11abe2b5c232967c79bf3e77b9e0eb6480f4177a Mon Sep 17 00:00:00 2001 From: lspassos1 Date: Wed, 1 Apr 2026 22:44:43 +0100 Subject: [PATCH 10/12] fix(regulatory): tighten action classification and TTL Raise the Redis retention window, classify against combined title and description text, reserve low for routine notices, and export the shared regulatory cache key for downstream health wiring. --- scripts/seed-regulatory-actions.mjs | 39 +++++++++++++---- server/_shared/cache-keys.ts | 1 + tests/regulatory-contract.test.mjs | 18 ++++++++ tests/regulatory-seed-unit.test.mjs | 65 ++++++++++++++++++++++------- 4 files changed, 98 insertions(+), 25 deletions(-) create mode 100644 tests/regulatory-contract.test.mjs diff --git a/scripts/seed-regulatory-actions.mjs b/scripts/seed-regulatory-actions.mjs index 529bf0202..a320c2389 100644 --- a/scripts/seed-regulatory-actions.mjs +++ b/scripts/seed-regulatory-actions.mjs @@ -8,7 +8,7 @@ loadEnvFile(import.meta.url); const CANONICAL_KEY = 'regulatory:actions:v1'; const FEED_TIMEOUT_MS = 15_000; -const TTL_SECONDS = 7200; +const TTL_SECONDS = 21600; const XML_ACCEPT = 'application/atom+xml, application/rss+xml, application/xml, text/xml, */*'; const SEC_USER_AGENT = 'WorldMonitor/2.0 (monitor@worldmonitor.app)'; const DEFAULT_FETCH = (...args) => globalThis.fetch(...args); @@ -16,12 +16,18 @@ const HIGH_KEYWORDS = [ 'enforcement', 'charges', 'charged', 'fraud', 'failure', 'failed bank', 'emergency', 'halt', 'suspension', 'suspended', 'cease', 'desist', 'penalty', 'fine', 'fined', 'settlement', 'indictment', 'manipulation', - 'ban', 'revocation', 'insolvency', + 'ban', 'revocation', 'insolvency', 'injunction', 'cease and desist', + 'cease-and-desist', 'consent order', 'debarment', 'suspension order', ]; const MEDIUM_KEYWORDS = [ 'proposed rule', 'final rule', 'rulemaking', 'guidance', 'warning', - 'notice', 'advisory', 'review', 'examination', 'investigation', + 'advisory', 'review', 'examination', 'investigation', 'stress test', 'capital requirement', 'disclosure requirement', + 'resolves action', 'settled charges', 'administrative proceeding', 'remedial action', +]; +const LOW_PRIORITY_TITLE_PATTERNS = [ + /^(Regulatory|Information|Technical) Notice\b/i, + /\bmonthly (highlights|bulletin)\b/i, ]; const REGULATORY_FEEDS = [ @@ -257,23 +263,37 @@ function compileKeywordPattern(keyword) { const HIGH_KEYWORD_PATTERNS = HIGH_KEYWORDS.map(compileKeywordPattern); const MEDIUM_KEYWORD_PATTERNS = MEDIUM_KEYWORDS.map(compileKeywordPattern); -function findMatchedKeywords(title, keywordPatterns) { - const lowerTitle = stripHtml(title).toLowerCase(); - return keywordPatterns.filter(({ regex }) => regex.test(lowerTitle)).map(({ keyword }) => keyword); +function findMatchedKeywords(text, keywordPatterns) { + const normalizedText = stripHtml(text).toLowerCase(); + return keywordPatterns.filter(({ regex }) => regex.test(normalizedText)).map(({ keyword }) => keyword); +} + +function buildClassificationText(action) { + return [action.title, action.description].filter(Boolean).join(' '); +} + +function isLowPriorityRoutineTitle(title) { + const normalizedTitle = stripHtml(title); + return LOW_PRIORITY_TITLE_PATTERNS.some((pattern) => pattern.test(normalizedTitle)); } function classifyAction(action) { - const highMatches = findMatchedKeywords(action.title, HIGH_KEYWORD_PATTERNS); + const classificationText = buildClassificationText(action); + const highMatches = findMatchedKeywords(classificationText, HIGH_KEYWORD_PATTERNS); if (highMatches.length > 0) { return { ...action, tier: 'high', matchedKeywords: [...new Set(highMatches)] }; } - const mediumMatches = findMatchedKeywords(action.title, MEDIUM_KEYWORD_PATTERNS); + if (isLowPriorityRoutineTitle(action.title)) { + return { ...action, tier: 'low', matchedKeywords: [] }; + } + + const mediumMatches = findMatchedKeywords(classificationText, MEDIUM_KEYWORD_PATTERNS); if (mediumMatches.length > 0) { return { ...action, tier: 'medium', matchedKeywords: [...new Set(mediumMatches)] }; } - return { ...action, tier: 'low', matchedKeywords: [] }; + return { ...action, tier: 'unknown', matchedKeywords: [] }; } function buildSeedPayload(actions, fetchedAt = Date.now()) { @@ -334,6 +354,7 @@ export { fetchRegulatoryActionPayload, findMatchedKeywords, getTagValue, + isLowPriorityRoutineTitle, main, normalizeFeedItems, parseAtomEntries, diff --git a/server/_shared/cache-keys.ts b/server/_shared/cache-keys.ts index fb48505af..f656e6fe8 100644 --- a/server/_shared/cache-keys.ts +++ b/server/_shared/cache-keys.ts @@ -5,6 +5,7 @@ */ export const SIMULATION_OUTCOME_LATEST_KEY = 'forecast:simulation-outcome:latest'; export const SIMULATION_PACKAGE_LATEST_KEY = 'forecast:simulation-package:latest'; +export const REGULATORY_ACTIONS_KEY = 'regulatory:actions:v1'; /** * Static cache keys for the bootstrap endpoint. diff --git a/tests/regulatory-contract.test.mjs b/tests/regulatory-contract.test.mjs new file mode 100644 index 000000000..687abc185 --- /dev/null +++ b/tests/regulatory-contract.test.mjs @@ -0,0 +1,18 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const root = join(__dirname, '..'); + +describe('regulatory cache contracts', () => { + it('exports REGULATORY_ACTIONS_KEY from cache-keys.ts', () => { + const cacheKeysSrc = readFileSync(join(root, 'server', '_shared', 'cache-keys.ts'), 'utf8'); + assert.match( + cacheKeysSrc, + /export const REGULATORY_ACTIONS_KEY = 'regulatory:actions:v1';/ + ); + }); +}); diff --git a/tests/regulatory-seed-unit.test.mjs b/tests/regulatory-seed-unit.test.mjs index b351ef307..02323218b 100644 --- a/tests/regulatory-seed-unit.test.mjs +++ b/tests/regulatory-seed-unit.test.mjs @@ -216,40 +216,61 @@ describe('fetchAllFeeds', () => { }); describe('classifyAction', () => { - it('marks high priority actions and captures matched keywords', () => { + it('marks high priority actions from combined title and description text', () => { const action = normalize(classifyAction({ id: 'sec-a', agency: 'SEC', - title: 'SEC Charges Bank for Accounting Fraud', + title: 'SEC action against issuer', + description: 'The SEC secured a permanent injunction for accounting fraud.', link: 'https://example.test/sec-a', publishedAt: '2026-03-30T18:00:00.000Z', })); assert.equal(action.tier, 'high'); - assert.deepEqual(action.matchedKeywords, ['charges', 'fraud']); + assert.deepEqual(action.matchedKeywords, ['fraud', 'injunction']); }); - it('falls back to medium and then low', () => { + it('marks medium actions from description text', () => { const medium = normalize(classifyAction({ id: 'fed-a', agency: 'Federal Reserve', - title: 'Federal Reserve Issues Capital Requirement Guidance', + title: 'Federal Reserve update', + description: 'The board resolves action through a remedial action plan.', link: 'https://example.test/fed-a', publishedAt: '2026-03-30T18:00:00.000Z', })); + + assert.equal(medium.tier, 'medium'); + assert.deepEqual(medium.matchedKeywords, ['resolves action', 'remedial action']); + }); + + it('uses low only for explicit routine notice titles', () => { const low = normalize(classifyAction({ id: 'finra-a', agency: 'FINRA', - title: 'FINRA Publishes Monthly Highlights', + title: 'Technical Notice 26-01', + description: 'Routine operational bulletin for members.', link: 'https://example.test/finra-a', publishedAt: '2026-03-30T18:00:00.000Z', })); - assert.equal(medium.tier, 'medium'); - assert.deepEqual(medium.matchedKeywords, ['guidance', 'capital requirement']); assert.equal(low.tier, 'low'); assert.deepEqual(low.matchedKeywords, []); }); + + it('falls back to unknown for unmatched actions', () => { + const unknown = normalize(classifyAction({ + id: 'fdic-a', + agency: 'FDIC', + title: 'FDIC consumer outreach update', + description: 'General event recap for community stakeholders.', + link: 'https://example.test/fdic-a', + publishedAt: '2026-03-30T18:00:00.000Z', + })); + + assert.equal(unknown.tier, 'unknown'); + assert.deepEqual(unknown.matchedKeywords, []); + }); }); describe('buildSeedPayload', () => { @@ -258,31 +279,43 @@ describe('buildSeedPayload', () => { { id: 'sec-a', agency: 'SEC', - title: 'SEC Charges Bank for Fraud', + title: 'SEC action against issuer', + description: 'The SEC secured a permanent injunction for accounting fraud.', link: 'https://example.test/sec-a', publishedAt: '2026-03-30T18:00:00.000Z', }, { id: 'fed-a', agency: 'Federal Reserve', - title: 'Federal Reserve Issues Guidance', + title: 'Federal Reserve update', + description: 'The board resolves action through a remedial action plan.', link: 'https://example.test/fed-a', publishedAt: '2026-03-29T18:00:00.000Z', }, { id: 'finra-a', agency: 'FINRA', - title: 'FINRA Monthly Bulletin', + title: 'Regulatory Notice 26-01', + description: 'Routine bulletin for members.', link: 'https://example.test/finra-a', publishedAt: '2026-03-28T18:00:00.000Z', }, + { + id: 'fdic-a', + agency: 'FDIC', + title: 'FDIC consumer outreach update', + description: 'General event recap for community stakeholders.', + link: 'https://example.test/fdic-a', + publishedAt: '2026-03-27T18:00:00.000Z', + }, ], 1711718400000)); assert.equal(payload.fetchedAt, 1711718400000); - assert.equal(payload.recordCount, 3); + assert.equal(payload.recordCount, 4); assert.equal(payload.highCount, 1); assert.equal(payload.mediumCount, 1); assert.equal(payload.actions[2].tier, 'low'); + assert.equal(payload.actions[3].tier, 'unknown'); }); }); @@ -290,14 +323,14 @@ describe('fetchRegulatoryActionPayload', () => { it('returns classified payload from fetched actions', async () => { const payload = normalize(await fetchRegulatoryActionPayload(async (url) => ({ ok: true, - text: async () => `FDIC Publishes Enforcement Orders${url}/itemMon, 30 Mar 2026 18:00:00 GMT`, + text: async () => `FDIC updateFDIC resolves action through a remedial action plan.${url}/itemMon, 30 Mar 2026 18:00:00 GMT`, }))); assert.equal(payload.actions.length, 5); assert.equal(payload.recordCount, 5); assert.ok(typeof payload.fetchedAt === 'number'); - assert.equal(payload.actions[0].tier, 'high'); - assert.deepEqual(payload.actions[0].matchedKeywords, ['enforcement']); + assert.equal(payload.actions[0].tier, 'medium'); + assert.deepEqual(payload.actions[0].matchedKeywords, ['resolves action', 'remedial action']); }); }); @@ -319,7 +352,7 @@ describe('main', () => { assert.equal(calls[0].domain, 'regulatory'); assert.equal(calls[0].resource, 'actions'); assert.equal(calls[0].canonicalKey, 'regulatory:actions:v1'); - assert.equal(calls[0].opts.ttlSeconds, 7200); + assert.equal(calls[0].opts.ttlSeconds, 21600); assert.equal(calls[0].opts.validateFn({ actions: [] }), true); assert.equal(calls[0].payload.recordCount, 5); }); From 3893366abfc8695f8f1abff82bec41f9f8053249 Mon Sep 17 00:00:00 2001 From: lspassos1 Date: Wed, 1 Apr 2026 22:46:07 +0100 Subject: [PATCH 11/12] fix(intelligence): prioritize regulatory signals by severity Reject malformed timestamps, sort regulatory actions by tier before recency, and keep only high/medium signals in the cross-source extractor. --- scripts/seed-cross-source-signals.mjs | 20 ++++++++++--- .../cross-source-signals-regulatory.test.mjs | 29 +++++++++++++++---- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/scripts/seed-cross-source-signals.mjs b/scripts/seed-cross-source-signals.mjs index c16800f4a..fdb49617c 100644 --- a/scripts/seed-cross-source-signals.mjs +++ b/scripts/seed-cross-source-signals.mjs @@ -720,21 +720,33 @@ function extractRegulatoryAction(d) { const payload = d['regulatory:actions:v1']; if (!payload) return []; const cutoff = Date.now() - 48 * 3600 * 1000; + const tierPriority = { high: 0, medium: 1 }; const recent = (payload.actions || []) - .filter((action) => new Date(action.publishedAt).getTime() > cutoff && action.tier !== 'low') - .sort((a, b) => new Date(b.publishedAt).getTime() - new Date(a.publishedAt).getTime()); + .map((action) => ({ + action, + publishedAtTs: safeNum(Date.parse(action.publishedAt)), + })) + .filter(({ action, publishedAtTs }) => (action.tier === 'high' || action.tier === 'medium') && publishedAtTs > cutoff) + .sort((a, b) => { + const tierOrder = tierPriority[a.action.tier] - tierPriority[b.action.tier]; + if (tierOrder !== 0) return tierOrder; + return b.publishedAtTs - a.publishedAtTs; + }) + .slice(0, 3); if (recent.length === 0) return []; - return recent.slice(0, 3).map((action) => { + return recent.map(({ action, publishedAtTs }) => { const tierMult = action.tier === 'high' ? 1.5 : 1.0; const score = BASE_WEIGHT.CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION * tierMult; return { id: `regulatory:${action.id}`, type: 'CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION', + // Temporary mapping for the current US-only regulator set; make this + // agency-aware when non-US regulatory feeds are added. theater: 'Global Markets', summary: `${action.agency}: ${action.title}`, severity: scoreTier(score), severityScore: score, - detectedAt: new Date(action.publishedAt).getTime(), + detectedAt: publishedAtTs || Date.now(), contributingTypes: [], signalCount: 0, }; diff --git a/tests/cross-source-signals-regulatory.test.mjs b/tests/cross-source-signals-regulatory.test.mjs index f79dd49a8..6a64ef33e 100644 --- a/tests/cross-source-signals-regulatory.test.mjs +++ b/tests/cross-source-signals-regulatory.test.mjs @@ -47,7 +47,7 @@ describe('extractRegulatoryAction', () => { assert.deepEqual(normalize(extractRegulatoryAction({})), []); }); - it('emits high and medium signals, ignores low, and limits output to 3', () => { + it('emits only high and medium signals, prioritizes high before fresher medium, and limits output to 3', () => { const now = Date.now(); const payload = { 'regulatory:actions:v1': { @@ -56,7 +56,7 @@ describe('extractRegulatoryAction', () => { id: 'fdic-a', agency: 'FDIC', title: 'FDIC Guidance Update', - publishedAt: new Date(now - 2 * 3600 * 1000).toISOString(), + publishedAt: new Date(now - 1 * 3600 * 1000).toISOString(), tier: 'medium', }, { @@ -70,7 +70,7 @@ describe('extractRegulatoryAction', () => { id: 'sec-a', agency: 'SEC', title: 'SEC Charges Issuer', - publishedAt: new Date(now - 1 * 3600 * 1000).toISOString(), + publishedAt: new Date(now - 2 * 3600 * 1000).toISOString(), tier: 'high', }, { @@ -87,6 +87,20 @@ describe('extractRegulatoryAction', () => { publishedAt: new Date(now - 4 * 3600 * 1000).toISOString(), tier: 'medium', }, + { + id: 'fed-unknown', + agency: 'Federal Reserve', + title: 'Federal Reserve outreach update', + publishedAt: new Date(now - 30 * 60 * 1000).toISOString(), + tier: 'unknown', + }, + { + id: 'fdic-invalid', + agency: 'FDIC', + title: 'FDIC malformed timestamp', + publishedAt: 'not-a-date', + tier: 'high', + }, { id: 'fed-old', agency: 'Federal Reserve', @@ -102,15 +116,18 @@ describe('extractRegulatoryAction', () => { assert.equal(signals.length, 3); assert.deepEqual(signals.map((signal) => signal.id), [ 'regulatory:sec-a', + 'regulatory:sec-c', 'regulatory:fdic-a', - 'regulatory:cftc-b', ]); assert.equal(signals[0].severityScore, 3.0); assert.equal(signals[0].severity, 'CROSS_SOURCE_SIGNAL_SEVERITY_HIGH'); - assert.equal(signals[1].severityScore, 2.0); - assert.equal(signals[1].severity, 'CROSS_SOURCE_SIGNAL_SEVERITY_MEDIUM'); + assert.equal(signals[1].severityScore, 3.0); + assert.equal(signals[1].severity, 'CROSS_SOURCE_SIGNAL_SEVERITY_HIGH'); + assert.equal(signals[2].severityScore, 2.0); + assert.equal(signals[2].severity, 'CROSS_SOURCE_SIGNAL_SEVERITY_MEDIUM'); assert.equal(signals[0].theater, 'Global Markets'); assert.equal(signals[0].summary, 'SEC: SEC Charges Issuer'); + assert.ok(signals.every((signal) => Number.isFinite(signal.detectedAt))); }); }); From 02f8adc0daa14e9b75917e2607311847691b6613 Mon Sep 17 00:00:00 2001 From: lspassos1 Date: Wed, 1 Apr 2026 22:47:12 +0100 Subject: [PATCH 12/12] fix(health): track regulatory action seed freshness Register the regulatory actions Redis key and its seed-meta freshness window in the health endpoint without expanding bootstrap coverage. --- api/health.js | 2 ++ tests/regulatory-contract.test.mjs | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/api/health.js b/api/health.js index b37f13177..fda34f6cc 100644 --- a/api/health.js +++ b/api/health.js @@ -120,6 +120,7 @@ const STANDALONE_KEYS = { simulationPackageLatest: 'forecast:simulation-package:latest', simulationOutcomeLatest: 'forecast:simulation-outcome:latest', newsThreatSummary: 'news:threat:summary:v1', + regulatoryActions: 'regulatory:actions:v1', }; const SEED_META = { @@ -180,6 +181,7 @@ const SEED_META = { blsSeries: { key: 'seed-meta:economic:bls-series', maxStaleMin: 2880 }, // daily seed; 2880min = 48h = 2x interval sanctionsPressure: { key: 'seed-meta:sanctions:pressure', maxStaleMin: 720 }, crossSourceSignals: { key: 'seed-meta:intelligence:cross-source-signals', maxStaleMin: 30 }, // 15min cron; 30min = 2x interval + regulatoryActions: { key: 'seed-meta:regulatory:actions', maxStaleMin: 240 }, sanctionsEntities: { key: 'seed-meta:sanctions:entities', maxStaleMin: 1440 }, // 12h cron; 1440min = 24h = 2x interval radiationWatch: { key: 'seed-meta:radiation:observations', maxStaleMin: 30 }, groceryBasket: { key: 'seed-meta:economic:grocery-basket', maxStaleMin: 10080 }, // weekly seed; 10080 = 7 days diff --git a/tests/regulatory-contract.test.mjs b/tests/regulatory-contract.test.mjs index 687abc185..9757ddf1a 100644 --- a/tests/regulatory-contract.test.mjs +++ b/tests/regulatory-contract.test.mjs @@ -15,4 +15,20 @@ describe('regulatory cache contracts', () => { /export const REGULATORY_ACTIONS_KEY = 'regulatory:actions:v1';/ ); }); + + it('registers regulatoryActions in health standalone keys', () => { + const healthSrc = readFileSync(join(root, 'api', 'health.js'), 'utf8'); + assert.match( + healthSrc, + /regulatoryActions:\s+'regulatory:actions:v1'/ + ); + }); + + it('registers regulatoryActions seed freshness metadata in health', () => { + const healthSrc = readFileSync(join(root, 'api', 'health.js'), 'utf8'); + assert.match( + healthSrc, + /regulatoryActions:\s+\{\s+key:\s+'seed-meta:regulatory:actions',\s+maxStaleMin:\s+240\s+\}/ + ); + }); });