diff --git a/api/health.js b/api/health.js index b37f13177a..fda34f6cc2 100644 --- a/api/health.js +++ b/api/health.js @@ -120,6 +120,7 @@ const STANDALONE_KEYS = { simulationPackageLatest: 'forecast:simulation-package:latest', simulationOutcomeLatest: 'forecast:simulation-outcome:latest', newsThreatSummary: 'news:threat:summary:v1', + regulatoryActions: 'regulatory:actions:v1', }; const SEED_META = { @@ -180,6 +181,7 @@ const SEED_META = { blsSeries: { key: 'seed-meta:economic:bls-series', maxStaleMin: 2880 }, // daily seed; 2880min = 48h = 2x interval sanctionsPressure: { key: 'seed-meta:sanctions:pressure', maxStaleMin: 720 }, crossSourceSignals: { key: 'seed-meta:intelligence:cross-source-signals', maxStaleMin: 30 }, // 15min cron; 30min = 2x interval + regulatoryActions: { key: 'seed-meta:regulatory:actions', maxStaleMin: 240 }, sanctionsEntities: { key: 'seed-meta:sanctions:entities', maxStaleMin: 1440 }, // 12h cron; 1440min = 24h = 2x interval radiationWatch: { key: 'seed-meta:radiation:observations', maxStaleMin: 30 }, groceryBasket: { key: 'seed-meta:economic:grocery-basket', maxStaleMin: 10080 }, // weekly seed; 10080 = 7 days diff --git a/scripts/railway-set-watch-paths.mjs b/scripts/railway-set-watch-paths.mjs index c5b2d38e0b..2866b77588 100644 --- a/scripts/railway-set-watch-paths.mjs +++ b/scripts/railway-set-watch-paths.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node /** - * Sets watchPatterns and validates startCommand on all Railway seed services. + * Sets watchPatterns, validates startCommand, and syncs cronSchedule on Railway seed services. * * All seed services use rootDirectory="scripts", so the correct startCommand * is `node seed-.mjs` (NOT `node scripts/seed-.mjs` — that path @@ -22,6 +22,10 @@ const DRY_RUN = process.argv.includes('--dry-run'); const PROJECT_ID = '29419572-0b0d-437f-8e71-4fa68daf514f'; const ENV_ID = '91a05726-0b83-4d44-a33e-6aec94e58780'; const API = 'https://backboard.railway.app/graphql/v2'; +const REQUIRED_SEED_SERVICES = new Set(['seed-regulatory-actions']); +const EXPECTED_CRON_SCHEDULES = new Map([ + ['seed-regulatory-actions', '0 */2 * * *'], +]); // Seeds that use loadSharedConfig (depend on scripts/shared/*.json) const USES_SHARED_CONFIG = new Set([ @@ -50,6 +54,21 @@ async function gql(token, query, variables = {}) { return json.data; } +function buildExpectedPatterns(serviceName) { + const scriptFile = `scripts/${serviceName}.mjs`; + const patterns = [scriptFile, 'scripts/_seed-utils.mjs', 'scripts/package.json']; + + if (USES_SHARED_CONFIG.has(serviceName)) { + patterns.push('scripts/shared/**', 'shared/**'); + } + + if (serviceName === 'seed-iran-events') { + patterns.push('scripts/data/iran-events-latest.json'); + } + + return patterns; +} + async function main() { const token = getToken(); @@ -66,15 +85,22 @@ async function main() { .map(e => e.node) .filter(s => s.name.startsWith('seed-')); + const missingRequiredServices = [...REQUIRED_SEED_SERVICES].filter( + (name) => !services.some((service) => service.name === name) + ); + if (missingRequiredServices.length > 0) { + throw new Error(`Missing required seed service(s): ${missingRequiredServices.join(', ')}`); + } + console.log(`Found ${services.length} seed services\n`); - // 2. Check each service's watchPatterns and startCommand + // 2. Check each service's watchPatterns, startCommand, and cronSchedule for (const svc of services) { const { service } = await gql(token, ` query ($id: String!, $envId: String!) { service(id: $id) { serviceInstances(first: 1, environmentId: $envId) { - edges { node { watchPatterns startCommand } } + edges { node { watchPatterns startCommand cronSchedule } } } } } @@ -83,26 +109,20 @@ async function main() { const instance = service.serviceInstances.edges[0]?.node || {}; const currentPatterns = instance.watchPatterns || []; const currentStartCmd = instance.startCommand || ''; + const currentCronSchedule = instance.cronSchedule || ''; // rootDirectory="scripts" so startCommand must NOT include the scripts/ prefix const expectedStartCmd = `node ${svc.name}.mjs`; const startCmdOk = currentStartCmd === expectedStartCmd; + const expectedCronSchedule = EXPECTED_CRON_SCHEDULES.get(svc.name) || ''; + const hasExpectedCronSchedule = EXPECTED_CRON_SCHEDULES.has(svc.name); + const cronScheduleOk = !hasExpectedCronSchedule || currentCronSchedule === expectedCronSchedule; // Build expected watch patterns (relative to git repo root) - const scriptFile = `scripts/${svc.name}.mjs`; - const patterns = [scriptFile, 'scripts/_seed-utils.mjs', 'scripts/package.json']; - - if (USES_SHARED_CONFIG.has(svc.name)) { - patterns.push('scripts/shared/**', 'shared/**'); - } + const patterns = buildExpectedPatterns(svc.name); + const patternsOk = JSON.stringify([...currentPatterns].sort()) === JSON.stringify([...patterns].sort()); - if (svc.name === 'seed-iran-events') { - patterns.push('scripts/data/iran-events-latest.json'); - } - - const patternsOk = JSON.stringify(currentPatterns.sort()) === JSON.stringify([...patterns].sort()); - - if (patternsOk && startCmdOk) { + if (patternsOk && startCmdOk && cronScheduleOk) { console.log(` ${svc.name}: already correct`); continue; } @@ -116,6 +136,10 @@ async function main() { console.log(` watchPatterns current: ${currentPatterns.length ? currentPatterns.join(', ') : '(none)'}`); console.log(` watchPatterns setting: ${patterns.join(', ')}`); } + if (hasExpectedCronSchedule && !cronScheduleOk) { + console.log(` cronSchedule current: ${currentCronSchedule || '(none)'}`); + console.log(` cronSchedule expected: ${expectedCronSchedule}`); + } if (DRY_RUN) { console.log(` [DRY RUN] skipped\n`); @@ -126,6 +150,7 @@ async function main() { const input = {}; if (!patternsOk) input.watchPatterns = patterns; if (!startCmdOk) input.startCommand = expectedStartCmd; + if (hasExpectedCronSchedule && !cronScheduleOk) input.cronSchedule = expectedCronSchedule; await gql(token, ` mutation ($serviceId: String!, $environmentId: String!, $input: ServiceInstanceUpdateInput!) { diff --git a/scripts/seed-cross-source-signals.mjs b/scripts/seed-cross-source-signals.mjs index c4579d507b..fdb49617c5 100644 --- a/scripts/seed-cross-source-signals.mjs +++ b/scripts/seed-cross-source-signals.mjs @@ -31,6 +31,7 @@ const SOURCE_KEYS = [ 'gdelt:intel:tone:maritime', 'weather:alerts:v1', 'risk:scores:sebuf:stale:v1', + 'regulatory:actions:v1', ]; // ── Theater classification helpers ──────────────────────────────────────────── @@ -109,6 +110,7 @@ const TYPE_CATEGORY = { CROSS_SOURCE_SIGNAL_TYPE_WEATHER_EXTREME: 'natural', CROSS_SOURCE_SIGNAL_TYPE_MEDIA_TONE_DETERIORATION: 'information', CROSS_SOURCE_SIGNAL_TYPE_RISK_SCORE_SPIKE: 'intelligence', + CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION: 'policy', }; // Base severity weights for each signal type @@ -139,6 +141,7 @@ const BASE_WEIGHT = { CROSS_SOURCE_SIGNAL_TYPE_FORECAST_DETERIORATION: 1.5, // predictive — lower confidence CROSS_SOURCE_SIGNAL_TYPE_WEATHER_EXTREME: 1.5, // environmental — regional CROSS_SOURCE_SIGNAL_TYPE_MEDIA_TONE_DETERIORATION: 1.5, // sentiment — lagging + CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION: 2.0, // policy action — direct market impact }; function scoreTier(score) { @@ -713,6 +716,43 @@ function extractRiskScoreSpike(d) { }); } +function extractRegulatoryAction(d) { + const payload = d['regulatory:actions:v1']; + if (!payload) return []; + const cutoff = Date.now() - 48 * 3600 * 1000; + const tierPriority = { high: 0, medium: 1 }; + const recent = (payload.actions || []) + .map((action) => ({ + action, + publishedAtTs: safeNum(Date.parse(action.publishedAt)), + })) + .filter(({ action, publishedAtTs }) => (action.tier === 'high' || action.tier === 'medium') && publishedAtTs > cutoff) + .sort((a, b) => { + const tierOrder = tierPriority[a.action.tier] - tierPriority[b.action.tier]; + if (tierOrder !== 0) return tierOrder; + return b.publishedAtTs - a.publishedAtTs; + }) + .slice(0, 3); + if (recent.length === 0) return []; + return recent.map(({ action, publishedAtTs }) => { + const tierMult = action.tier === 'high' ? 1.5 : 1.0; + const score = BASE_WEIGHT.CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION * tierMult; + return { + id: `regulatory:${action.id}`, + type: 'CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION', + // Temporary mapping for the current US-only regulator set; make this + // agency-aware when non-US regulatory feeds are added. + theater: 'Global Markets', + summary: `${action.agency}: ${action.title}`, + severity: scoreTier(score), + severityScore: score, + detectedAt: publishedAtTs || Date.now(), + contributingTypes: [], + signalCount: 0, + }; + }); +} + // ── Composite escalation detector ───────────────────────────────────────────── // Fires when >=3 signals from DIFFERENT categories share the same theater. function detectCompositeEscalation(signals) { @@ -790,6 +830,7 @@ async function aggregateCrossSourceSignals() { extractWeatherExtreme, extractMediaToneDeterioration, extractRiskScoreSpike, + extractRegulatoryAction, ]; for (const extractor of extractors) { diff --git a/scripts/seed-regulatory-actions.mjs b/scripts/seed-regulatory-actions.mjs new file mode 100644 index 0000000000..a320c23898 --- /dev/null +++ b/scripts/seed-regulatory-actions.mjs @@ -0,0 +1,367 @@ +#!/usr/bin/env node +// @ts-check + +import { pathToFileURL } from 'node:url'; +import { CHROME_UA, loadEnvFile, runSeed } from './_seed-utils.mjs'; + +loadEnvFile(import.meta.url); + +const CANONICAL_KEY = 'regulatory:actions:v1'; +const FEED_TIMEOUT_MS = 15_000; +const TTL_SECONDS = 21600; +const XML_ACCEPT = 'application/atom+xml, application/rss+xml, application/xml, text/xml, */*'; +const SEC_USER_AGENT = 'WorldMonitor/2.0 (monitor@worldmonitor.app)'; +const DEFAULT_FETCH = (...args) => globalThis.fetch(...args); +const HIGH_KEYWORDS = [ + 'enforcement', 'charges', 'charged', 'fraud', 'failure', 'failed bank', + 'emergency', 'halt', 'suspension', 'suspended', 'cease', 'desist', + 'penalty', 'fine', 'fined', 'settlement', 'indictment', 'manipulation', + 'ban', 'revocation', 'insolvency', 'injunction', 'cease and desist', + 'cease-and-desist', 'consent order', 'debarment', 'suspension order', +]; +const MEDIUM_KEYWORDS = [ + 'proposed rule', 'final rule', 'rulemaking', 'guidance', 'warning', + 'advisory', 'review', 'examination', 'investigation', + 'stress test', 'capital requirement', 'disclosure requirement', + 'resolves action', 'settled charges', 'administrative proceeding', 'remedial action', +]; +const LOW_PRIORITY_TITLE_PATTERNS = [ + /^(Regulatory|Information|Technical) Notice\b/i, + /\bmonthly (highlights|bulletin)\b/i, +]; + +const REGULATORY_FEEDS = [ + { agency: 'SEC', url: 'https://www.sec.gov/news/pressreleases.rss', userAgent: SEC_USER_AGENT }, + { agency: 'CFTC', url: 'https://www.cftc.gov/RSS/RSSENF/rssenf.xml' }, + { agency: 'Federal Reserve', url: 'https://www.federalreserve.gov/feeds/press_all.xml' }, + { agency: 'FDIC', url: 'https://public.govdelivery.com/topics/USFDIC_26/feed.rss' }, + // FINRA still publishes this RSS endpoint over plain HTTP; HTTPS requests fail + // from both Node fetch and curl in validation, so keep the official feed URL + // and periodically recheck whether HTTPS starts working. + { agency: 'FINRA', url: 'http://feeds.finra.org/FINRANotices' }, +]; + +function decodeEntities(input) { + if (!input) return ''; + const named = input + .replace(/&/gi, '&') + .replace(/</gi, '<') + .replace(/>/gi, '>') + .replace(/"/gi, '"') + .replace(/'/gi, "'") + .replace(/ /gi, ' '); + + return named + .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code))) + .replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCodePoint(parseInt(code, 16))); +} + +function stripHtml(input) { + return decodeEntities( + String(input || '') + .replace(//g, '$1') + .replace(/<[^>]+>/g, ' ') + ).replace(/\s+/g, ' ').trim(); +} + +function getTagValue(block, tagName) { + const match = block.match(new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>`, 'i')); + return stripHtml(match?.[1] || ''); +} + +function extractAtomLink(block) { + const linkTags = [...block.matchAll(/]*)\/?>/gi)]; + if (linkTags.length === 0) return ''; + + for (const [, attrs] of linkTags) { + const href = attrs.match(/\bhref=["']([^"']+)["']/i)?.[1]; + const rel = attrs.match(/\brel=["']([^"']+)["']/i)?.[1]?.toLowerCase() || ''; + if (href && (!rel || rel === 'alternate')) return decodeEntities(href.trim()); + } + + for (const [, attrs] of linkTags) { + const href = attrs.match(/\bhref=["']([^"']+)["']/i)?.[1]; + if (href) return decodeEntities(href.trim()); + } + + return ''; +} + +function resolveFeedLink(link, feedUrl) { + if (!link) return ''; + try { + return new URL(link).href; + } catch {} + try { + return new URL(link, feedUrl).href; + } catch { + return ''; + } +} + +function canonicalizeLink(link, feedUrl = '') { + const resolved = resolveFeedLink(link, feedUrl); + if (!resolved) return ''; + try { + const url = new URL(resolved); + url.hash = ''; + return url.href; + } catch { + return ''; + } +} + +function toIsoDate(rawDate) { + const value = stripHtml(rawDate); + if (!value) return ''; + const ts = Date.parse(value); + return Number.isFinite(ts) ? new Date(ts).toISOString() : ''; +} + +function slugifyTitle(title) { + return stripHtml(title) + .normalize('NFKD') + .replace(/[\u0300-\u036f]/g, '') + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 80); +} + +function yyyymmdd(isoDate) { + return String(isoDate || '').slice(0, 10).replace(/-/g, ''); +} + +function hhmmss(isoDate) { + return String(isoDate || '').slice(11, 19).replace(/:/g, ''); +} + +function buildActionId(agency, title, publishedAt) { + const agencySlug = slugifyTitle(agency) || 'agency'; + const titleSlug = slugifyTitle(title) || 'untitled'; + const datePart = yyyymmdd(publishedAt) || 'undated'; + const timePart = hhmmss(publishedAt) || '000000'; + return `${agencySlug}-${titleSlug}-${datePart}-${timePart}`; +} + +function parseRssItems(xml, feedUrl) { + const items = []; + const itemRegex = /]*>([\s\S]*?)<\/item>/gi; + let match; + while ((match = itemRegex.exec(xml)) !== null) { + const block = match[1]; + const title = getTagValue(block, 'title'); + const description = getTagValue(block, 'description'); + const link = canonicalizeLink(getTagValue(block, 'link'), feedUrl); + const publishedAt = toIsoDate(getTagValue(block, 'pubDate') || getTagValue(block, 'updated')); + items.push({ title, description, link, publishedAt }); + } + return items; +} + +function parseAtomEntries(xml, feedUrl) { + const entries = []; + const entryRegex = /]*>([\s\S]*?)<\/entry>/gi; + let match; + while ((match = entryRegex.exec(xml)) !== null) { + const block = match[1]; + const title = getTagValue(block, 'title'); + const description = getTagValue(block, 'summary') || getTagValue(block, 'content'); + const link = canonicalizeLink(extractAtomLink(block), feedUrl); + const publishedAt = toIsoDate( + getTagValue(block, 'updated') || getTagValue(block, 'published') || getTagValue(block, 'pubDate') + ); + entries.push({ title, description, link, publishedAt }); + } + return entries; +} + +function parseFeed(xml, feedUrl) { + if (/ item.title && item.link && item.publishedAt) + .map((item) => ({ + id: buildActionId(agency, item.title, item.publishedAt), + agency, + title: item.title, + description: item.description || '', + link: item.link, + publishedAt: item.publishedAt, + })); +} + +function dedupeAndSortActions(actions) { + const seen = new Set(); + const deduped = []; + for (const action of actions) { + const key = canonicalizeLink(action.link); + if (!key || seen.has(key)) continue; + seen.add(key); + deduped.push({ ...action, link: key }); + } + + deduped.sort((a, b) => Date.parse(b.publishedAt) - Date.parse(a.publishedAt)); + return deduped; +} + +async function fetchFeed(feed, fetchImpl = DEFAULT_FETCH) { + const headers = { + Accept: XML_ACCEPT, + 'User-Agent': feed.userAgent || CHROME_UA, + }; + + const response = await fetchImpl(feed.url, { + headers, + signal: AbortSignal.timeout(FEED_TIMEOUT_MS), + }); + + if (!response.ok) { + throw new Error(`${feed.agency}: HTTP ${response.status}`); + } + + const xml = await response.text(); + const parsed = parseFeed(xml, feed.url); + return normalizeFeedItems(parsed, feed.agency); +} + +async function fetchAllFeeds(fetchImpl = DEFAULT_FETCH, feeds = REGULATORY_FEEDS) { + const results = await Promise.allSettled(feeds.map((feed) => fetchFeed(feed, fetchImpl))); + const actions = []; + let successCount = 0; + + for (let index = 0; index < results.length; index += 1) { + const result = results[index]; + const feed = feeds[index]; + if (result.status === 'fulfilled') { + successCount += 1; + actions.push(...result.value); + continue; + } + console.error(`[regulatory] ${feed.agency}: ${result.reason?.message || result.reason}`); + } + + if (successCount === 0) { + throw new Error('All regulatory feeds failed'); + } + + return dedupeAndSortActions(actions); +} + +function escapeRegex(value) { + return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function compileKeywordPattern(keyword) { + const pattern = `\\b${escapeRegex(keyword.toLowerCase()).replace(/\s+/g, '\\s+')}\\b`; + return { keyword, regex: new RegExp(pattern, 'i') }; +} + +const HIGH_KEYWORD_PATTERNS = HIGH_KEYWORDS.map(compileKeywordPattern); +const MEDIUM_KEYWORD_PATTERNS = MEDIUM_KEYWORDS.map(compileKeywordPattern); + +function findMatchedKeywords(text, keywordPatterns) { + const normalizedText = stripHtml(text).toLowerCase(); + return keywordPatterns.filter(({ regex }) => regex.test(normalizedText)).map(({ keyword }) => keyword); +} + +function buildClassificationText(action) { + return [action.title, action.description].filter(Boolean).join(' '); +} + +function isLowPriorityRoutineTitle(title) { + const normalizedTitle = stripHtml(title); + return LOW_PRIORITY_TITLE_PATTERNS.some((pattern) => pattern.test(normalizedTitle)); +} + +function classifyAction(action) { + const classificationText = buildClassificationText(action); + const highMatches = findMatchedKeywords(classificationText, HIGH_KEYWORD_PATTERNS); + if (highMatches.length > 0) { + return { ...action, tier: 'high', matchedKeywords: [...new Set(highMatches)] }; + } + + if (isLowPriorityRoutineTitle(action.title)) { + return { ...action, tier: 'low', matchedKeywords: [] }; + } + + const mediumMatches = findMatchedKeywords(classificationText, MEDIUM_KEYWORD_PATTERNS); + if (mediumMatches.length > 0) { + return { ...action, tier: 'medium', matchedKeywords: [...new Set(mediumMatches)] }; + } + + return { ...action, tier: 'unknown', matchedKeywords: [] }; +} + +function buildSeedPayload(actions, fetchedAt = Date.now()) { + const classified = actions.map(classifyAction); + const highCount = classified.filter((action) => action.tier === 'high').length; + const mediumCount = classified.filter((action) => action.tier === 'medium').length; + + return { + actions: classified, + fetchedAt, + recordCount: classified.length, + highCount, + mediumCount, + }; +} + +async function fetchRegulatoryActionPayload(fetchImpl = DEFAULT_FETCH) { + const actions = await fetchAllFeeds(fetchImpl); + return buildSeedPayload(actions, Date.now()); +} + +async function main(fetchImpl = DEFAULT_FETCH, runSeedImpl = runSeed) { + return runSeedImpl('regulatory', 'actions', CANONICAL_KEY, () => fetchRegulatoryActionPayload(fetchImpl), { + ttlSeconds: TTL_SECONDS, + validateFn: (data) => Array.isArray(data?.actions), + recordCount: (data) => data?.recordCount || 0, + sourceVersion: 'regulatory-rss-v1', + }); +} + +const isDirectRun = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; + +if (isDirectRun) { + main().catch((err) => { + console.error(`FETCH FAILED: ${err.message || err}`); + process.exit(1); + }); +} + +export { + CANONICAL_KEY, + CHROME_UA, + FEED_TIMEOUT_MS, + HIGH_KEYWORDS, + MEDIUM_KEYWORDS, + REGULATORY_FEEDS, + SEC_USER_AGENT, + TTL_SECONDS, + buildActionId, + buildSeedPayload, + canonicalizeLink, + classifyAction, + decodeEntities, + dedupeAndSortActions, + extractAtomLink, + fetchAllFeeds, + fetchFeed, + fetchRegulatoryActionPayload, + findMatchedKeywords, + getTagValue, + isLowPriorityRoutineTitle, + main, + normalizeFeedItems, + parseAtomEntries, + parseFeed, + parseRssItems, + resolveFeedLink, + slugifyTitle, + stripHtml, + toIsoDate, +}; diff --git a/server/_shared/cache-keys.ts b/server/_shared/cache-keys.ts index fb48505afd..f656e6fe86 100644 --- a/server/_shared/cache-keys.ts +++ b/server/_shared/cache-keys.ts @@ -5,6 +5,7 @@ */ export const SIMULATION_OUTCOME_LATEST_KEY = 'forecast:simulation-outcome:latest'; export const SIMULATION_PACKAGE_LATEST_KEY = 'forecast:simulation-package:latest'; +export const REGULATORY_ACTIONS_KEY = 'regulatory:actions:v1'; /** * Static cache keys for the bootstrap endpoint. diff --git a/tests/cross-source-signals-regulatory.test.mjs b/tests/cross-source-signals-regulatory.test.mjs new file mode 100644 index 0000000000..6a64ef33e5 --- /dev/null +++ b/tests/cross-source-signals-regulatory.test.mjs @@ -0,0 +1,179 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import vm from 'node:vm'; + +function normalize(value) { + return JSON.parse(JSON.stringify(value)); +} + +const seedSrc = readFileSync('scripts/seed-cross-source-signals.mjs', 'utf8'); + +const pureSrc = seedSrc + .replace(/^import\s.*$/gm, '') + .replace(/loadEnvFile\([^)]+\);\n/, '') + .replace(/async function readAllSourceKeys[\s\S]*?\n}\n\n\/\/ ── Signal extractors/m, '// readAllSourceKeys removed for unit test\n\n// ── Signal extractors') + .replace(/runSeed\('intelligence'[\s\S]*$/m, ''); + +const ctx = vm.createContext({ console, Date, Math, Number, Array, Map, Set, String, RegExp }); +vm.runInContext(`${pureSrc}\n;globalThis.__exports = { SOURCE_KEYS, TYPE_CATEGORY, BASE_WEIGHT, scoreTier, extractRegulatoryAction, detectCompositeEscalation };`, ctx); + +const { + SOURCE_KEYS, + TYPE_CATEGORY, + BASE_WEIGHT, + scoreTier, + extractRegulatoryAction, + detectCompositeEscalation, +} = ctx.__exports; + +describe('source registration', () => { + it('adds the regulatory seed key to SOURCE_KEYS', () => { + assert.ok(SOURCE_KEYS.includes('regulatory:actions:v1')); + }); + + it('maps regulatory actions to the policy category and a 2.0 base weight', () => { + assert.equal(TYPE_CATEGORY.CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION, 'policy'); + assert.equal(BASE_WEIGHT.CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION, 2.0); + }); + + it('registers the extractor in the extractor list', () => { + assert.match(seedSrc, /extractRegulatoryAction,\n\s*];/); + }); +}); + +describe('extractRegulatoryAction', () => { + it('returns an empty array when the source key is missing', () => { + assert.deepEqual(normalize(extractRegulatoryAction({})), []); + }); + + it('emits only high and medium signals, prioritizes high before fresher medium, and limits output to 3', () => { + const now = Date.now(); + const payload = { + 'regulatory:actions:v1': { + actions: [ + { + id: 'fdic-a', + agency: 'FDIC', + title: 'FDIC Guidance Update', + publishedAt: new Date(now - 1 * 3600 * 1000).toISOString(), + tier: 'medium', + }, + { + id: 'sec-c', + agency: 'SEC', + title: 'SEC Settlement', + publishedAt: new Date(now - 5 * 3600 * 1000).toISOString(), + tier: 'high', + }, + { + id: 'sec-a', + agency: 'SEC', + title: 'SEC Charges Issuer', + publishedAt: new Date(now - 2 * 3600 * 1000).toISOString(), + tier: 'high', + }, + { + id: 'finra-low', + agency: 'FINRA', + title: 'FINRA Monthly Bulletin', + publishedAt: new Date(now - 3 * 3600 * 1000).toISOString(), + tier: 'low', + }, + { + id: 'cftc-b', + agency: 'CFTC', + title: 'CFTC Advisory Notice', + publishedAt: new Date(now - 4 * 3600 * 1000).toISOString(), + tier: 'medium', + }, + { + id: 'fed-unknown', + agency: 'Federal Reserve', + title: 'Federal Reserve outreach update', + publishedAt: new Date(now - 30 * 60 * 1000).toISOString(), + tier: 'unknown', + }, + { + id: 'fdic-invalid', + agency: 'FDIC', + title: 'FDIC malformed timestamp', + publishedAt: 'not-a-date', + tier: 'high', + }, + { + id: 'fed-old', + agency: 'Federal Reserve', + title: 'Old Enforcement Notice', + publishedAt: new Date(now - 72 * 3600 * 1000).toISOString(), + tier: 'high', + }, + ], + }, + }; + + const signals = normalize(extractRegulatoryAction(payload)); + assert.equal(signals.length, 3); + assert.deepEqual(signals.map((signal) => signal.id), [ + 'regulatory:sec-a', + 'regulatory:sec-c', + 'regulatory:fdic-a', + ]); + assert.equal(signals[0].severityScore, 3.0); + assert.equal(signals[0].severity, 'CROSS_SOURCE_SIGNAL_SEVERITY_HIGH'); + assert.equal(signals[1].severityScore, 3.0); + assert.equal(signals[1].severity, 'CROSS_SOURCE_SIGNAL_SEVERITY_HIGH'); + assert.equal(signals[2].severityScore, 2.0); + assert.equal(signals[2].severity, 'CROSS_SOURCE_SIGNAL_SEVERITY_MEDIUM'); + assert.equal(signals[0].theater, 'Global Markets'); + assert.equal(signals[0].summary, 'SEC: SEC Charges Issuer'); + assert.ok(signals.every((signal) => Number.isFinite(signal.detectedAt))); + }); +}); + +describe('detectCompositeEscalation', () => { + it('fires when policy, financial, and economic signals co-fire in Global Markets', () => { + const composite = normalize(detectCompositeEscalation([ + { + id: 'regulatory:sec-a', + type: 'CROSS_SOURCE_SIGNAL_TYPE_REGULATORY_ACTION', + theater: 'Global Markets', + summary: 'SEC: SEC Charges Issuer', + severity: scoreTier(3.0), + severityScore: 3.0, + detectedAt: Date.now(), + contributingTypes: [], + signalCount: 0, + }, + { + id: 'vix:global', + type: 'CROSS_SOURCE_SIGNAL_TYPE_VIX_SPIKE', + theater: 'Global Markets', + summary: 'VIX elevated', + severity: scoreTier(2.0), + severityScore: 2.0, + detectedAt: Date.now(), + contributingTypes: [], + signalCount: 0, + }, + { + id: 'commodity:oil', + type: 'CROSS_SOURCE_SIGNAL_TYPE_COMMODITY_SHOCK', + theater: 'Global Markets', + summary: 'Oil shock', + severity: scoreTier(2.0), + severityScore: 2.0, + detectedAt: Date.now(), + contributingTypes: [], + signalCount: 0, + }, + ])); + + assert.equal(composite.length, 1); + assert.equal(composite[0].type, 'CROSS_SOURCE_SIGNAL_TYPE_COMPOSITE_ESCALATION'); + assert.equal(composite[0].theater, 'Global Markets'); + assert.ok(composite[0].contributingTypes.includes('regulatory action')); + assert.ok(composite[0].contributingTypes.includes('vix spike')); + assert.ok(composite[0].contributingTypes.includes('commodity shock')); + }); +}); diff --git a/tests/railway-set-watch-paths.test.mjs b/tests/railway-set-watch-paths.test.mjs new file mode 100644 index 0000000000..528a362ffb --- /dev/null +++ b/tests/railway-set-watch-paths.test.mjs @@ -0,0 +1,26 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; + +const scriptSrc = readFileSync('scripts/railway-set-watch-paths.mjs', 'utf8'); + +describe('railway-set-watch-paths regulatory cron sync', () => { + it('declares seed-regulatory-actions as a required seed service', () => { + assert.match(scriptSrc, /REQUIRED_SEED_SERVICES = new Set\(\['seed-regulatory-actions'\]\)/); + assert.match(scriptSrc, /Missing required seed service\(s\):/); + }); + + it('defines the expected 2-hour cron schedule for seed-regulatory-actions', () => { + assert.match(scriptSrc, /EXPECTED_CRON_SCHEDULES = new Map\(\[\s*\['seed-regulatory-actions', '0 \*\/2 \* \* \*'\]/s); + }); + + it('queries and updates cronSchedule via serviceInstanceUpdate', () => { + assert.match(scriptSrc, /node \{ watchPatterns startCommand cronSchedule \}/); + assert.match(scriptSrc, /input\.cronSchedule = expectedCronSchedule/); + }); + + it('continues to derive watch patterns from the seed name', () => { + assert.match(scriptSrc, /function buildExpectedPatterns\(serviceName\)/); + assert.match(scriptSrc, /const scriptFile = `scripts\/\$\{serviceName\}\.mjs`/); + }); +}); diff --git a/tests/regulatory-contract.test.mjs b/tests/regulatory-contract.test.mjs new file mode 100644 index 0000000000..9757ddf1a1 --- /dev/null +++ b/tests/regulatory-contract.test.mjs @@ -0,0 +1,34 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const root = join(__dirname, '..'); + +describe('regulatory cache contracts', () => { + it('exports REGULATORY_ACTIONS_KEY from cache-keys.ts', () => { + const cacheKeysSrc = readFileSync(join(root, 'server', '_shared', 'cache-keys.ts'), 'utf8'); + assert.match( + cacheKeysSrc, + /export const REGULATORY_ACTIONS_KEY = 'regulatory:actions:v1';/ + ); + }); + + it('registers regulatoryActions in health standalone keys', () => { + const healthSrc = readFileSync(join(root, 'api', 'health.js'), 'utf8'); + assert.match( + healthSrc, + /regulatoryActions:\s+'regulatory:actions:v1'/ + ); + }); + + it('registers regulatoryActions seed freshness metadata in health', () => { + const healthSrc = readFileSync(join(root, 'api', 'health.js'), 'utf8'); + assert.match( + healthSrc, + /regulatoryActions:\s+\{\s+key:\s+'seed-meta:regulatory:actions',\s+maxStaleMin:\s+240\s+\}/ + ); + }); +}); diff --git a/tests/regulatory-seed-unit.test.mjs b/tests/regulatory-seed-unit.test.mjs new file mode 100644 index 0000000000..02323218b7 --- /dev/null +++ b/tests/regulatory-seed-unit.test.mjs @@ -0,0 +1,359 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import vm from 'node:vm'; + +function normalize(value) { + return JSON.parse(JSON.stringify(value)); +} + +const seedSrc = readFileSync('scripts/seed-regulatory-actions.mjs', 'utf8'); + +const pureSrc = seedSrc + .replace(/^import\s.*$/gm, '') + .replace(/loadEnvFile\([^)]+\);\n/, '') + .replace(/const isDirectRun[\s\S]*?}\n\nexport\s*{[\s\S]*?};?\s*$/m, ''); + +const ctx = vm.createContext({ + console, + Date, + Math, + Number, + Array, + Set, + String, + RegExp, + URL, + URLSearchParams, + AbortSignal, + CHROME_UA: 'Mozilla/5.0 (test)', + loadEnvFile: () => {}, + runSeed: async () => {}, +}); + +vm.runInContext(pureSrc, ctx); + +const { + decodeEntities, + stripHtml, + extractAtomLink, + parseRssItems, + parseAtomEntries, + parseFeed, + normalizeFeedItems, + dedupeAndSortActions, + fetchAllFeeds, + classifyAction, + buildSeedPayload, + fetchRegulatoryActionPayload, + main, +} = ctx; + +describe('decodeEntities', () => { + it('decodes named and numeric entities', () => { + assert.equal(decodeEntities('Tom & Jerry & &'), 'Tom & Jerry & &'); + }); +}); + +describe('stripHtml', () => { + it('removes tags and CDATA while preserving text', () => { + assert.equal(stripHtml('world]]>'), 'Hello world'); + }); +}); + +describe('parseRssItems', () => { + it('extracts RSS items with description, normalized links, and pubDate', () => { + const xml = ` + + + <![CDATA[SEC & Co. Charges <b>Issuer</b>]]> + fraud & disclosure failures]]> + /news/press-release/2026-10 + Mon, 30 Mar 2026 18:00:00 GMT + + `; + + assert.deepEqual(normalize(parseRssItems(xml, 'https://www.sec.gov/news/pressreleases.rss')), [{ + title: 'SEC & Co. Charges Issuer', + description: 'Alleges fraud & disclosure failures', + link: 'https://www.sec.gov/news/press-release/2026-10', + publishedAt: '2026-03-30T18:00:00.000Z', + }]); + }); +}); + +describe('extractAtomLink + parseAtomEntries', () => { + it('prefers alternate href and extracts summary/content with normalized publishedAt', () => { + const xml = ` + + + Fed issues notice + policy summary]]> + + + 2026-03-29T12:30:00Z + + `; + + assert.equal( + extractAtomLink(''), + '/press/notice-a' + ); + + assert.deepEqual(normalize(parseAtomEntries(xml, 'https://www.federalreserve.gov/feeds/press_all.xml')), [{ + title: 'Fed issues notice', + description: 'Detailed policy summary', + link: 'https://www.federalreserve.gov/press/notice-a', + publishedAt: '2026-03-29T12:30:00.000Z', + }]); + + const contentXml = ` + + + FDIC update + Formal administrative note

]]> + + 2026-03-28T09:15:00Z + + `; + + assert.deepEqual(normalize(parseAtomEntries(contentXml, 'https://www.fdic.gov/feed')), [{ + title: 'FDIC update', + description: 'Formal administrative note', + link: 'https://fdic.example.test/a', + publishedAt: '2026-03-28T09:15:00.000Z', + }]); + }); +}); + +describe('parseFeed', () => { + it('detects Atom feeds automatically', () => { + const atom = 'A2026-03-28T00:00:00Z'; + const parsed = normalize(parseFeed(atom, 'https://example.test/feed')); + assert.equal(parsed.length, 1); + assert.equal(parsed[0].link, 'https://example.test/a'); + }); +}); + +describe('normalizeFeedItems', () => { + it('skips incomplete entries and generates deterministic ids', () => { + const normalized = normalize(normalizeFeedItems([ + { title: 'SEC Charges XYZ Corp', link: 'https://example.test/sec', publishedAt: '2026-03-29T14:00:00.000Z' }, + { title: 'SEC Summary', description: 'extra context', link: 'https://example.test/sec-2', publishedAt: '2026-03-29T14:30:00.000Z' }, + { title: '', link: 'https://example.test/missing', publishedAt: '2026-03-29T14:00:00.000Z' }, + ], 'SEC')); + + assert.equal(normalized.length, 2); + assert.equal(normalized[0].id, 'sec-sec-charges-xyz-corp-20260329-140000'); + assert.equal(normalized[0].description, ''); + assert.equal(normalized[1].description, 'extra context'); + }); +}); + +describe('dedupeAndSortActions', () => { + it('deduplicates by canonical link and sorts newest first', () => { + const actions = normalize(dedupeAndSortActions([ + { + id: 'older', + agency: 'SEC', + title: 'Older', + link: 'https://example.test/path#frag', + publishedAt: '2026-03-28T10:00:00.000Z', + }, + { + id: 'newer', + agency: 'FDIC', + title: 'Newer', + link: 'https://example.test/new', + publishedAt: '2026-03-30T10:00:00.000Z', + }, + { + id: 'duplicate', + agency: 'SEC', + title: 'Duplicate', + link: 'https://example.test/path', + publishedAt: '2026-03-29T10:00:00.000Z', + }, + ])); + + assert.deepEqual(actions.map((item) => item.id), ['newer', 'older']); + assert.equal(actions[1].link, 'https://example.test/path'); + }); +}); + +describe('fetchAllFeeds', () => { + const feeds = [ + { agency: 'SEC', url: 'https://feeds.test/sec', userAgent: 'Custom-SEC-UA' }, + { agency: 'FDIC', url: 'https://feeds.test/fdic' }, + ]; + + it('returns normalized aggregate when at least one feed succeeds', async () => { + const requests = []; + const fetchStub = async (url, options) => { + requests.push({ url, options }); + if (url.endsWith('/sec')) { + return { + ok: true, + text: async () => `SEC Charges Bankhttps://sec.test/aMon, 30 Mar 2026 18:00:00 GMT`, + }; + } + throw new Error('FDIC timeout'); + }; + + const result = normalize(await fetchAllFeeds(fetchStub, feeds)); + assert.equal(result.length, 1); + assert.equal(result[0].agency, 'SEC'); + assert.equal(requests[0].options.headers['User-Agent'], 'Custom-SEC-UA'); + assert.equal(requests[1].options.headers['User-Agent'], ctx.CHROME_UA); + }); + + it('throws when all feeds fail', async () => { + await assert.rejects( + fetchAllFeeds(async () => { throw new Error('nope'); }, feeds), + /All regulatory feeds failed/ + ); + }); +}); + +describe('classifyAction', () => { + it('marks high priority actions from combined title and description text', () => { + const action = normalize(classifyAction({ + id: 'sec-a', + agency: 'SEC', + title: 'SEC action against issuer', + description: 'The SEC secured a permanent injunction for accounting fraud.', + link: 'https://example.test/sec-a', + publishedAt: '2026-03-30T18:00:00.000Z', + })); + + assert.equal(action.tier, 'high'); + assert.deepEqual(action.matchedKeywords, ['fraud', 'injunction']); + }); + + it('marks medium actions from description text', () => { + const medium = normalize(classifyAction({ + id: 'fed-a', + agency: 'Federal Reserve', + title: 'Federal Reserve update', + description: 'The board resolves action through a remedial action plan.', + link: 'https://example.test/fed-a', + publishedAt: '2026-03-30T18:00:00.000Z', + })); + + assert.equal(medium.tier, 'medium'); + assert.deepEqual(medium.matchedKeywords, ['resolves action', 'remedial action']); + }); + + it('uses low only for explicit routine notice titles', () => { + const low = normalize(classifyAction({ + id: 'finra-a', + agency: 'FINRA', + title: 'Technical Notice 26-01', + description: 'Routine operational bulletin for members.', + link: 'https://example.test/finra-a', + publishedAt: '2026-03-30T18:00:00.000Z', + })); + + assert.equal(low.tier, 'low'); + assert.deepEqual(low.matchedKeywords, []); + }); + + it('falls back to unknown for unmatched actions', () => { + const unknown = normalize(classifyAction({ + id: 'fdic-a', + agency: 'FDIC', + title: 'FDIC consumer outreach update', + description: 'General event recap for community stakeholders.', + link: 'https://example.test/fdic-a', + publishedAt: '2026-03-30T18:00:00.000Z', + })); + + assert.equal(unknown.tier, 'unknown'); + assert.deepEqual(unknown.matchedKeywords, []); + }); +}); + +describe('buildSeedPayload', () => { + it('adds fetchedAt and aggregate counts', () => { + const payload = normalize(buildSeedPayload([ + { + id: 'sec-a', + agency: 'SEC', + title: 'SEC action against issuer', + description: 'The SEC secured a permanent injunction for accounting fraud.', + link: 'https://example.test/sec-a', + publishedAt: '2026-03-30T18:00:00.000Z', + }, + { + id: 'fed-a', + agency: 'Federal Reserve', + title: 'Federal Reserve update', + description: 'The board resolves action through a remedial action plan.', + link: 'https://example.test/fed-a', + publishedAt: '2026-03-29T18:00:00.000Z', + }, + { + id: 'finra-a', + agency: 'FINRA', + title: 'Regulatory Notice 26-01', + description: 'Routine bulletin for members.', + link: 'https://example.test/finra-a', + publishedAt: '2026-03-28T18:00:00.000Z', + }, + { + id: 'fdic-a', + agency: 'FDIC', + title: 'FDIC consumer outreach update', + description: 'General event recap for community stakeholders.', + link: 'https://example.test/fdic-a', + publishedAt: '2026-03-27T18:00:00.000Z', + }, + ], 1711718400000)); + + assert.equal(payload.fetchedAt, 1711718400000); + assert.equal(payload.recordCount, 4); + assert.equal(payload.highCount, 1); + assert.equal(payload.mediumCount, 1); + assert.equal(payload.actions[2].tier, 'low'); + assert.equal(payload.actions[3].tier, 'unknown'); + }); +}); + +describe('fetchRegulatoryActionPayload', () => { + it('returns classified payload from fetched actions', async () => { + const payload = normalize(await fetchRegulatoryActionPayload(async (url) => ({ + ok: true, + text: async () => `FDIC updateFDIC resolves action through a remedial action plan.${url}/itemMon, 30 Mar 2026 18:00:00 GMT`, + }))); + + assert.equal(payload.actions.length, 5); + assert.equal(payload.recordCount, 5); + assert.ok(typeof payload.fetchedAt === 'number'); + assert.equal(payload.actions[0].tier, 'medium'); + assert.deepEqual(payload.actions[0].matchedKeywords, ['resolves action', 'remedial action']); + }); +}); + +describe('main', () => { + it('wires runSeed with the regulatory key, TTL, and validateFn', async () => { + const calls = []; + const runSeedStub = async (domain, resource, canonicalKey, fetchFn, opts) => { + calls.push({ domain, resource, canonicalKey, opts, payload: await fetchFn() }); + return 'ok'; + }; + const fetchStub = async (url) => ({ + ok: true, + text: async () => `CFTC Issues Advisory${url}/itemMon, 30 Mar 2026 18:00:00 GMT`, + }); + + const result = await main(fetchStub, runSeedStub); + assert.equal(result, 'ok'); + assert.equal(calls.length, 1); + assert.equal(calls[0].domain, 'regulatory'); + assert.equal(calls[0].resource, 'actions'); + assert.equal(calls[0].canonicalKey, 'regulatory:actions:v1'); + assert.equal(calls[0].opts.ttlSeconds, 21600); + assert.equal(calls[0].opts.validateFn({ actions: [] }), true); + assert.equal(calls[0].payload.recordCount, 5); + }); +});