Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 81 additions & 22 deletions scripts/seed-forecasts.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4604,14 +4604,16 @@ function summarizeImpactPathScore(path = null) {
if (path.simulationAdjustmentDetail !== undefined) {
const d = path.simulationAdjustmentDetail;
summary.simDetail = {
bucketChannelMatch: Boolean(d.bucketChannelMatch),
actorOverlapCount: Number(d.actorOverlapCount),
candidateActorCount: Number(d.candidateActorCount),
actorSource: d.actorSource,
resolvedChannel: d.resolvedChannel || '',
channelSource: d.channelSource,
invalidatorHit: Boolean(d.invalidatorHit),
stabilizerHit: Boolean(d.stabilizerHit),
bucketChannelMatch: Boolean(d.bucketChannelMatch),
actorOverlapCount: Number(d.actorOverlapCount),
roleOverlapCount: Number(d.roleOverlapCount ?? d.actorOverlapCount),
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 roleOverlapCount fallback misattributes old entity-overlap data

The fallback ?? d.actorOverlapCount is intended to be defensive for old simulationAdjustmentDetail objects that pre-date this PR, but it has the opposite effect: pre-PR, actorOverlapCount was the entity-space overlap count. Any reprocessed old scorecard would report that value as roleOverlapCount, incorrectly classifying entity overlap as role-category overlap.

This conflicts with the PR's own monitoring guidance ("Old sim output → roleOverlapCount=0"). Since computeSimulationAdjustment now always initialises roleOverlapCount: 0, the fallback is only reached for genuinely old stored artifacts. Using ?? 0 is both more accurate and matches the documented expected behaviour:

Suggested change
roleOverlapCount: Number(d.roleOverlapCount ?? d.actorOverlapCount),
roleOverlapCount: Number(d.roleOverlapCount ?? 0),

keyActorsOverlapCount: Number(d.keyActorsOverlapCount ?? 0),
candidateActorCount: Number(d.candidateActorCount),
actorSource: d.actorSource,
resolvedChannel: d.resolvedChannel || '',
channelSource: d.channelSource,
invalidatorHit: Boolean(d.invalidatorHit),
stabilizerHit: Boolean(d.stabilizerHit),
};
}
}
Expand Down Expand Up @@ -11427,7 +11429,7 @@ function negatesDisruption(stabilizer, candidatePacket) {
*/
function computeSimulationAdjustment(expandedPath, simTheaterResult, candidatePacket) {
let adjustment = 0;
const details = { bucketChannelMatch: false, actorOverlapCount: 0, invalidatorHit: false, stabilizerHit: false, resolvedChannel: '', channelSource: 'none', candidateActorCount: 0, actorSource: 'none', simPathConfidence: 1.0 };
const details = { bucketChannelMatch: false, actorOverlapCount: 0, roleOverlapCount: 0, keyActorsOverlapCount: 0, invalidatorHit: false, stabilizerHit: false, resolvedChannel: '', channelSource: 'none', candidateActorCount: 0, actorSource: 'none', simPathConfidence: 1.0 };

const { topPaths = [], invalidators = [], stabilizers = [] } = simTheaterResult || {};
const pathBucket = expandedPath?.direct?.targetBucket
Expand Down Expand Up @@ -11484,13 +11486,27 @@ function computeSimulationAdjustment(expandedPath, simTheaterResult, candidatePa
adjustment += +parseFloat((0.08 * simConf).toFixed(3));
details.bucketChannelMatch = true;
details.simPathConfidence = simConf;
const simActors = new Set((Array.isArray(bucketChannelMatch.keyActors) ? bucketChannelMatch.keyActors : []).map(normalizeActorName));
const overlap = candidateActors.filter((a) => simActors.has(a));
details.actorOverlapCount = overlap.length;
// Overlap bonus fires only when both sides have named geo-political actors.
// Macro-financial theaters with role-based stateSummary.actors (e.g. "Commodity traders",
// "Central banks") will have actorOverlapCount=0 — this is expected, not a bug.
if (overlap.length >= 2) {
// Role overlap: candidate stateSummary.actors vs sim keyActorRoles (role-category vocabulary).
// Drives +0.04 bonus when actorSource=stateSummary. keyActorRoles absent → overlap=0 (graceful).
if (actorSrc === 'stateSummary') {
const simRoles = new Set((Array.isArray(bucketChannelMatch.keyActorRoles) ? bucketChannelMatch.keyActorRoles : []).map(normalizeActorName).filter(Boolean));
details.roleOverlapCount = candidateActors.filter((a) => simRoles.has(a)).length;
}

// Entity overlap: candidate actors vs sim keyActors.
// Drives +0.04 bonus when actorSource=affectedAssets (backwards compat). Telemetry when actorSource=stateSummary.
const simEntities = new Set((Array.isArray(bucketChannelMatch.keyActors) ? bucketChannelMatch.keyActors : []).map(normalizeActorName).filter(Boolean));
details.keyActorsOverlapCount = candidateActors.filter((a) => simEntities.has(a)).length;

// Bonus decision: role overlap for stateSummary path; entity overlap for affectedAssets fallback.
// Explicit third branch for actorSource='none' (no actors) so future values don't fall through silently.
const bonusOverlap = actorSrc === 'stateSummary'
? details.roleOverlapCount
: actorSrc === 'affectedAssets'
? details.keyActorsOverlapCount
: 0;
details.actorOverlapCount = bonusOverlap; // backwards-compat alias
if (bonusOverlap >= 2) {
adjustment += +parseFloat((0.04 * simConf).toFixed(3));
}
}
Expand Down Expand Up @@ -12814,6 +12830,11 @@ function buildSimulationPackageFromDeepSnapshot(snapshot, priorWorldState = null
topChannel: c.marketContext?.topChannel || '',
rankingScore: c.rankingScore,
criticalSignalTypes: c.criticalSignalTypes || [],
actorRoles: [...new Set(
(Array.isArray(c?.stateSummary?.actors) ? c.stateSummary.actors : [])
.map((s) => String(s || '').trim())
.filter(Boolean),
)].slice(0, 12),
}));

const simulationRequirement = Object.fromEntries(
Expand Down Expand Up @@ -14089,7 +14110,7 @@ function validateCaseNarratives(items, predictions) {
}

function sanitizeForPrompt(text) {
return (text || '').replace(/[\n\r]/g, ' ').replace(/[<>{}\x00-\x1f]/g, '').slice(0, 200).trim();
return (text || '').replace(/[\n\r\u2028\u2029]/g, ' ').replace(/[<>{}\x00-\x1f]/g, '').slice(0, 200).trim();
}

// Sanitizes LLM-returned text before writing to Redis as a prompt section.
Expand Down Expand Up @@ -16173,6 +16194,11 @@ function buildSimulationRound2SystemPrompt(theater, pkg, round1) {
const evalTargets = (r2EvalTargets?.requiredPaths || [])
.map((p) => `- ${sanitizeForPrompt(p.pathType)}: ${sanitizeForPrompt(p.question)}`).join('\n') || '- General market and security dynamics';

const actorRoles = Array.isArray(theater.actorRoles) ? theater.actorRoles : [];
const rolesSection = actorRoles.length > 0
? `\nCANDIDATE ACTOR ROLES (copy these EXACT strings into keyActorRoles; return [] if none apply):\n${actorRoles.map((r) => `- "${sanitizeForPrompt(r)}"`).join('\n')}`
: '';

return `You are a geopolitical simulation engine. This is ROUND 2 of a 2-round theater simulation.

THEATER: ${sanitizeForPrompt(theater.theaterLabel || theater.theaterId)} | Region: ${sanitizeForPrompt(theater.theaterRegion || theater.dominantRegion || '')}
Expand All @@ -16183,12 +16209,13 @@ ${pathSummaries}
VALID ACTOR IDs: ${entityIds || '(see round 1)'}

EVALUATION TARGETS:
${evalTargets}
${evalTargets}${rolesSection}

INSTRUCTIONS:
For each of the 3 paths from Round 1 (escalation, containment, market_cascade), generate the EVOLVED outcome after 72 hours.

- keyActors: 2-4 actor IDs that drive this path
- keyActors: 2-4 actor IDs that drive this path (entity names)
- keyActorRoles: 0-4 strings copied verbatim from CANDIDATE ACTOR ROLES above (return [] if list is absent or none apply)
- roundByRoundEvolution: 2 entries (round 1 summary, round 2 evolution)
- timingMarkers: 2-4 key events with timing (T+Nh format)
- stabilizers: 2-4 factors that could prevent the worst outcome
Expand All @@ -16203,15 +16230,16 @@ Return ONLY a JSON object with no markdown fences:
"label": "<short label>",
"summary": "<≤200 char evolved summary>",
"keyActors": ["<entityId>"],
"keyActorRoles": ["<exact string from CANDIDATE ACTOR ROLES, or empty array>"],
"roundByRoundEvolution": [
{ "round": 1, "summary": "<≤160 char>" },
{ "round": 2, "summary": "<≤160 char>" }
],
"confidence": 0.35,
"timingMarkers": [{ "event": "<≤80 char>", "timing": "T+Nh" }]
},
{ "pathId": "containment", "label": "...", "summary": "...", "keyActors": [], "roundByRoundEvolution": [], "confidence": 0.50, "timingMarkers": [] },
{ "pathId": "market_cascade", "label": "...", "summary": "...", "keyActors": [], "roundByRoundEvolution": [], "confidence": 0.15, "timingMarkers": [] }
{ "pathId": "containment", "label": "...", "summary": "...", "keyActors": [], "keyActorRoles": [], "roundByRoundEvolution": [], "confidence": 0.50, "timingMarkers": [] },
{ "pathId": "market_cascade", "label": "...", "summary": "...", "keyActors": [], "keyActorRoles": [], "roundByRoundEvolution": [], "confidence": 0.15, "timingMarkers": [] }
],
"stabilizers": ["<≤100 char>"],
"invalidators": ["<≤100 char>"],
Expand All @@ -16220,6 +16248,11 @@ Return ONLY a JSON object with no markdown fences:
}`;
}

/**
* @param {string} text - raw LLM response text (JSON or JSON-with-prefix)
* @param {1 | 2} round - simulation round number
* @returns {{ paths: object[] | null, stabilizers?: string[], invalidators?: string[], globalObservations?: string, confidenceNotes?: string, dominantReactions?: string[], note?: string }}
*/
function tryParseSimulationRoundPayload(text, round) {
try {
const parsed = JSON.parse(text);
Expand All @@ -16229,7 +16262,12 @@ function tryParseSimulationRoundPayload(text, round) {
if (paths.length === 0) return { paths: null };
if (round === 2) {
return {
paths,
paths: paths.map((p) => ({
...p,
keyActorRoles: Array.isArray(p.keyActorRoles)
? p.keyActorRoles.map((s) => sanitizeForPrompt(String(s || '')).trim()).filter(Boolean).slice(0, 10)
: [],
})),
stabilizers: Array.isArray(parsed.stabilizers) ? parsed.stabilizers.map(String).slice(0, 6) : [],
invalidators: Array.isArray(parsed.invalidators) ? parsed.invalidators.map(String).slice(0, 6) : [],
globalObservations: String(parsed.globalObservations || '').slice(0, 300),
Expand Down Expand Up @@ -16806,6 +16844,7 @@ async function writeSimulationOutcome(pkg, outcome, { storageConfig } = {}) {
summary: p.summary,
confidence: p.confidence,
keyActors: (p.keyActors || []).slice(0, 4),
keyActorRoles: (p.keyActorRoles || []).slice(0, 8),
})),
dominantReactions: (tr.dominantReactions || []).slice(0, 3),
stabilizers: (tr.stabilizers || []).slice(0, 3),
Expand Down Expand Up @@ -16880,6 +16919,23 @@ async function listQueuedSimulationTasks(limit = 10) {
return Array.isArray(response?.result) ? response.result : [];
}

/**
* Sanitize and allowlist-filter LLM-returned keyActorRoles strings.
* Filters against theater.actorRoles (exact match after normalizeActorName).
* When allowedRoles is empty (old package, no actorRoles field), returns [] — no vocab = no valid roles.
* This prevents hallucinated keyActorRoles from old packages triggering the +0.04 overlap bonus.
* @param {string[] | undefined} rawRoles
* @param {string[]} allowedRoles
* @returns {string[]}
*/
function sanitizeKeyActorRoles(rawRoles, allowedRoles) {
if (!allowedRoles.length) return [];
const sanitized = (Array.isArray(rawRoles) ? rawRoles : [])
.map((s) => sanitizeForPrompt(String(s)).slice(0, 80));
const allowedNorm = new Set(allowedRoles.map(normalizeActorName));
return sanitized.filter((s) => allowedNorm.has(normalizeActorName(s))).slice(0, 8);
}

async function processNextSimulationTask(options = {}) {
const workerId = options.workerId || `sim-worker-${process.pid}-${Date.now()}`;
const queuedRunIds = options.runId ? [options.runId] : await listQueuedSimulationTasks(10);
Expand Down Expand Up @@ -16943,13 +16999,15 @@ async function processNextSimulationTask(options = {}) {

const r2Paths = result.round2?.paths || [];
const r1Paths = result.round1?.paths || [];
const allowedRoles = Array.isArray(theater.actorRoles) ? theater.actorRoles : [];
const mergedPaths = (r2Paths.length ? r2Paths : r1Paths).map((p) => {
const r1Path = r1Paths.find((r) => r.pathId === p.pathId);
return {
pathId: p.pathId,
label: sanitizeForPrompt(p.label || p.pathId).slice(0, 80),
summary: sanitizeForPrompt(p.summary || '').slice(0, 200),
keyActors: Array.isArray(p.keyActors) ? p.keyActors.map((s) => sanitizeForPrompt(String(s)).slice(0, 80)).slice(0, 6) : [],
keyActorRoles: sanitizeKeyActorRoles(p.keyActorRoles, allowedRoles),
roundByRoundEvolution: Array.isArray(p.roundByRoundEvolution)
? p.roundByRoundEvolution.map((r) => ({ round: r.round, summary: sanitizeForPrompt(r.summary || '').slice(0, 160) }))
: [{ round: 1, summary: sanitizeForPrompt((r1Path?.summary || p.summary || '')).slice(0, 160) }],
Expand Down Expand Up @@ -17181,6 +17239,7 @@ export {
writeSimulationOutcome,
buildSimulationRound1SystemPrompt,
buildSimulationRound2SystemPrompt,
tryParseSimulationRoundPayload,
extractSimulationRoundPayload,
runTheaterSimulation,
enqueueSimulationTask,
Expand Down
63 changes: 54 additions & 9 deletions scripts/seed-forecasts.types.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,39 @@ interface ExpandedPath {
candidate?: ExpandedPathCandidate;
}

// ---------------------------------------------------------------------------
// Simulation package (buildSimulationPackageFromDeepSnapshot output)
// ---------------------------------------------------------------------------

/**
* One theater entry in SimulationPackage.selectedTheaters.
* Distinct from TheaterResult (LLM output shape stored in SimulationOutcome).
*
* NOTE: When adding fields here, also add them to the uiTheaters projection
* in writeSimulationOutcome() or they will be invisible in the Redis snapshot.
*/
interface SimulationPackageTheater {
theaterId: string;
candidateStateId: string;
theaterLabel?: string;
theaterRegion?: string;
stateKind?: string;
dominantRegion?: string;
macroRegions?: string[];
routeFacilityKey?: string;
commodityKey?: string;
topBucketId: string;
topChannel: string;
rankingScore?: number;
criticalSignalTypes: string[];
/**
* Role-category strings from candidate stateSummary.actors. Theater-scoped (no cross-theater aggregation).
* Injected into Round 2 prompt as CANDIDATE ACTOR ROLES. Used as allowlist in sanitizeKeyActorRoles guardrail.
* keyActors (entity-space) and actorRoles (role-category) are intentionally disjoint vocabularies.
*/
actorRoles: string[];
}

// ---------------------------------------------------------------------------
// Theater simulation structures
// ---------------------------------------------------------------------------
Expand All @@ -133,7 +166,10 @@ interface SimulationTopPath {
label: string;
summary: string;
confidence: number;
/** Entity-space actor names (geo-political). Used for narrative/audit. NOT used for overlap bonus scoring. */
keyActors: string[];
/** Role-category actor strings from the candidate's stateSummary.actors vocabulary. Used for the +0.04 overlap bonus when actorSource=stateSummary. */
keyActorRoles?: string[];
roundByRoundEvolution?: Array<{ round: number; summary: string }>;
timingMarkers?: Array<{ event: string; timing: string }>;
}
Expand Down Expand Up @@ -179,8 +215,12 @@ interface SimulationOutcome {

interface SimulationAdjustmentDetail {
bucketChannelMatch: boolean;
/** Number of overlapping actors between path and simulation top paths (>=2 triggers +0.04 bonus). */
/** Backwards-compat alias: equals roleOverlapCount when actorSource=stateSummary, else keyActorsOverlapCount. >=2 triggered the +0.04 bonus. */
actorOverlapCount: number;
/** Role-category overlap count (candidate stateSummary.actors vs sim keyActorRoles). Drives +0.04 bonus when actorSource=stateSummary. */
roleOverlapCount: number;
/** Entity-space overlap count (candidate actors vs sim keyActors). Drives +0.04 bonus when actorSource=affectedAssets. Telemetry only when actorSource=stateSummary. */
keyActorsOverlapCount: number;
invalidatorHit: boolean;
stabilizerHit: boolean;
/** Number of candidate-theater actors used for overlap matching. Source is stateSummary.actors if raw list present, else affectedAssets. Never a union. */
Expand Down Expand Up @@ -208,14 +248,19 @@ interface SimulationAdjustmentRecord {

/** Flat projection of SimulationAdjustmentDetail written into path-scorecards.json entries. simPathConfidence is omitted (already in simulationSignal). */
interface ScorecardSimDetail {
bucketChannelMatch: boolean;
actorOverlapCount: number;
candidateActorCount: number;
actorSource: 'stateSummary' | 'affectedAssets' | 'none';
resolvedChannel: string;
channelSource: 'direct' | 'market' | 'none';
invalidatorHit: boolean;
stabilizerHit: boolean;
bucketChannelMatch: boolean;
/** Backwards-compat alias for roleOverlapCount or keyActorsOverlapCount (whichever drove the bonus). */
actorOverlapCount: number;
/** Role-category overlap (stateSummary path). Drives +0.04 when actorSource=stateSummary. */
roleOverlapCount: number;
/** Entity-space overlap via keyActors (affectedAssets path). Drives +0.04 when actorSource=affectedAssets. */
keyActorsOverlapCount: number;
candidateActorCount: number;
actorSource: 'stateSummary' | 'affectedAssets' | 'none';
resolvedChannel: string;
channelSource: 'direct' | 'market' | 'none';
invalidatorHit: boolean;
stabilizerHit: boolean;
}

interface SimulationEvidence {
Expand Down
Loading
Loading