diff --git a/src/composer/validation.ts b/src/composer/validation.ts new file mode 100644 index 0000000..6de15b0 --- /dev/null +++ b/src/composer/validation.ts @@ -0,0 +1,513 @@ +import type { JSONSchema7 } from "json-schema"; +import type { + ConditionalStep, + SamplingStep, + StepConfig, + TransformStep, +} from "../workflow/types.js"; +import { + ComposerError, + type ComposerWarning, + type ComposeStepInput, +} from "./types.js"; +import { + BARE_PARAM_REF_RE, + JS_BUILTIN_METHODS, + PARAM_REF_RE, + STEP_FIELD_ACCESS_RE, + STEP_REF_RE, + extractTemplateStrings, +} from "./utils.js"; +import { + autoReturnExpression, + validateSafeExpression, +} from "../workflow/expression-security.js"; +import { detectJsonIntent } from "../workflow/json-intent.js"; + +export function validateUniqueIds(steps: ComposeStepInput[]): void { + const ids = new Set(); + for (const step of steps) { + if (ids.has(step.id)) { + throw new ComposerError(`Duplicate step ID: "${step.id}"`); + } + ids.add(step.id); + } +} + +export function validateReferences(steps: ComposeStepInput[]): void { + const ids = new Set(steps.map((s) => s.id)); + + for (const step of steps) { + if (step.dependsOn) { + for (const dep of step.dependsOn) { + if (!ids.has(dep)) { + throw new ComposerError( + `Step "${step.id}" depends on unknown step "${dep}"`, + ); + } + if (dep === step.id) { + throw new ComposerError(`Step "${step.id}" cannot depend on itself`); + } + } + } + + if (step.config.type === "conditional") { + const cfg = step.config as ConditionalStep; + if (!ids.has(cfg.thenStep)) { + throw new ComposerError( + `Step "${step.id}" references unknown thenStep "${cfg.thenStep}"`, + ); + } + if (cfg.elseStep && !ids.has(cfg.elseStep)) { + throw new ComposerError( + `Step "${step.id}" references unknown elseStep "${cfg.elseStep}"`, + ); + } + } + } +} + +export function detectCycles(steps: ComposeStepInput[]): void { + const adj = new Map(); + for (const step of steps) { + adj.set(step.id, step.dependsOn ?? []); + } + + const visited = new Set(); + const inStack = new Set(); + + function dfs(node: string, path: string[]): void { + if (inStack.has(node)) { + const cycleStart = path.indexOf(node); + const cycle = path.slice(cycleStart).concat(node); + throw new ComposerError( + `Circular dependency detected: ${cycle.join(" → ")}`, + ); + } + if (visited.has(node)) return; + + visited.add(node); + inStack.add(node); + + for (const dep of adj.get(node) ?? []) { + dfs(dep, [...path, node]); + } + + inStack.delete(node); + } + + for (const step of steps) { + if (!visited.has(step.id)) { + dfs(step.id, []); + } + } +} + +export function validateStepConfig(step: ComposeStepInput): void { + const cfg = step.config; + switch (cfg.type) { + case "api_call": + if (!cfg.operationId) { + throw new ComposerError( + `Step "${step.id}" (api_call): operationId is required`, + ); + } + break; + case "sampling": + if (!cfg.prompt) { + throw new ComposerError( + `Step "${step.id}" (sampling): prompt is required`, + ); + } + if (cfg.content) { + for (const item of cfg.content) { + if (item.type === "text" && !item.text) { + throw new ComposerError( + `Step "${step.id}" (sampling): content text item requires a text field`, + ); + } + if (item.type === "image" && !item.url) { + throw new ComposerError( + `Step "${step.id}" (sampling): content image item requires a url field`, + ); + } + } + } + break; + case "elicitation": + if (!cfg.message) { + throw new ComposerError( + `Step "${step.id}" (elicitation): message is required`, + ); + } + if (!cfg.requestedSchema) { + throw new ComposerError( + `Step "${step.id}" (elicitation): requestedSchema is required`, + ); + } + break; + case "transform": + if (!cfg.expression) { + throw new ComposerError( + `Step "${step.id}" (transform): expression is required`, + ); + } + break; + case "conditional": + if (!cfg.condition) { + throw new ComposerError( + `Step "${step.id}" (conditional): condition is required`, + ); + } + if (!cfg.thenStep) { + throw new ComposerError( + `Step "${step.id}" (conditional): thenStep is required`, + ); + } + break; + default: + throw new ComposerError( + `Step "${step.id}": unknown step type "${(cfg as { type: string }).type}"`, + ); + } +} + +function findDomainKeyHint(field: string, params: JSONSchema7): string | null { + const props = params.properties as + | Record + | undefined; + if (!props) return null; + for (const [key, schema] of Object.entries(props)) { + if (typeof schema === "boolean") continue; + const subProps = schema.properties as Record | undefined; + if (subProps && (field in subProps || `${field}?` in subProps)) { + return key; + } + } + return null; +} + +export function validateTemplateReferences( + steps: ComposeStepInput[], + params: JSONSchema7, +): ComposerWarning[] { + const warnings: ComposerWarning[] = []; + const stepIds = new Set(steps.map((s) => s.id)); + for (const step of steps) { + if (step.config.type === "api_call" && step.config.as) { + stepIds.add(step.config.as); + } + } + const paramProps = new Set( + params.properties ? Object.keys(params.properties) : [], + ); + + for (const step of steps) { + const templates = extractTemplateStrings(step.config); + for (const tmpl of templates) { + for (const match of tmpl.matchAll(STEP_REF_RE)) { + const refId = match[1]; + if (!stepIds.has(refId)) { + throw new ComposerError( + `Step "${step.id}" references unknown step "${refId}" in template: "${tmpl.slice(0, 80)}"`, + ); + } + } + if (paramProps.size > 0) { + for (const match of tmpl.matchAll(PARAM_REF_RE)) { + const fullPath = match[1]; + const segments = fullPath.split("."); + const topField = segments[0]; + if (!paramProps.has(topField)) { + const hint = findDomainKeyHint(topField, params); + throw new ComposerError( + `Step "${step.id}" references "params.${topField}" but "${topField}" is not in the workflow params schema. Available: ${[...paramProps].join(", ") || "(none)"}` + + (hint + ? `. Did you mean "params.${hint}.${topField}"? Event params are nested under the domain key.` + : ""), + ); + } + if (segments.length > 1) { + const subWarning = validateParamSubField( + segments, + params, + step.id, + fullPath, + ); + if (subWarning) warnings.push(subWarning); + } + } + const isJsContext = + step.config.type === "transform" || + step.config.type === "conditional"; + if (isJsContext) { + for (const match of tmpl.matchAll(BARE_PARAM_REF_RE)) { + const fullPath = match[1]; + const segments = fullPath.split("."); + const topField = segments[0]; + if (!paramProps.has(topField)) { + const hint = findDomainKeyHint(topField, params); + throw new ComposerError( + `Step "${step.id}" references "params.${topField}" but "${topField}" is not in the workflow params schema. Available: ${[...paramProps].join(", ") || "(none)"}` + + (hint + ? `. Did you mean "params.${hint}.${topField}"? Event params are nested under the domain key.` + : ""), + ); + } + if (segments.length > 1) { + const subWarning = validateParamSubField( + segments, + params, + step.id, + fullPath, + ); + if (subWarning) warnings.push(subWarning); + } + } + } + } + } + } + return warnings; +} + +function validateParamSubField( + segments: string[], + schema: JSONSchema7, + stepId: string, + fullPath: string, +): ComposerWarning | null { + let current: JSONSchema7 | boolean | undefined = schema; + for (let i = 0; i < segments.length; i++) { + const seg = segments[i]; + if (!current || typeof current === "boolean") return null; + const props: Record | undefined = ( + current as JSONSchema7 + ).properties as Record | undefined; + if (!props) return null; // No properties defined — can't validate further + if (!(seg in props)) { + // Segment not found in this object's properties + const available = Object.keys(props); + const parentPath = + i === 0 ? "params" : `params.${segments.slice(0, i).join(".")}`; + throw new ComposerError( + `Step "${stepId}" references "params.${fullPath}" but "${seg}" is not a known property of ${parentPath}. Available: ${available.join(", ")}`, + ); + } + const next: JSONSchema7 | boolean = props[seg]; + if (typeof next === "boolean") return null; + // If this is a leaf type, everything after is JS — allow + if (i < segments.length - 1 && next.type !== "object" && !next.properties) { + return null; + } + current = next; + } + return null; +} + +type StepOutputType = "string" | "boolean" | "object" | "unknown"; + +const STEP_OUTPUT_TYPES: Record = { + sampling: "unknown", // resolved per-step from JSON mode (see resolveStepOutputType) + conditional: "boolean", + api_call: "object", + elicitation: "object", + transform: "unknown", +}; + +/** + * True when a sampling step is meant to yield structured JSON — either because it + * explicitly declares `responseFormat: "json"` or because its prompt/systemPrompt + * signal JSON intent. This mirrors the runtime decision (the sampling executor + * JSON-parses the model output under the same condition), so compose-time + * validation and execution agree on whether field access is meaningful. + */ +function isJsonModeSampling(cfg: SamplingStep): boolean { + return cfg.responseFormat === "json" || detectJsonIntent(cfg); +} + +/** + * Resolve a step's effective output type. Sampling is special: its result is a + * plain text string unless the step is in JSON mode, in which case the runtime + * parses it into an object. Field access is only valid against the object form, + * so a plain-text sampling result reports "string" and is rejected by the + * data-flow guard below. + */ +function resolveStepOutputType(config: StepConfig): StepOutputType { + if (config.type === "sampling") { + return isJsonModeSampling(config) ? "object" : "string"; + } + return STEP_OUTPUT_TYPES[config.type]; +} + +export function validateDataFlowTypes(steps: ComposeStepInput[]): void { + const stepById = new Map(steps.map((s) => [s.id, s])); + + for (const step of steps) { + const templates = extractTemplateStrings(step.config); + for (const tmpl of templates) { + for (const match of tmpl.matchAll(STEP_FIELD_ACCESS_RE)) { + const refStepId = match[1]; + const field = match[2]; + if (JS_BUILTIN_METHODS.has(field)) continue; + const refStep = stepById.get(refStepId); + if (!refStep) continue; + + const outputType = resolveStepOutputType(refStep.config); + if (outputType === "string") { + throw new ComposerError( + `Step "${step.id}" accesses ".${field}" on step "${refStepId}" (${refStep.config.type}), ` + + `but ${refStep.config.type} results are plain text strings with no properties. ` + + `Fix: (1) Add a systemPrompt to step "${refStepId}" asking the LLM to respond in JSON only, ` + + `(2) Add a transform step after "${refStepId}" with expression 'JSON.parse(steps.${refStepId})', ` + + `then (3) reference 'steps..${field}' instead. ` + + `Alternatively, evaluate the raw string directly (e.g. 'steps.${refStepId}.includes("...")').`, + ); + } + if (outputType === "boolean") { + throw new ComposerError( + `Step "${step.id}" accesses ".${field}" on step "${refStepId}" (conditional), ` + + `but conditional results are booleans with no properties. ` + + `Use the boolean value directly: 'steps.${refStepId} === true'.`, + ); + } + } + } + } +} + +export function validateSafeWorkflowExpressions( + steps: ComposeStepInput[], +): void { + for (const step of steps) { + if (step.config.type === "transform") { + validateSafeExpression( + autoReturnExpression((step.config as TransformStep).expression), + `transform step "${step.id}"`, + ); + } else if (step.config.type === "conditional") { + validateSafeExpression( + (step.config as ConditionalStep).condition, + `conditional step "${step.id}"`, + ); + } + } +} + +/** + * Auto-infer a `responseSchema` for JSON-mode sampling steps from the way + * downstream steps access their result. + * + * When a later step reads `steps..` (dot or bracket notation), + * the field name — and a coarse type inferred from how it is used — is recorded + * on the sampling step's `responseSchema`. This makes the structured shape the + * workflow depends on explicit instead of implicit, and lets the runtime/provider + * steer the model toward returning those fields. A `SAMPLING_SCHEMA_MISMATCH` + * warning is emitted when a consumed field is never mentioned in the prompt, since + * the model is then unlikely to produce it. + * + * Only JSON-mode sampling steps are considered (see {@link isJsonModeSampling}); + * plain-text sampling steps that are field-accessed are rejected earlier by + * {@link validateDataFlowTypes}. + */ +export function inferSamplingResponseSchemas( + steps: ComposeStepInput[], +): ComposerWarning[] { + const warnings: ComposerWarning[] = []; + + const jsonSamplingIds = new Set(); + for (const step of steps) { + if (step.config.type === "sampling" && isJsonModeSampling(step.config)) { + jsonSamplingIds.add(step.id); + } + } + if (jsonSamplingIds.size === 0) return warnings; + + // For each JSON sampling step: field name -> inferred JSON type. + const fieldAccesses = new Map>(); + for (const id of jsonSamplingIds) fieldAccesses.set(id, new Map()); + + // Dot access: steps.X.field or steps.X?.field, optionally followed by a + // usage hint (=== bool, .join/.map/.filter/.length) we use to guess the type. + const FIELD_CONTEXT_RE = + /steps\.(\w+)\??\.(\w+)\s*(?:===\s*(true|false)|\.join\b|\.map\b|\.filter\b|\.length\b|\.includes\b)?/g; + // Bracket access: steps.X["field"] or steps.X?.["field"] + const BRACKET_ACCESS_RE = /steps\.(\w+)\??\[\s*(['"])(\w+)\2\s*\]/g; + + function classifyField( + refId: string, + field: string, + boolLiteral: string | undefined, + tmpl: string, + matchStart: number, + matchEnd: number, + ): void { + if (!fieldAccesses.has(refId)) return; + if (JS_BUILTIN_METHODS.has(field)) return; + const fields = fieldAccesses.get(refId)!; + if (fields.has(field)) return; + + let inferredType = "string"; + if (boolLiteral === "true" || boolLiteral === "false") { + inferredType = "boolean"; + } else if ( + /\.join\b|\.map\b|\.filter\b|\.length\b/.test( + tmpl.slice(matchStart, matchEnd + 10), + ) + ) { + inferredType = "array"; + } + fields.set(field, inferredType); + } + + for (const step of steps) { + const templates = extractTemplateStrings(step.config); + for (const tmpl of templates) { + for (const match of tmpl.matchAll(FIELD_CONTEXT_RE)) { + classifyField( + match[1], + match[2], + match[3], + tmpl, + match.index!, + match.index! + match[0].length, + ); + } + for (const match of tmpl.matchAll(BRACKET_ACCESS_RE)) { + classifyField( + match[1], + match[3], + undefined, + tmpl, + match.index!, + match.index! + match[0].length, + ); + } + } + } + + for (const [stepId, fields] of fieldAccesses) { + if (fields.size === 0) continue; + const step = steps.find((s) => s.id === stepId)!; + if (step.config.type !== "sampling") continue; + const cfg = step.config; + + const schema: Record = {}; + for (const [name, type] of fields) schema[name] = type; + cfg.responseSchema = schema; + + const promptText = + `${cfg.prompt || ""} ${cfg.systemPrompt || ""}`.toLowerCase(); + for (const field of fields.keys()) { + if (!promptText.includes(field.toLowerCase())) { + warnings.push({ + stepId, + code: "SAMPLING_SCHEMA_MISMATCH", + message: `Step "${stepId}" result field "${field}" is used by downstream steps but not mentioned in the sampling prompt — the AI may not include it.`, + }); + } + } + } + + return warnings; +} diff --git a/src/composer/warnings.ts b/src/composer/warnings.ts new file mode 100644 index 0000000..ad2d184 --- /dev/null +++ b/src/composer/warnings.ts @@ -0,0 +1,199 @@ +import type { JSONSchema7 } from "json-schema"; +import type { + ApiCallStep, + ConditionalStep, + SamplingStep, +} from "../workflow/types.js"; +import { + ComposerError, + type ComposerWarning, + type ComposeStepInput, +} from "./types.js"; +import { extractStepRefs, findLiteralRid } from "./utils.js"; + +export function generateSemanticWarnings( + steps: ComposeStepInput[], + params?: JSONSchema7, +): ComposerWarning[] { + const warnings: ComposerWarning[] = []; + const hasParams = + params && + Object.keys((params.properties as Record) ?? {}).length > + 0; + + const referencedSteps = new Set(); + for (const step of steps) { + for (const dep of step.dependsOn ?? []) referencedSteps.add(dep); + const refs = extractStepRefs(step.config); + for (const r of refs) referencedSteps.add(r); + if (step.config.type === "conditional") { + const cfg = step.config as ConditionalStep; + referencedSteps.add(cfg.thenStep); + if (cfg.elseStep) referencedSteps.add(cfg.elseStep); + } + } + + for (const step of steps) { + if (step.config.type === "sampling") { + let referenced = false; + for (const other of steps) { + if (other.id === step.id) continue; + const refs = extractStepRefs(other.config); + if (refs.has(step.id)) { + referenced = true; + break; + } + } + if (!referenced) { + warnings.push({ + stepId: step.id, + code: "UNUSED_SAMPLING", + message: `Sampling step "${step.id}" result is never referenced by any other step. This wastes an LLM call.`, + }); + } + + const cfg = step.config as SamplingStep; + const allText = [ + cfg.prompt ?? "", + cfg.systemPrompt ?? "", + ...(cfg.content ?? []).map((c) => + c.type === "text" ? (c.text ?? "") : (c.url ?? ""), + ), + ].join(" "); + const hasTemplateRef = /\{\{(params|steps)\./.test(allText); + if (!hasTemplateRef && hasParams) { + throw new ComposerError( + `Sampling step "${step.id}" prompt does not reference any {{params.*}} or {{steps.*}} data. ` + + `The AI will have no input to analyze. Include the relevant data in the prompt ` + + `(e.g. {{params.message.text}}).`, + ); + } + } + } + + const forward = new Map>(); + for (const step of steps) { + if (!forward.has(step.id)) forward.set(step.id, new Set()); + for (const dep of step.dependsOn ?? []) { + if (!forward.has(dep)) forward.set(dep, new Set()); + forward.get(dep)!.add(step.id); + } + if (step.config.type === "conditional") { + const cfg = step.config as ConditionalStep; + forward.get(step.id)!.add(cfg.thenStep); + if (cfg.elseStep) forward.get(step.id)!.add(cfg.elseStep); + } + } + + const reachable = new Set(); + const queue: string[] = []; + for (const step of steps) { + if (!step.dependsOn || step.dependsOn.length === 0) { + reachable.add(step.id); + queue.push(step.id); + } + } + while (queue.length > 0) { + const id = queue.shift()!; + for (const next of forward.get(id) ?? []) { + if (!reachable.has(next)) { + reachable.add(next); + queue.push(next); + } + } + } + + for (const step of steps) { + if (!reachable.has(step.id)) { + warnings.push({ + stepId: step.id, + code: "ORPHANED_STEP", + message: `Step "${step.id}" is not reachable from any entry point. It will never execute.`, + }); + } + } + + const rootSteps = steps.filter( + (s) => !s.dependsOn || s.dependsOn.length === 0, + ); + if (rootSteps.length > 1) { + const rootIds = rootSteps.map((s) => s.id); + warnings.push({ + stepId: null, + code: "MULTIPLE_ROOTS", + message: + `Workflow has ${rootSteps.length} root steps (no dependsOn): [${rootIds.join(", ")}]. ` + + `Most workflows should have exactly 1 entry point. ` + + `If these steps should run after other steps, add them to dependsOn.`, + }); + } + + const opCounts = new Map(); + for (const step of steps) { + if (step.config.type === "api_call") { + const opId = (step.config as ApiCallStep).operationId; + const list = opCounts.get(opId) ?? []; + list.push(step.id); + opCounts.set(opId, list); + } + } + for (const [opId, stepIds] of opCounts) { + if (stepIds.length > 1) { + warnings.push({ + stepId: null, + code: "DUPLICATE_API_CALL", + message: `Multiple steps (${stepIds.join(", ")}) call the same endpoint "${opId}". Is this intentional?`, + }); + } + } + + for (const step of steps) { + if (step.config.type !== "api_call") continue; + const cfg = step.config as ApiCallStep; + if ( + !cfg.operationId.includes("chat_sendMessage") && + !cfg.operationId.includes("chat.sendMessage") + ) + continue; + const rid = findLiteralRid(cfg.inputMapping); + if (rid) { + warnings.push({ + stepId: step.id, + code: "HARDCODED_RID", + message: + `Step "${step.id}" uses chat.sendMessage with hardcoded rid "${rid}". ` + + `The rid field requires a room ID (e.g. "6oaKzj..."), not a channel name. ` + + `Use "post-api-v1-chat-postMessage" with { channel: "#${rid}", text: "..." } instead, ` + + `which resolves channel names natively.`, + }); + } + } + + const depthCache = new Map(); + const stepMap = new Map(steps.map((s) => [s.id, s])); + function getDepth(id: string): number { + if (depthCache.has(id)) return depthCache.get(id)!; + const step = stepMap.get(id); + if (!step || !step.dependsOn || step.dependsOn.length === 0) { + depthCache.set(id, 0); + return 0; + } + const maxParent = Math.max(...step.dependsOn.map(getDepth)); + const depth = maxParent + 1; + depthCache.set(id, depth); + return depth; + } + for (const step of steps) { + const depth = getDepth(step.id); + if (depth > 8) { + warnings.push({ + stepId: step.id, + code: "DEEP_CHAIN", + message: `Step "${step.id}" is ${depth} levels deep in the dependency chain. Consider simplifying.`, + }); + break; + } + } + + return warnings; +} diff --git a/src/tests/composer/data-flow-validation.unit.test.ts b/src/tests/composer/data-flow-validation.unit.test.ts new file mode 100644 index 0000000..b969f71 --- /dev/null +++ b/src/tests/composer/data-flow-validation.unit.test.ts @@ -0,0 +1,176 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { validateDataFlowTypes } from "../../composer/validation.js"; +import type { ComposeStepInput } from "../../composer/types.js"; + +/** + * These exercise the guard that closes the "plain-text sampling used like JSON" + * gap: field access is only valid against an object-shaped result. Sampling is a + * plain-text string unless it is in JSON mode (explicit responseFormat: "json" + * or JSON intent in the prompt), in which case the runtime parses it to an object. + */ +describe("validateDataFlowTypes", () => { + it("rejects field access on a plain-text sampling result", () => { + const steps: ComposeStepInput[] = [ + { + id: "analyze", + label: "Analyze", + // No responseFormat and no JSON intent -> plain text string. + config: { type: "sampling", prompt: "Analyze: {{params.msg}}" }, + }, + { + id: "route", + label: "Route", + config: { + type: "conditional", + condition: "steps.analyze.toxicityScore > 0.7", + thenStep: "act", + }, + dependsOn: ["analyze"], + }, + ]; + assert.throws( + () => validateDataFlowTypes(steps), + /plain text strings with no properties/, + ); + }); + + it("allows field access when the sampling step declares responseFormat: json", () => { + const steps: ComposeStepInput[] = [ + { + id: "analyze", + label: "Analyze", + config: { + type: "sampling", + prompt: "Analyze {{params.msg}} and return a JSON object", + responseFormat: "json", + }, + }, + { + id: "route", + label: "Route", + config: { + type: "conditional", + condition: "steps.analyze.toxicityScore > 0.7", + thenStep: "act", + }, + dependsOn: ["analyze"], + }, + ]; + assert.doesNotThrow(() => validateDataFlowTypes(steps)); + }); + + it("allows field access when the prompt signals JSON intent (no explicit responseFormat)", () => { + const steps: ComposeStepInput[] = [ + { + id: "analyze", + label: "Analyze", + config: { + type: "sampling", + prompt: "Analyze {{params.msg}} and respond with a JSON object", + }, + }, + { + id: "route", + label: "Route", + config: { + type: "conditional", + condition: "steps.analyze.toxicityScore > 0.7", + thenStep: "act", + }, + dependsOn: ["analyze"], + }, + ]; + assert.doesNotThrow(() => validateDataFlowTypes(steps)); + }); + + it("allows string method calls on a plain-text sampling result", () => { + const steps: ComposeStepInput[] = [ + { + id: "analyze", + label: "Analyze", + config: { type: "sampling", prompt: "Check: {{params.msg}}" }, + }, + { + id: "route", + label: "Route", + config: { + type: "conditional", + condition: 'steps.analyze.includes("toxic")', + thenStep: "act", + }, + dependsOn: ["analyze"], + }, + ]; + assert.doesNotThrow(() => validateDataFlowTypes(steps)); + }); + + it("rejects field access on a conditional (boolean) result", () => { + const steps: ComposeStepInput[] = [ + { + id: "check", + label: "Check", + config: { type: "conditional", condition: "true", thenStep: "next" }, + }, + { + id: "next", + label: "Next", + config: { type: "transform", expression: "steps.check.someField" }, + dependsOn: ["check"], + }, + ]; + assert.throws( + () => validateDataFlowTypes(steps), + /booleans with no properties/, + ); + }); + + it("allows field access on api_call results", () => { + const steps: ComposeStepInput[] = [ + { + id: "fetch", + label: "Fetch", + config: { + type: "api_call", + operationId: "get-api-v1-channels-list", + inputMapping: {}, + }, + }, + { + id: "extract", + label: "Extract", + config: { type: "transform", expression: "steps.fetch.channels" }, + dependsOn: ["fetch"], + }, + ]; + assert.doesNotThrow(() => validateDataFlowTypes(steps)); + }); + + it("allows field access on elicitation results", () => { + const steps: ComposeStepInput[] = [ + { + id: "ask", + label: "Ask", + config: { + type: "elicitation", + message: "Confirm?", + requestedSchema: { + type: "object", + properties: { confirm: { type: "boolean" } }, + }, + }, + }, + { + id: "check", + label: "Check", + config: { + type: "conditional", + condition: "steps.ask.confirm === true", + thenStep: "done", + }, + dependsOn: ["ask"], + }, + ]; + assert.doesNotThrow(() => validateDataFlowTypes(steps)); + }); +}); diff --git a/src/tests/composer/sampling-response-schema.unit.test.ts b/src/tests/composer/sampling-response-schema.unit.test.ts new file mode 100644 index 0000000..45e66cb --- /dev/null +++ b/src/tests/composer/sampling-response-schema.unit.test.ts @@ -0,0 +1,162 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { inferSamplingResponseSchemas } from "../../composer/validation.js"; +import type { ComposeStepInput } from "../../composer/types.js"; +import type { SamplingStep } from "../../workflow/types.js"; + +describe("inferSamplingResponseSchemas", () => { + it("infers a responseSchema from downstream dot access, guessing types from usage", () => { + const steps: ComposeStepInput[] = [ + { + id: "rank", + label: "Rank", + config: { + type: "sampling", + prompt: + "Return a JSON object with fields: safe, score and items. {{params.msg}}", + responseFormat: "json", + }, + }, + { + id: "gate", + label: "Gate", + config: { + type: "conditional", + condition: "steps.rank.safe === true", + thenStep: "join", + }, + dependsOn: ["rank"], + }, + { + id: "join", + label: "Join", + config: { + type: "transform", + expression: "steps.rank.items.join(', ') + steps.rank.score", + }, + dependsOn: ["gate"], + }, + ]; + + const warnings = inferSamplingResponseSchemas(steps); + const cfg = steps[0].config as SamplingStep; + + assert.deepEqual(cfg.responseSchema, { + safe: "boolean", + items: "array", + score: "string", + }); + assert.equal(warnings.length, 0); + }); + + it("infers JSON mode from prompt intent even without responseFormat", () => { + const steps: ComposeStepInput[] = [ + { + id: "rank", + label: "Rank", + config: { + type: "sampling", + prompt: "Respond with a JSON object containing a score field.", + }, + }, + { + id: "use", + label: "Use", + config: { type: "transform", expression: "steps.rank.score" }, + dependsOn: ["rank"], + }, + ]; + + inferSamplingResponseSchemas(steps); + const cfg = steps[0].config as SamplingStep; + assert.deepEqual(cfg.responseSchema, { score: "string" }); + }); + + it("recognizes bracket-notation field access", () => { + const steps: ComposeStepInput[] = [ + { + id: "rank", + label: "Rank", + config: { + type: "sampling", + prompt: "Return JSON with a safe field.", + responseFormat: "json", + }, + }, + { + id: "use", + label: "Use", + config: { type: "transform", expression: 'steps.rank["safe"]' }, + dependsOn: ["rank"], + }, + ]; + + inferSamplingResponseSchemas(steps); + const cfg = steps[0].config as SamplingStep; + assert.deepEqual(cfg.responseSchema, { safe: "string" }); + }); + + it("warns when a consumed field is not mentioned in the prompt", () => { + const steps: ComposeStepInput[] = [ + { + id: "rank", + label: "Rank", + config: { + type: "sampling", + prompt: "Return a JSON object.", + responseFormat: "json", + }, + }, + { + id: "use", + label: "Use", + config: { type: "transform", expression: "steps.rank.score" }, + dependsOn: ["rank"], + }, + ]; + + const warnings = inferSamplingResponseSchemas(steps); + assert.equal(warnings.length, 1); + assert.equal(warnings[0].code, "SAMPLING_SCHEMA_MISMATCH"); + assert.equal(warnings[0].stepId, "rank"); + }); + + it("leaves plain-text sampling steps untouched", () => { + const steps: ComposeStepInput[] = [ + { + id: "rank", + label: "Rank", + config: { type: "sampling", prompt: "Summarize: {{params.msg}}" }, + }, + ]; + + const warnings = inferSamplingResponseSchemas(steps); + const cfg = steps[0].config as SamplingStep; + assert.equal(cfg.responseSchema, undefined); + assert.equal(warnings.length, 0); + }); + + it("ignores JS builtin methods when inferring fields", () => { + const steps: ComposeStepInput[] = [ + { + id: "rank", + label: "Rank", + config: { + type: "sampling", + prompt: "Return JSON.", + responseFormat: "json", + }, + }, + { + id: "use", + label: "Use", + config: { type: "transform", expression: "steps.rank.length" }, + dependsOn: ["rank"], + }, + ]; + + inferSamplingResponseSchemas(steps); + const cfg = steps[0].config as SamplingStep; + assert.equal(cfg.responseSchema, undefined); + }); +}); diff --git a/src/tests/workflow/json-intent.unit.test.ts b/src/tests/workflow/json-intent.unit.test.ts new file mode 100644 index 0000000..d681c6c --- /dev/null +++ b/src/tests/workflow/json-intent.unit.test.ts @@ -0,0 +1,27 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { detectJsonIntent } from "../../workflow/json-intent.js"; + +describe("detectJsonIntent", () => { + it("detects JSON intent from the prompt", () => { + assert.equal( + detectJsonIntent({ prompt: "Respond with a JSON object" }), + true, + ); + }); + + it("detects JSON intent from the systemPrompt", () => { + assert.equal( + detectJsonIntent({ systemPrompt: "Output format: json only" }), + true, + ); + }); + + it("is false for plain-text prompts", () => { + assert.equal(detectJsonIntent({ prompt: "Summarize this message" }), false); + }); + + it("is false when no prompt fields are provided", () => { + assert.equal(detectJsonIntent({}), false); + }); +}); diff --git a/src/workflow/json-intent.ts b/src/workflow/json-intent.ts new file mode 100644 index 0000000..445cbc6 --- /dev/null +++ b/src/workflow/json-intent.ts @@ -0,0 +1,29 @@ +/** + * Shared heuristic for deciding whether a sampling step is meant to produce JSON. + * + * This is the single source of truth for JSON-mode detection. Compose-time + * data-flow validation uses it to decide whether downstream field access on a + * sampling result is valid; the runtime sampling executor uses the same helper to + * decide whether to JSON-parse the model output. Keeping the rule in one place + * (here, in the workflow layer both consumers share) prevents the two sides from + * drifting apart. + */ + +/** A sampling-like config exposing the free-text fields we inspect for JSON intent. */ +export interface JsonIntentInput { + prompt?: string; + systemPrompt?: string; +} + +/** True when the prompt/systemPrompt signal that the model should respond with JSON. */ +export function detectJsonIntent(step: JsonIntentInput): boolean { + const haystack = + `${step.systemPrompt || ""} ${step.prompt || ""}`.toLowerCase(); + return ( + haystack.includes("json") || + haystack.includes("respond with a json") || + haystack.includes("respond only with") || + haystack.includes("return a json") || + haystack.includes("output format:") + ); +}