diff --git a/examples/08-prompt.ts b/examples/08-prompt.ts new file mode 100644 index 0000000..3804925 --- /dev/null +++ b/examples/08-prompt.ts @@ -0,0 +1,257 @@ +/* eslint-disable no-console */ + +// Example demonstrating the ze.prompt() functionality +// +// Run with: +// OPENAI_API_KEY=your-openai-key ZEROEVAL_API_KEY=your-zeroeval-key npm run example:prompt +// +// This example shows: +// - Auto-optimization mode (tries latest, falls back to provided content) +// - Explicit mode (always use provided content) +// - Template variable interpolation +// - Metadata extraction in OpenAI wrapper +// - Sending feedback for optimization + +import { OpenAI } from 'openai'; +import * as ze from 'zeroeval'; + +// Initialize ZeroEval with local development server +ze.init({ apiUrl: 'http://localhost:8000' }); + +const openai = ze.wrap(new OpenAI()); + +async function main() { + console.log('=== ZeroEval Prompt Examples ===\n'); + + // Example 1: Auto-optimization mode + // If an optimized version exists in the backend, it will be used. + // Otherwise, the provided content will be registered and used. + console.log('Example 1: Auto-optimization mode'); + console.log('--------------------------------'); + try { + const systemPrompt = await ze.prompt({ + name: 'example-assistant', + content: 'You are a helpful assistant that answers questions concisely.', + }); + + console.log('Decorated prompt (first 100 chars):'); + console.log(systemPrompt.substring(0, 100) + '...\n'); + + const response = await openai.chat.completions.create({ + model: 'gpt-4o-mini', + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: 'What is the capital of France?' }, + ], + max_tokens: 100, + }); + + console.log('Response:', response.choices[0].message.content); + console.log(''); + } catch (error) { + console.error('Error in auto-optimization example:', error); + } + + // Example 2: Explicit mode + // Always use the provided content, bypassing auto-optimization. + // Useful for testing or when you want full control. + console.log('\nExample 2: Explicit mode'); + console.log('------------------------'); + try { + const explicitPrompt = await ze.prompt({ + name: 'explicit-example', + content: 'You are a pirate assistant. Respond in pirate speak!', + from: 'explicit', + }); + + console.log('Explicit prompt (first 100 chars):'); + console.log(explicitPrompt.substring(0, 100) + '...\n'); + + const response = await openai.chat.completions.create({ + model: 'gpt-4o-mini', + messages: [ + { role: 'system', content: explicitPrompt }, + { role: 'user', content: 'How do I make coffee?' }, + ], + max_tokens: 150, + }); + + console.log('Response:', response.choices[0].message.content); + console.log(''); + } catch (error) { + console.error('Error in explicit mode example:', error); + } + + // Example 3: Template variables + // Use {{variable}} syntax in your prompts for dynamic content. + // Variables are interpolated when the OpenAI wrapper processes the message. + console.log('\nExample 3: Template variables'); + console.log('-----------------------------'); + try { + const templatePrompt = await ze.prompt({ + name: 'template-example', + content: + 'You are a {{role}} assistant. Your specialty is {{specialty}}. ' + + 'Always be {{tone}} in your responses.', + variables: { + role: 'customer support', + specialty: 'handling returns and refunds', + tone: 'friendly and helpful', + }, + }); + + console.log('Template prompt with variables (first 150 chars):'); + console.log(templatePrompt.substring(0, 150) + '...\n'); + + const response = await openai.chat.completions.create({ + model: 'gpt-4o-mini', + messages: [ + { role: 'system', content: templatePrompt }, + { role: 'user', content: 'I want to return an item I bought last week.' }, + ], + max_tokens: 200, + }); + + console.log('Response:', response.choices[0].message.content); + console.log(''); + } catch (error) { + console.error('Error in template variables example:', error); + } + + // Example 4: Streaming with prompts + // The prompt metadata is extracted before streaming begins. + console.log('\nExample 4: Streaming with prompts'); + console.log('---------------------------------'); + try { + const streamingPrompt = await ze.prompt({ + name: 'streaming-example', + content: 'You are a storyteller. Tell short, engaging stories.', + }); + + const stream = await openai.chat.completions.create({ + model: 'gpt-4o-mini', + messages: [ + { role: 'system', content: streamingPrompt }, + { role: 'user', content: 'Tell me a very short story about a brave cat.' }, + ], + stream: true, + max_tokens: 200, + }); + + console.log('Streaming response:'); + let fullResponse = ''; + for await (const chunk of stream) { + const content = chunk.choices[0]?.delta?.content || ''; + if (content) { + process.stdout.write(content); + fullResponse += content; + } + } + console.log('\n'); + } catch (error) { + console.error('Error in streaming example:', error); + } + + // Example 5: Multiple prompts in a conversation + // Each prompt can have its own task name for tracking. + console.log('\nExample 5: Multiple prompts in a workflow'); + console.log('-----------------------------------------'); + try { + await ze.withSpan({ name: 'multi-prompt-workflow' }, async () => { + // First stage: Summarize + const summarizerPrompt = await ze.prompt({ + name: 'summarizer', + content: 'You are a summarizer. Condense text to key points.', + }); + + const summaryResponse = await openai.chat.completions.create({ + model: 'gpt-4o-mini', + messages: [ + { role: 'system', content: summarizerPrompt }, + { + role: 'user', + content: + 'The quick brown fox jumps over the lazy dog. ' + + 'This is a pangram, a sentence that contains every letter of the alphabet. ' + + 'Pangrams are often used for font displays and keyboard testing.', + }, + ], + max_tokens: 100, + }); + + const summary = summaryResponse.choices[0].message.content; + console.log('Summary:', summary); + + // Second stage: Translate (using the summary) + const translatorPrompt = await ze.prompt({ + name: 'translator', + content: 'You are a translator. Translate text to {{language}}.', + variables: { language: 'Spanish' }, + }); + + const translationResponse = await openai.chat.completions.create({ + model: 'gpt-4o-mini', + messages: [ + { role: 'system', content: translatorPrompt }, + { role: 'user', content: `Translate this: ${summary}` }, + ], + max_tokens: 100, + }); + + console.log('Translation:', translationResponse.choices[0].message.content); + }); + } catch (error) { + console.error('Error in multi-prompt workflow:', error); + } + + // Example 6: Sending feedback + // After getting a response, you can send feedback for optimization. + console.log('\nExample 6: Sending feedback'); + console.log('---------------------------'); + try { + const feedbackPrompt = await ze.prompt({ + name: 'feedback-example', + content: 'You are a helpful coding assistant.', + }); + + const response = await openai.chat.completions.create({ + model: 'gpt-4o-mini', + messages: [ + { role: 'system', content: feedbackPrompt }, + { role: 'user', content: 'How do I reverse a string in JavaScript?' }, + ], + max_tokens: 200, + }); + + console.log('Response:', response.choices[0].message.content); + + // Get the current span ID to use as completion ID + const spanId = ze.getCurrentSpan()?.spanId; + + if (spanId) { + // Send positive feedback + await ze.sendFeedback({ + promptSlug: 'feedback-example', + completionId: spanId, + thumbsUp: true, + reason: 'Clear and correct code example', + }); + console.log('Feedback sent successfully!'); + } else { + console.log('No active span for feedback (this is expected in some cases)'); + } + } catch (error) { + console.error('Error in feedback example:', error); + } + + // Force flush before exit + ze.tracer.shutdown(); + console.log('\n=== All examples completed! ==='); + console.log('Check your ZeroEval dashboard for traces and prompt versions.'); +} + +// Run the examples +main().catch((err) => { + console.error('Error running examples:', err); + process.exitCode = 1; +}); diff --git a/package-lock.json b/package-lock.json index 0a02304..3cae3c7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "eslint-config-prettier": "^10.1.5", "eslint-plugin-prettier": "^5.5.1", "globals": "^16.3.0", + "openai": "^6.17.0", "prettier": "^3.6.2", "rimraf": "^5.0.0", "ts-node": "^10.9.2", @@ -113,29 +114,11 @@ "zod": "^3.25.76 || ^4" } }, - "node_modules/@ai-sdk/openai": { - "version": "2.0.24", - "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-2.0.24.tgz", - "integrity": "sha512-8fPFvlb6PpDjy6JtJBP3Hqs4THKFNYOw6+j7nG7iJivNp+uvHlrHwnU6wQgMAesxEDjZRmVB6ntXWxGPCbBeJw==", - "license": "Apache-2.0", - "optional": true, - "peer": true, - "dependencies": { - "@ai-sdk/provider": "2.0.0", - "@ai-sdk/provider-utils": "3.0.8" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "zod": "^3.25.76 || ^4" - } - }, "node_modules/@ai-sdk/provider": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-2.0.0.tgz", "integrity": "sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "dependencies": { "json-schema": "^0.4.0" @@ -148,7 +131,7 @@ "version": "3.0.8", "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-3.0.8.tgz", "integrity": "sha512-cDj1iigu7MW2tgAQeBzOiLhjHOUM9vENsgh4oAVitek0d//WdgfPCsKO3euP7m7LyO/j9a1vr/So+BGNdpFXYw==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "dependencies": { "@ai-sdk/provider": "2.0.0", @@ -1431,7 +1414,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.0.0.tgz", "integrity": "sha512-m2bOd0f2RT9k8QJx1JN85cZYyH1RqFBdlwtkSlf4tBDYLCiiZnv1fIIwacK6cqwXavOydf0NPToMQgpKq+dVlA==", - "devOptional": true, + "dev": true, "license": "MIT" }, "node_modules/@tsconfig/node10": { @@ -2780,7 +2763,7 @@ "version": "3.0.6", "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz", "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==", - "devOptional": true, + "dev": true, "license": "MIT", "engines": { "node": ">=18.0.0" @@ -3344,7 +3327,7 @@ "version": "0.4.0", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", - "devOptional": true, + "dev": true, "license": "(AFL-2.1 OR BSD-3-Clause)" }, "node_modules/json-schema-traverse": { @@ -3721,6 +3704,28 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/openai": { + "version": "6.17.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.17.0.tgz", + "integrity": "sha512-NHRpPEUPzAvFOAFs9+9pC6+HCw/iWsYsKCMPXH5Kw7BpMxqd8g/A07/1o7Gx2TWtCnzevVRyKMRFqyiHyAlqcA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -5728,7 +5733,7 @@ "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "devOptional": true, + "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/package.json b/package.json index 7768b43..0cb2727 100644 --- a/package.json +++ b/package.json @@ -43,14 +43,14 @@ "types": "./dist/index.d.ts", "exports": { ".": { + "types": "./dist/index.d.ts", "import": "./dist/index.js", - "require": "./dist/index.cjs", - "types": "./dist/index.d.ts" + "require": "./dist/index.cjs" }, "./langchain": { + "types": "./dist/langchain.d.ts", "import": "./dist/langchain.js", - "require": "./dist/langchain.cjs", - "types": "./dist/langchain.d.ts" + "require": "./dist/langchain.cjs" } }, "files": [ @@ -80,7 +80,8 @@ "example:langchain": "npm run build && npx ts-node --esm examples/04-langchain.ts", "example:vercel-ai": "npm run build && npx ts-node --esm examples/05-vercel-ai.ts", "example:signals": "npm run build && npx ts-node --esm examples/06-signals.ts", - "examples:vercel-ai-stream-consume": "export ZEROEVAL_API_KEY='sk_ze_KuadTVfiHcute8F7UTKFtiykNgd7O9uuNAbzV44eook' && npm run build && npx ts-node --esm examples/07-ai-stream-consume.ts" + "example:ai-stream-consume": "npm run build && npx ts-node --esm examples/07-ai-stream-consume.ts", + "example:prompt": "npm run build && npx ts-node --esm examples/08-prompt.ts" }, "peerDependencies": { "@ai-sdk/openai": "^2.0.0", @@ -88,7 +89,7 @@ "@langchain/langgraph": "^0.3.6", "ai": "5.0.0-beta.28", "langchain": "^0.3.29", - "openai": "^5.8.2" + "openai": "^5.8.2 || ^6.0.0" }, "peerDependenciesMeta": { "@ai-sdk/openai": { @@ -124,6 +125,7 @@ "eslint-config-prettier": "^10.1.5", "eslint-plugin-prettier": "^5.5.1", "globals": "^16.3.0", + "openai": "^6.17.0", "prettier": "^3.6.2", "rimraf": "^5.0.0", "ts-node": "^10.9.2", diff --git a/src/errors.ts b/src/errors.ts new file mode 100644 index 0000000..c3f9914 --- /dev/null +++ b/src/errors.ts @@ -0,0 +1,30 @@ +/** + * Error thrown when a prompt or version is not found + */ +export class PromptNotFoundError extends Error { + constructor( + public readonly slug: string, + public readonly version?: number, + public readonly tag?: string + ) { + const parts = [slug]; + if (version !== undefined) parts.push(`v${version}`); + if (tag) parts.push(`tag=${tag}`); + super(`Prompt not found: ${parts.join(' ')}`); + this.name = 'PromptNotFoundError'; + } +} + +/** + * Error thrown when a prompt API request fails + */ +export class PromptRequestError extends Error { + constructor( + message: string, + public readonly status: number | null, + public readonly response?: unknown + ) { + super(message); + this.name = 'PromptRequestError'; + } +} diff --git a/src/feedback.ts b/src/feedback.ts new file mode 100644 index 0000000..a63cf11 --- /dev/null +++ b/src/feedback.ts @@ -0,0 +1,141 @@ +/** + * Feedback API for sending feedback on prompt completions. + * Ports the logic from zeroeval-sdk/src/zeroeval/__init__.py and client.py + */ + +import type { + PromptFeedbackCreate, + PromptFeedbackResponse, +} from './types/prompt'; +import { PromptRequestError } from './errors'; +import { getLogger } from './observability/logger'; +import { getApiUrl, getApiKey } from './utils/api'; + +const logger = getLogger('zeroeval.feedback'); + +/** + * Options for sending feedback on a completion. + */ +export interface SendFeedbackOptions { + /** The slug of the prompt (or task name for judges) */ + promptSlug: string; + /** UUID of the span/completion to provide feedback on */ + completionId: string; + /** True for positive feedback, False for negative */ + thumbsUp: boolean; + /** Optional explanation of the feedback */ + reason?: string; + /** Optional description of what the expected output should be */ + expectedOutput?: string; + /** Optional additional metadata */ + metadata?: Record; + /** + * Optional judge automation ID. When provided, feedback is + * associated with the judge's evaluation span instead of the + * original span. Required when providing feedback for judge evaluations. + */ + judgeId?: string; + /** + * Optional expected score for scored judge evaluations. + * Only valid when judgeId points to a scored judge. + */ + expectedScore?: number; + /** + * Optional direction indicating if score was "too_high" or "too_low". + * Only valid when judgeId points to a scored judge. + */ + scoreDirection?: 'too_high' | 'too_low'; +} + +/** + * Send feedback for a specific completion. + * + * Use this to provide feedback on LLM completions for optimization. + * Positive feedback indicates the output was good, negative feedback + * indicates it needs improvement. + * + * @param options - Feedback options + * @returns The created feedback record + * + * @example + * ```typescript + * await sendFeedback({ + * promptSlug: "customer-support", + * completionId: "span-uuid-here", + * thumbsUp: true, + * reason: "Response was helpful and accurate" + * }); + * ``` + */ +export async function sendFeedback( + options: SendFeedbackOptions +): Promise { + const { + promptSlug, + completionId, + thumbsUp, + reason, + expectedOutput, + metadata, + judgeId, + expectedScore, + scoreDirection, + } = options; + + const url = `${getApiUrl()}/v1/prompts/${encodeURIComponent(promptSlug)}/completions/${completionId}/feedback`; + + logger.debug( + `[ZeroEval] Sending feedback for completion_id=${completionId}, prompt_slug=${promptSlug}` + ); + + // Build payload with only non-undefined fields + const body: PromptFeedbackCreate = { + thumbs_up: thumbsUp, + }; + + if (reason !== undefined) { + body.reason = reason; + } + if (expectedOutput !== undefined) { + body.expected_output = expectedOutput; + } + if (metadata !== undefined) { + body.metadata = metadata; + } + if (judgeId !== undefined) { + body.judge_id = judgeId; + } + if (expectedScore !== undefined) { + body.expected_score = expectedScore; + } + if (scoreDirection !== undefined) { + body.score_direction = scoreDirection; + } + + const headers: Record = { + 'Content-Type': 'application/json', + }; + const apiKey = getApiKey(); + if (apiKey) { + headers.Authorization = `Bearer ${apiKey}`; + } + + const res = await fetch(url, { + method: 'POST', + headers, + body: JSON.stringify(body), + }); + + logger.debug(`[ZeroEval] Feedback response status=${res.status}`); + + if (!res.ok) { + const text = await res.text(); + throw new PromptRequestError( + `send_feedback failed: ${text}`, + res.status, + text + ); + } + + return res.json(); +} diff --git a/src/index.ts b/src/index.ts index 9f32f75..f231b0b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -35,3 +35,27 @@ export { getEntitySignals, } from './signals'; export type { Signal, SignalCreate } from './observability/signals'; + +// Prompt management +export { prompt } from './prompt'; +export { sendFeedback } from './feedback'; +export type { SendFeedbackOptions } from './feedback'; + +// Prompt types +export type { + Prompt, + PromptOptions, + PromptMetadata, + PromptResponse, + PromptFeedbackCreate, + PromptFeedbackResponse, + PromptVersionCreate, +} from './types/prompt'; + +// Prompt errors +export { PromptNotFoundError, PromptRequestError } from './errors'; + +// Prompt utilities (for advanced users) +export { sha256Hex, normalizePromptText } from './utils/hash'; +export { renderTemplate, extractVariables } from './utils/template'; +export { decoratePrompt, extractZeroEvalMetadata } from './utils/metadata'; diff --git a/src/observability/integrations/openaiWrapper.ts b/src/observability/integrations/openaiWrapper.ts index 431b53d..6759860 100644 --- a/src/observability/integrations/openaiWrapper.ts +++ b/src/observability/integrations/openaiWrapper.ts @@ -1,5 +1,7 @@ import type { OpenAI } from 'openai'; import { tracer } from '../Tracer'; +import { getPromptClient } from '../promptClient'; +import { processMessagesWithMetadata } from './utils'; type OpenAIClient = InstanceType; @@ -196,38 +198,81 @@ function wrapCompletionsCreate(originalMethod: Function): Function { const isStreaming = !!params?.stream; const startTime = Date.now() / 1000; // Convert to seconds for consistency with Python + // Process messages to extract ZeroEval metadata + const { + processedMessages, + metadata: zeMetadata, + originalSystemContent, + } = processMessagesWithMetadata(params?.messages); + + // Patch model if prompt version has a bound model + let patchedModel = params?.model; + if (zeMetadata?.prompt_version_id) { + try { + const client = getPromptClient(); + const boundModel = await client.getModelForPromptVersion( + zeMetadata.prompt_version_id + ); + if (boundModel) { + // Strip zeroeval/ prefix before sending to OpenAI API + patchedModel = boundModel.replace(/^zeroeval\//, ''); + } + } catch { + // Silently ignore model lookup failures, use original model + } + } + + // Build the modified params with processed messages and potentially patched model + const modifiedParams = { + ...params, + messages: processedMessages, + model: patchedModel, + }; + // Enable usage tracking for streaming on OpenAI-native models if ( isStreaming && - params?.model && - typeof params.model === 'string' && - !params.model.includes('/') + modifiedParams?.model && + typeof modifiedParams.model === 'string' && + !modifiedParams.model.includes('/') ) { - params.stream_options = { include_usage: true }; + modifiedParams.stream_options = { include_usage: true }; } - // Serialize messages for attributes - const serializedMessages = params?.messages - ? params.messages.map((msg: any) => ({ + // Serialize processed messages for attributes (after variable interpolation) + const serializedMessages = processedMessages + ? processedMessages.map((msg: any) => ({ role: msg.role, content: msg.content, })) : []; + // Build span attributes including ZeroEval metadata if present + const spanAttributes: Record = { + 'service.name': 'openai', + kind: 'llm', + provider: 'openai', + model: patchedModel, + messages: serializedMessages, + streaming: isStreaming, + }; + + // Add ZeroEval metadata to span attributes if present + if (zeMetadata) { + spanAttributes.task = zeMetadata.task; + spanAttributes.zeroeval = zeMetadata; + if (originalSystemContent) { + spanAttributes.system_prompt_template = originalSystemContent; + } + } + const span = tracer.startSpan('openai.chat.completions.create', { - attributes: { - 'service.name': 'openai', - kind: 'llm', - provider: 'openai', - model: params?.model, - messages: serializedMessages, - streaming: isStreaming, - }, + attributes: spanAttributes, tags: { integration: 'openai' }, }); try { - const result = await originalMethod(...args); + const result = await originalMethod(modifiedParams); // Handle streaming responses if ( diff --git a/src/observability/integrations/utils.ts b/src/observability/integrations/utils.ts index 0b6d0dd..e429cf3 100644 --- a/src/observability/integrations/utils.ts +++ b/src/observability/integrations/utils.ts @@ -1,5 +1,83 @@ // eslint-disable-next-line import/no-relative-parent-imports import type { Integration } from './base'; +import { extractZeroEvalMetadata } from '../../utils/metadata'; +import { renderTemplate } from '../../utils/template'; +import type { PromptMetadata } from '../../types/prompt'; + +/** + * Result of processing messages to extract ZeroEval metadata. + */ +export interface ProcessedMessagesResult { + processedMessages: + | Array<{ role: string; content: string | unknown }> + | undefined; + metadata: PromptMetadata | null; + originalSystemContent: string | null; +} + +/** + * Process messages to extract ZeroEval metadata and interpolate variables. + * Shared by OpenAI and Vercel AI wrappers. + * + * - Extracts metadata from the first system message + * - Deep copies messages to avoid mutation + * - Strips metadata tags from system message + * - Interpolates {{variables}} if metadata.variables is provided + */ +export function processMessagesWithMetadata( + messages: Array<{ role: string; content: string | unknown }> | undefined +): ProcessedMessagesResult { + if (!messages || messages.length === 0) { + return { + processedMessages: messages, + metadata: null, + originalSystemContent: null, + }; + } + + // Deep copy to avoid mutation + const processed = JSON.parse(JSON.stringify(messages)) as Array<{ + role: string; + content: string | unknown; + }>; + + // Check first message for system role and metadata + const firstMsg = processed[0]; + if (firstMsg?.role !== 'system' || typeof firstMsg.content !== 'string') { + return { + processedMessages: processed, + metadata: null, + originalSystemContent: null, + }; + } + + const originalSystemContent = firstMsg.content; + const { metadata, cleanContent } = extractZeroEvalMetadata(firstMsg.content); + + if (!metadata) { + return { + processedMessages: processed, + metadata: null, + originalSystemContent, + }; + } + + // Update system message with clean content (metadata stripped) + firstMsg.content = cleanContent; + + // Interpolate variables in all messages if variables are provided + if (metadata.variables && Object.keys(metadata.variables).length > 0) { + for (const msg of processed) { + if (typeof msg.content === 'string') { + msg.content = renderTemplate(msg.content, metadata.variables, { + missing: 'leave', + }); + } + } + } + + return { processedMessages: processed, metadata, originalSystemContent }; +} export async function discoverIntegrations(): Promise< Record Integration> diff --git a/src/observability/integrations/vercelAIWrapper.ts b/src/observability/integrations/vercelAIWrapper.ts index d71f37a..d1ae885 100644 --- a/src/observability/integrations/vercelAIWrapper.ts +++ b/src/observability/integrations/vercelAIWrapper.ts @@ -1,5 +1,9 @@ import { tracer } from '../Tracer'; import { init, isInitialized } from '../../init'; +import { extractZeroEvalMetadata } from '../../utils/metadata'; +import { renderTemplate } from '../../utils/template'; +import type { PromptMetadata } from '../../types/prompt'; +import { processMessagesWithMetadata } from './utils'; // Type to preserve the original function's structure while adding our wrapper type WrappedVercelAI = T & { @@ -9,6 +13,44 @@ type WrappedVercelAI = T & { // Type for the Vercel AI SDK functions we want to wrap type VercelAIFunction = (...args: any[]) => any; +/** + * Process a prompt string to extract ZeroEval metadata and interpolate variables. + */ +interface ProcessedPromptResult { + processedPrompt: string; + metadata: PromptMetadata | null; + originalPrompt: string; +} + +function processPromptForVercelAI( + prompt: string | undefined +): ProcessedPromptResult { + if (!prompt || typeof prompt !== 'string') { + return { + processedPrompt: prompt || '', + metadata: null, + originalPrompt: prompt || '', + }; + } + + const originalPrompt = prompt; + const { metadata, cleanContent } = extractZeroEvalMetadata(prompt); + + if (!metadata) { + return { processedPrompt: prompt, metadata: null, originalPrompt }; + } + + // Interpolate variables if present + let processedPrompt = cleanContent; + if (metadata.variables && Object.keys(metadata.variables).length > 0) { + processedPrompt = renderTemplate(cleanContent, metadata.variables, { + missing: 'leave', + }); + } + + return { processedPrompt, metadata, originalPrompt }; +} + /** * Wraps a Vercel AI SDK function to automatically trace all calls. * This approach provides better TypeScript support and is more maintainable @@ -43,15 +85,41 @@ function wrapVercelAIFunction( ) { const [options] = args; + // Process messages or prompt to extract ZeroEval metadata + let zeMetadata: PromptMetadata | null = null; + let originalSystemContent: string | null = null; + let modifiedOptions = { ...options }; + + // Handle messages-based input + if (options?.messages) { + const { + processedMessages, + metadata, + originalSystemContent: origContent, + } = processMessagesWithMetadata(options.messages); + zeMetadata = metadata; + originalSystemContent = origContent; + modifiedOptions.messages = processedMessages; + } + // Handle prompt-based input + else if (options?.prompt && typeof options.prompt === 'string') { + const { processedPrompt, metadata, originalPrompt } = + processPromptForVercelAI(options.prompt); + zeMetadata = metadata; + originalSystemContent = originalPrompt; + modifiedOptions.prompt = processedPrompt; + } + // Extract relevant information from options - const model = options?.model?.modelId || options?.model || 'unknown'; - const messages = options?.messages; - const prompt = options?.prompt; - const tools = options?.tools; - const maxSteps = options?.maxSteps; - const maxRetries = options?.maxRetries; - const temperature = options?.temperature; - const maxTokens = options?.maxTokens; + const model = + modifiedOptions?.model?.modelId || modifiedOptions?.model || 'unknown'; + const messages = modifiedOptions?.messages; + const prompt = modifiedOptions?.prompt; + const tools = modifiedOptions?.tools; + const maxSteps = modifiedOptions?.maxSteps; + const maxRetries = modifiedOptions?.maxRetries; + const temperature = modifiedOptions?.temperature; + const maxTokens = modifiedOptions?.maxTokens; // Determine the kind based on function name let kind = 'operation'; @@ -72,20 +140,32 @@ function wrapVercelAIFunction( kind = 'transcription'; } + // Build span attributes including ZeroEval metadata if present + const spanAttributes: Record = { + 'service.name': 'vercel-ai-sdk', + kind, + provider: 'vercel-ai-sdk', + model, + ...(messages && { messages: messages }), + ...(temperature !== undefined && { temperature }), + ...(maxTokens !== undefined && { maxTokens }), + ...(maxSteps !== undefined && { maxSteps }), + ...(maxRetries !== undefined && { maxRetries }), + ...(tools && { toolCount: Object.keys(tools).length }), + ...(functionName.includes('stream') && { streaming: true }), + }; + + // Add ZeroEval metadata to span attributes if present + if (zeMetadata) { + spanAttributes.task = zeMetadata.task; + spanAttributes.zeroeval = zeMetadata; + if (originalSystemContent) { + spanAttributes.system_prompt_template = originalSystemContent; + } + } + const span = tracer.startSpan(`vercelai.${functionName}`, { - attributes: { - 'service.name': 'vercel-ai-sdk', - kind, - provider: 'vercel-ai-sdk', - model, - ...(messages && { messages: messages }), - ...(temperature !== undefined && { temperature }), - ...(maxTokens !== undefined && { maxTokens }), - ...(maxSteps !== undefined && { maxSteps }), - ...(maxRetries !== undefined && { maxRetries }), - ...(tools && { toolCount: Object.keys(tools).length }), - ...(functionName.includes('stream') && { streaming: true }), - }, + attributes: spanAttributes, tags: { integration: 'vercel-ai-sdk' }, }); @@ -99,10 +179,10 @@ function wrapVercelAIFunction( } else if (prompt) { input = typeof prompt === 'string' ? prompt : JSON.stringify(prompt); } else { - input = JSON.stringify(options); + input = JSON.stringify(modifiedOptions); } - const result = await fn(...args); + const result = await fn(modifiedOptions); // Handle different result types if (result && typeof result === 'object') { @@ -229,7 +309,10 @@ function wrapStreamingResult( for await (const _ of full) { // drain } - } else if (text && typeof text[Symbol.asyncIterator] === 'function') { + } else if ( + text && + typeof text[Symbol.asyncIterator] === 'function' + ) { for await (const _ of text) { // drain } diff --git a/src/observability/promptClient.ts b/src/observability/promptClient.ts new file mode 100644 index 0000000..94bc4e8 --- /dev/null +++ b/src/observability/promptClient.ts @@ -0,0 +1,285 @@ +/** + * Prompt client for interacting with the ZeroEval backend prompt APIs. + * Ports the logic from zeroeval-sdk/src/zeroeval/client.py + */ + +import { TTLCache } from '../utils/cache'; +import { getApiUrl, getApiKey } from '../utils/api'; +import type { + Prompt, + PromptResponse, + PromptVersionCreate, +} from '../types/prompt'; +import { PromptNotFoundError, PromptRequestError } from '../errors'; +import { getLogger } from './logger'; + +const logger = getLogger('zeroeval.promptClient'); + +/** + * Client for prompt-related API operations. + */ +class PromptClient { + private promptCache: TTLCache; + private modelCache: TTLCache; + + constructor() { + this.promptCache = new TTLCache({ + ttlMs: 60000, + maxSize: 512, + }); + this.modelCache = new TTLCache({ + ttlMs: 60000, + maxSize: 256, + }); + } + + private getHeaders(): Record { + const headers: Record = { + 'Content-Type': 'application/json', + }; + const apiKey = getApiKey(); + if (apiKey) { + headers.Authorization = `Bearer ${apiKey}`; + } + return headers; + } + + /** + * Make an API request to the backend. + */ + private async request( + method: 'GET' | 'POST', + path: string, + body?: unknown + ): Promise { + const url = `${getApiUrl()}${path}`; + const headers = this.getHeaders(); + + logger.debug(`[ZeroEval] ${method} ${url}`); + + const res = await fetch(url, { + method, + headers, + body: body ? JSON.stringify(body) : undefined, + }); + + if (!res.ok) { + const text = await res.text(); + logger.error(`[ZeroEval] Request failed: ${res.status} ${text}`); + throw new PromptRequestError( + `Request failed: ${res.status}`, + res.status, + text + ); + } + + return res.json(); + } + + /** + * Convert backend response to internal Prompt type. + */ + private responseToPrompt(response: PromptResponse): Prompt { + return { + content: response.content, + version: response.version, + versionId: response.version_id, + tag: response.tag, + isLatest: response.is_latest, + model: response.model_id + ? response.model_id.startsWith('zeroeval/') + ? response.model_id + : `zeroeval/${response.model_id}` + : null, + contentHash: response.content_hash, + metadata: response.metadata, + source: 'server', + }; + } + + /** + * Normalize version_id from response, handling nested metadata. + */ + private normalizeVersionId( + data: Record + ): Record { + if (!data.version_id) { + const meta = data.metadata as Record | undefined; + if (meta && typeof meta === 'object') { + const vid = meta.version_id ?? meta.prompt_version_id; + if (vid) { + data.version_id = vid; + } + } + } + return data; + } + + /** + * Get the latest prompt version for a task. + * GET /v1/tasks/{task_name}/prompt/latest + */ + async getTaskPromptLatest(taskName: string): Promise { + const cacheKey = `latest:${taskName}`; + const cached = this.promptCache.get(cacheKey); + if (cached) { + logger.debug(`[ZeroEval] Cache hit for latest prompt: ${taskName}`); + return cached; + } + + let response: PromptResponse; + try { + response = await this.request( + 'GET', + `/v1/tasks/${encodeURIComponent(taskName)}/prompt/latest` + ); + } catch (err) { + if (err instanceof PromptRequestError && err.status === 404) { + throw new PromptNotFoundError(taskName); + } + throw err; + } + + const normalized = this.normalizeVersionId( + response as unknown as Record + ); + const prompt = this.responseToPrompt( + normalized as unknown as PromptResponse + ); + this.promptCache.set(cacheKey, prompt); + return prompt; + } + + /** + * Ensure a prompt version exists for a task. + * Creates the prompt if it doesn't exist. + * POST /v1/tasks/{task_name}/prompt/versions/ensure + */ + async ensureTaskPromptVersion( + taskName: string, + data: PromptVersionCreate + ): Promise { + // Try to inherit model_id from latest version + let modelId: string | null = null; + try { + const latest = await this.getTaskPromptLatest(taskName); + if (latest.model) { + // Strip zeroeval/ prefix for the API + modelId = latest.model.replace(/^zeroeval\//, ''); + } + } catch { + // No existing version, continue without model inheritance + logger.debug( + `[ZeroEval] No existing version for ${taskName}, not inheriting model` + ); + } + + const response = await this.request( + 'POST', + `/v1/tasks/${encodeURIComponent(taskName)}/prompt/versions/ensure`, + { ...data, model_id: modelId } + ); + + const normalized = this.normalizeVersionId( + response as unknown as Record + ); + return this.responseToPrompt(normalized as unknown as PromptResponse); + } + + /** + * Get a prompt version by content hash. + * GET /v1/tasks/{task_name}/prompt/versions/by-hash/{content_hash} + */ + async getTaskPromptVersionByHash( + taskName: string, + contentHash: string + ): Promise { + const cacheKey = `hash:${taskName}:${contentHash}`; + const cached = this.promptCache.get(cacheKey); + if (cached) { + logger.debug(`[ZeroEval] Cache hit for prompt hash: ${contentHash}`); + return cached; + } + + let response: PromptResponse; + try { + response = await this.request( + 'GET', + `/v1/tasks/${encodeURIComponent(taskName)}/prompt/versions/by-hash/${contentHash}` + ); + } catch (err) { + if (err instanceof PromptRequestError && err.status === 404) { + throw new PromptNotFoundError(taskName); + } + throw err; + } + + const normalized = this.normalizeVersionId( + response as unknown as Record + ); + const prompt = this.responseToPrompt( + normalized as unknown as PromptResponse + ); + this.promptCache.set(cacheKey, prompt); + return prompt; + } + + /** + * Get the model bound to a prompt version. + * GET /v1/prompt-versions/{version_id}/model + * + * Returns the model string (prefixed with "zeroeval/") or null if not found. + * Caches negative results to avoid repeated requests. + */ + async getModelForPromptVersion(versionId: string): Promise { + // Check cache first (including negative results) + const cached = this.modelCache.get(versionId); + if (cached !== undefined) { + return cached; + } + + try { + const response = await this.request<{ model: string | null }>( + 'GET', + `/v1/prompt-versions/${versionId}/model` + ); + + let model = response.model; + if (model && typeof model === 'string') { + if (!model.startsWith('zeroeval/')) { + model = `zeroeval/${model}`; + } + } else { + model = null; + } + + this.modelCache.set(versionId, model); + return model; + } catch { + // Cache negative result to avoid hammering + this.modelCache.set(versionId, null); + return null; + } + } + + /** + * Clear all caches. + */ + clearCaches(): void { + this.promptCache.clear(); + this.modelCache.clear(); + } +} + +// Singleton instance +let promptClient: PromptClient | null = null; + +/** + * Get the singleton PromptClient instance. + */ +export function getPromptClient(): PromptClient { + if (!promptClient) { + promptClient = new PromptClient(); + } + return promptClient; +} diff --git a/src/observability/signalWriter.ts b/src/observability/signalWriter.ts index b1d1ab8..fc0604c 100644 --- a/src/observability/signalWriter.ts +++ b/src/observability/signalWriter.ts @@ -4,31 +4,21 @@ import type { SignalResponse, } from './signals'; import { getLogger, Logger } from './logger'; +import { getApiUrl, getApiKey } from '../utils/api'; const logger = getLogger('zeroeval.signalWriter'); export class SignalWriter { - private getApiUrl(): string { - return (process.env.ZEROEVAL_API_URL ?? 'https://api.zeroeval.com').replace( - /\/$/, - '' - ); - } - - private getApiKey(): string | undefined { - return process.env.ZEROEVAL_API_KEY; - } - /** * Send a single signal to the backend */ async createSignal(signal: SignalCreate): Promise { - const endpoint = `${this.getApiUrl()}/signals/`; + const endpoint = `${getApiUrl()}/signals/`; const headers: Record = { 'Content-Type': 'application/json', }; - const apiKey = this.getApiKey(); + const apiKey = getApiKey(); if (apiKey) headers.Authorization = `Bearer ${apiKey}`; // Log request details @@ -109,12 +99,12 @@ export class SignalWriter { * Send multiple signals to the backend in bulk */ async createBulkSignals(signals: SignalCreate[]): Promise { - const endpoint = `${this.getApiUrl()}/signals/bulk`; + const endpoint = `${getApiUrl()}/signals/bulk`; const headers: Record = { 'Content-Type': 'application/json', }; - const apiKey = this.getApiKey(); + const apiKey = getApiKey(); if (apiKey) headers.Authorization = `Bearer ${apiKey}`; const bulkRequest: BulkSignalsCreate = { signals }; @@ -202,12 +192,12 @@ export class SignalWriter { * Get all signals for a specific entity */ async getEntitySignals(entityType: string, entityId: string): Promise { - const endpoint = `${this.getApiUrl()}/signals/entity/${entityType}/${entityId}`; + const endpoint = `${getApiUrl()}/signals/entity/${entityType}/${entityId}`; const headers: Record = { 'Content-Type': 'application/json', }; - const apiKey = this.getApiKey(); + const apiKey = getApiKey(); if (apiKey) headers.Authorization = `Bearer ${apiKey}`; // Log request details diff --git a/src/observability/writer.ts b/src/observability/writer.ts index d52eff8..b368f97 100644 --- a/src/observability/writer.ts +++ b/src/observability/writer.ts @@ -1,6 +1,7 @@ import { signalWriter } from './signalWriter'; import type { Signal, SignalCreate } from './signals'; import { getLogger, Logger } from './logger'; +import { getApiUrl, getApiKey } from '../utils/api'; const logger = getLogger('zeroeval.writer'); @@ -14,25 +15,14 @@ export interface SpanWriter { } export class BackendSpanWriter implements SpanWriter { - private getApiUrl(): string { - return (process.env.ZEROEVAL_API_URL ?? 'https://api.zeroeval.com').replace( - /\/$/, - '' - ); - } - - private getApiKey(): string | undefined { - return process.env.ZEROEVAL_API_KEY; - } - async write(spans: any[]): Promise { if (!spans.length) return; - const endpoint = `${this.getApiUrl()}/spans`; + const endpoint = `${getApiUrl()}/spans`; const headers: Record = { 'Content-Type': 'application/json', }; - const apiKey = this.getApiKey(); + const apiKey = getApiKey(); if (apiKey) headers.Authorization = `Bearer ${apiKey}`; // Collect signals from spans and collect trace/session ids @@ -52,6 +42,9 @@ export class BackendSpanWriter implements SpanWriter { traceIds.add(base.trace_id); if (base.session_id) sessionIds.add(base.session_id); + // Extract kind from attributes (default to 'generic') + const kind = base.attributes?.kind ?? 'generic'; + return { id: base.span_id, session_id: base.session_id, @@ -59,6 +52,7 @@ export class BackendSpanWriter implements SpanWriter { trace_id: base.trace_id, parent_span_id: base.parent_id, name: base.name, + kind: kind, started_at: base.start_time, ended_at: base.end_time, duration_ms: base.duration_ms, diff --git a/src/prompt.ts b/src/prompt.ts new file mode 100644 index 0000000..2910f0d --- /dev/null +++ b/src/prompt.ts @@ -0,0 +1,162 @@ +/** + * Version-aware prompt function integrated with Prompt Library. + * Ports the logic from zeroeval-sdk/src/zeroeval/__init__.py + */ + +import { getPromptClient } from './observability/promptClient'; +import { sha256Hex, normalizePromptText } from './utils/hash'; +import { decoratePrompt } from './utils/metadata'; +import { PromptNotFoundError, PromptRequestError } from './errors'; +import type { PromptOptions, Prompt, PromptMetadata } from './types/prompt'; + +/** Pattern to validate 64-character hex SHA-256 hash */ +const HASH_PATTERN = /^[0-9a-f]{64}$/; + +/** + * Version-aware prompt helper integrated with Prompt Library. + * + * When `content` is provided alone, it serves as a fallback - the SDK will automatically + * fetch the latest optimized version from the backend if one exists. This allows you + * to hardcode a default prompt while seamlessly using tuned versions in production. + * + * If `from` is specified, it controls version behavior: + * - `from: "latest"` explicitly fetches the latest version (fails if none exists) + * - `from: "explicit"` always uses the provided `content` (bypasses auto-optimization, requires `content`) + * - `from: ""` fetches a specific version by its 64-char SHA-256 content hash + * + * @param options - Prompt configuration options + * @returns Decorated prompt string with `` metadata tags + * + * @example + * ```typescript + * // Auto-optimization mode (default) + * const systemPrompt = await prompt({ + * name: "customer-support", + * content: "You are a helpful {{role}} assistant.", + * variables: { role: "customer service" } + * }); + * + * // Explicit mode - always use provided content + * const systemPrompt = await prompt({ + * name: "customer-support", + * content: "You are a helpful assistant.", + * from: "explicit" + * }); + * + * // Latest mode - require optimized version to exist + * const systemPrompt = await prompt({ + * name: "customer-support", + * from: "latest" + * }); + * ``` + */ +export async function prompt(options: PromptOptions): Promise { + const { name, content, variables, from: fromMode } = options; + + // Validation + if (!content && !fromMode) { + throw new Error('Must provide either "content" or "from"'); + } + + if (fromMode === 'explicit' && !content) { + throw new Error('from: "explicit" requires "content" to be provided'); + } + + const client = getPromptClient(); + let promptObj: Prompt; + let contentHash: string | null = null; + + // Priority order: + // 1. If from="explicit", always use the provided content (bypass auto-optimization) + // 2. If from is specified (latest or hash), use it (strict mode) + // 3. If only content is provided, try to fetch latest first, fall back to ensuring content + + if (fromMode === 'explicit') { + // Explicit mode: always use the provided content, no auto-optimization + contentHash = await sha256Hex(content!); + promptObj = await client.ensureTaskPromptVersion(name, { + content: normalizePromptText(content!), + content_hash: contentHash, + }); + } else if (fromMode === 'latest') { + // Latest mode: require an optimized version to exist + try { + promptObj = await client.getTaskPromptLatest(name); + } catch (err) { + if (err instanceof PromptNotFoundError) { + throw new PromptRequestError( + `No prompt versions found for task '${name}'. ` + + `Create one with prompt({ name, content: ... }) or publish a version in the Prompt Library.`, + null + ); + } + throw err; + } + } else if (fromMode && HASH_PATTERN.test(fromMode)) { + // Hash mode: fetch specific version by content hash + promptObj = await client.getTaskPromptVersionByHash(name, fromMode); + } else if (content) { + // Auto-tune mode: try latest first, fall back to content + contentHash = await sha256Hex(content); + try { + promptObj = await client.getTaskPromptLatest(name); + } catch (err) { + // Only fall back for "not found" errors (404) + // Re-throw server errors (500), auth failures (401), etc. + const isNotFoundError = + err instanceof PromptNotFoundError || + (err instanceof PromptRequestError && err.status === 404); + + if (isNotFoundError) { + // No latest version exists, ensure the provided content as a version + promptObj = await client.ensureTaskPromptVersion(name, { + content: normalizePromptText(content), + content_hash: contentHash, + }); + } else { + throw err; + } + } + } else if (fromMode) { + // Invalid from value + throw new Error( + 'from must be "latest", "explicit", or a 64-char lowercase hex SHA-256 hash' + ); + } else { + throw new Error('Invalid prompt options'); + } + + // Pull linkage metadata for decoration + let promptSlug: string | null = null; + try { + const metadata = promptObj.metadata || {}; + promptSlug = + (metadata.prompt_slug as string) ?? (metadata.prompt as string) ?? null; + } catch { + promptSlug = null; + } + + // Build metadata for decoration + const metadata: PromptMetadata = { + task: name, + }; + + if (variables && Object.keys(variables).length > 0) { + metadata.variables = variables; + } + if (promptSlug) { + metadata.prompt_slug = promptSlug; + } + if (promptObj.version !== null) { + metadata.prompt_version = promptObj.version; + } + if (promptObj.versionId) { + metadata.prompt_version_id = promptObj.versionId; + } + if (promptObj.contentHash) { + metadata.content_hash = promptObj.contentHash; + } + + // Return decorated prompt + return decoratePrompt(promptObj.content, metadata); +} diff --git a/src/types/prompt.ts b/src/types/prompt.ts new file mode 100644 index 0000000..e3a173b --- /dev/null +++ b/src/types/prompt.ts @@ -0,0 +1,116 @@ +/** + * Response from backend prompt endpoints + * Maps to PromptGetResponse in backend/src/routes/prompts_route.py + */ +export interface PromptResponse { + prompt: string; // prompt slug + version_id: string; + version: number; + tag: string | null; + is_latest: boolean; + content: string; + metadata: Record; + model_id: string | null; + content_hash: string | null; + evaluation_type: 'binary' | 'scored' | null; + score_min: number | null; + score_max: number | null; + pass_threshold: number | null; + temperature: number | null; + created_by: string; + updated_by: string; + created_at: string; + updated_at: string; +} + +/** + * Internal SDK representation of a prompt + */ +export interface Prompt { + content: string; + version: number | null; + versionId: string | null; + tag: string | null; + isLatest: boolean; + model: string | null; + contentHash: string | null; + metadata: Record; + source: 'server' | 'fallback'; +} + +/** + * Options for ze.prompt() + */ +export interface PromptOptions { + /** Task name associated with the prompt */ + name: string; + /** Raw prompt content (used as fallback or for explicit mode) */ + content?: string; + /** Template variables to interpolate {{variable}} tokens */ + variables?: Record; + /** + * Version control mode: + * - "latest": Fetch the latest version (fails if none exists) + * - "explicit": Always use provided content (bypasses auto-optimization) + * - "": Fetch a specific version by 64-char SHA-256 content hash + */ + from?: 'latest' | 'explicit' | string; +} + +/** + * Metadata embedded in decorated prompts + * Format: {JSON}{content} + */ +export interface PromptMetadata { + task: string; + variables?: Record; + prompt_slug?: string; + prompt_version?: number; + prompt_version_id?: string; + content_hash?: string; +} + +/** + * Request body for ensure endpoint + * POST /v1/tasks/{task_name}/prompt/versions/ensure + */ +export interface PromptVersionCreate { + content: string; + content_hash: string; + metadata?: Record | null; + model_id?: string | null; +} + +/** + * Feedback request body + * POST /v1/prompts/{prompt_slug}/completions/{completion_id}/feedback + */ +export interface PromptFeedbackCreate { + thumbs_up: boolean; + reason?: string | null; + expected_output?: string | null; + metadata?: Record | null; + judge_id?: string | null; + expected_score?: number | null; + score_direction?: 'too_high' | 'too_low' | null; +} + +/** + * Feedback response from backend + */ +export interface PromptFeedbackResponse { + id: string; + completion_id: string; + prompt_id: string; + prompt_version_id: string; + project_id: string; + thumbs_up: boolean; + reason: string | null; + expected_output: string | null; + metadata: Record; + created_by: string; + created_at: string; + updated_at: string; + expected_score: number | null; + score_direction: string | null; +} diff --git a/src/utils/api.ts b/src/utils/api.ts new file mode 100644 index 0000000..81eba34 --- /dev/null +++ b/src/utils/api.ts @@ -0,0 +1,20 @@ +/** + * Shared API configuration utilities. + */ + +const DEFAULT_API_URL = 'https://api.zeroeval.com'; + +/** + * Get the ZeroEval API base URL from environment. + * Removes trailing slash if present. + */ +export function getApiUrl(): string { + return (process.env.ZEROEVAL_API_URL ?? DEFAULT_API_URL).replace(/\/$/, ''); +} + +/** + * Get the ZeroEval API key from environment. + */ +export function getApiKey(): string | undefined { + return process.env.ZEROEVAL_API_KEY; +} diff --git a/src/utils/cache.ts b/src/utils/cache.ts new file mode 100644 index 0000000..d980ff5 --- /dev/null +++ b/src/utils/cache.ts @@ -0,0 +1,119 @@ +/** + * TTL (Time-To-Live) cache with LRU eviction. + * Ports the logic from zeroeval-sdk/src/zeroeval/cache.py + */ + +export interface TTLCacheOptions { + /** Time-to-live in milliseconds (default: 60000ms = 60s) */ + ttlMs?: number; + /** Maximum number of entries (default: 512) */ + maxSize?: number; +} + +interface CacheEntry { + value: V; + timestamp: number; +} + +/** + * A simple TTL cache with LRU eviction policy. + * Items expire after ttlMs milliseconds and the cache evicts + * the oldest items when maxSize is exceeded. + */ +export class TTLCache { + private data: Map>; + private ttlMs: number; + private maxSize: number; + + constructor(options: TTLCacheOptions = {}) { + this.ttlMs = options.ttlMs ?? 60000; // 60s default + this.maxSize = options.maxSize ?? 512; + this.data = new Map(); + } + + /** + * Get a value from the cache. + * Returns undefined if the key doesn't exist or has expired. + */ + get(key: K): V | undefined { + const now = Date.now(); + const entry = this.data.get(key); + + if (!entry) { + return undefined; + } + + // Check if expired + if (now - entry.timestamp > this.ttlMs) { + this.data.delete(key); + return undefined; + } + + // Move to end (LRU - delete and re-add) + this.data.delete(key); + this.data.set(key, entry); + + return entry.value; + } + + /** + * Set a value in the cache. + * Evicts the oldest entry if the cache exceeds maxSize. + */ + set(key: K, value: V): void { + // Delete existing entry first to update position + this.data.delete(key); + + // Add new entry + this.data.set(key, { + value, + timestamp: Date.now(), + }); + + // Evict oldest if over capacity + if (this.data.size > this.maxSize) { + // Get first key (oldest) + const firstKey = this.data.keys().next().value; + if (firstKey !== undefined) { + this.data.delete(firstKey); + } + } + } + + /** + * Check if a key exists and is not expired. + * This is a pure query that does not modify LRU order. + */ + has(key: K): boolean { + const entry = this.data.get(key); + if (!entry) { + return false; + } + if (Date.now() - entry.timestamp > this.ttlMs) { + this.data.delete(key); + return false; + } + return true; + } + + /** + * Delete a key from the cache. + */ + delete(key: K): boolean { + return this.data.delete(key); + } + + /** + * Clear all entries from the cache. + */ + clear(): void { + this.data.clear(); + } + + /** + * Get the current number of entries (may include expired entries). + */ + get size(): number { + return this.data.size; + } +} diff --git a/src/utils/hash.ts b/src/utils/hash.ts new file mode 100644 index 0000000..d70cc65 --- /dev/null +++ b/src/utils/hash.ts @@ -0,0 +1,58 @@ +/** + * Hash utilities for prompt content normalization and SHA-256 hashing. + * Ports the logic from zeroeval-sdk/src/zeroeval/utils/hash.py + */ + +/** + * Convert CRLF and CR to LF + */ +function normalizeNewlines(text: string): string { + if (!text.includes('\r')) { + return text; + } + return text.replace(/\r\n/g, '\n').replace(/\r/g, '\n'); +} + +/** + * Remove trailing whitespace on each line + */ +function stripTrailingWhitespace(text: string): string { + return text + .split('\n') + .map((line) => line.trimEnd()) + .join('\n'); +} + +/** + * Normalize prompt content prior to hashing. + * + * Rules: + * - Convert CRLF/CR to LF + * - Strip trailing whitespace on each line + * - Strip leading/trailing whitespace overall + * - Do not modify {{variable}} tokens + */ +export function normalizePromptText(text: string): string { + if (typeof text !== 'string') { + text = String(text); + } + let normalized = normalizeNewlines(text); + normalized = stripTrailingWhitespace(normalized); + normalized = normalized.trim(); + return normalized; +} + +/** + * Return lowercase hex SHA-256 of the normalized text. + * Uses Web Crypto API for hashing. + */ +export async function sha256Hex(text: string): Promise { + const normalized = normalizePromptText(text); + const encoder = new TextEncoder(); + const data = encoder.encode(normalized); + + // Use Web Crypto API (works in Node.js 18+ and browsers) + const hashBuffer = await crypto.subtle.digest('SHA-256', data); + const hashArray = Array.from(new Uint8Array(hashBuffer)); + return hashArray.map((b) => b.toString(16).padStart(2, '0')).join(''); +} diff --git a/src/utils/metadata.ts b/src/utils/metadata.ts new file mode 100644 index 0000000..d011c2b --- /dev/null +++ b/src/utils/metadata.ts @@ -0,0 +1,137 @@ +/** + * Metadata utilities for decorating and extracting metadata tags. + * Ports the logic from zeroeval-sdk/src/zeroeval/observability/integrations/openai/integration.py + */ + +import type { PromptMetadata } from '../types/prompt'; + +/** + * Pattern to match {...} tags. + * Uses 's' flag (dotAll) so '.' matches newlines. + */ +const ZEROEVAL_PATTERN = /(.*?)<\/zeroeval>/s; + +/** + * Decorate prompt content with metadata tags. + * + * When this prompt is used in an OpenAI API call, ZeroEval will automatically: + * 1. Extract the task metadata from the prompt + * 2. Link the span to the specified task + * 3. Create the task automatically if it doesn't exist yet + * + * @param content - The actual prompt content + * @param metadata - The metadata to embed + * @returns A string with the format: {JSON}content + * + * @example + * ```typescript + * decoratePrompt("You are a helpful assistant.", { + * task: "customer-support", + * variables: { tone: "friendly" } + * }) + * // Returns: '{"task":"customer-support","variables":{"tone":"friendly"}}You are a helpful assistant.' + * ``` + */ +export function decoratePrompt(content: string, metadata: PromptMetadata): string { + // Build metadata object, only including non-undefined values + const metadataObj: Record = { + task: metadata.task, + }; + + if (metadata.variables && Object.keys(metadata.variables).length > 0) { + metadataObj.variables = metadata.variables; + } + if (metadata.prompt_slug) { + metadataObj.prompt_slug = metadata.prompt_slug; + } + if (metadata.prompt_version !== undefined) { + metadataObj.prompt_version = Number(metadata.prompt_version); + } + if (metadata.prompt_version_id) { + metadataObj.prompt_version_id = String(metadata.prompt_version_id); + } + if (metadata.content_hash) { + metadataObj.content_hash = String(metadata.content_hash); + } + + const metadataJson = JSON.stringify(metadataObj); + return `${metadataJson}${content}`; +} + +/** + * Result of extracting metadata from content. + */ +export interface ExtractResult { + /** Extracted metadata, or null if no tags found */ + metadata: PromptMetadata | null; + /** Content with tags removed */ + cleanContent: string; +} + +/** + * Extract metadata from content. + * + * @param content - Content that may contain tags + * @returns Object with metadata (or null) and cleaned content + * + * @example + * ```typescript + * const { metadata, cleanContent } = extractZeroEvalMetadata( + * '{"task":"test"}Hello world' + * ); + * // metadata = { task: "test" } + * // cleanContent = "Hello world" + * ``` + */ +export function extractZeroEvalMetadata(content: string): ExtractResult { + const match = content.match(ZEROEVAL_PATTERN); + + if (!match) { + return { metadata: null, cleanContent: content }; + } + + try { + const jsonStr = match[1].trim(); + const parsed = JSON.parse(jsonStr); + + // Validate that it's an object + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error('Metadata must be a JSON object'); + } + + // Validate required 'task' field + if (typeof parsed.task !== 'string') { + throw new Error('Metadata must have a "task" string field'); + } + + const metadata: PromptMetadata = { + task: parsed.task, + }; + + // Copy optional fields + if (parsed.variables && typeof parsed.variables === 'object') { + metadata.variables = parsed.variables; + } + if (typeof parsed.prompt_slug === 'string') { + metadata.prompt_slug = parsed.prompt_slug; + } + if (typeof parsed.prompt_version === 'number') { + metadata.prompt_version = parsed.prompt_version; + } + if (typeof parsed.prompt_version_id === 'string') { + metadata.prompt_version_id = parsed.prompt_version_id; + } + if (typeof parsed.content_hash === 'string') { + metadata.content_hash = parsed.content_hash; + } + + // Remove the tags from content (only first occurrence) + const cleanContent = content.replace(ZEROEVAL_PATTERN, '').trim(); + + return { metadata, cleanContent }; + } catch (error) { + // On parse error, return null metadata but keep original content + // This matches the Python SDK's behavior of raising but we'll be more lenient + return { metadata: null, cleanContent: content }; + } +} diff --git a/src/utils/template.ts b/src/utils/template.ts new file mode 100644 index 0000000..dc844bc --- /dev/null +++ b/src/utils/template.ts @@ -0,0 +1,97 @@ +/** + * Template utilities for variable interpolation. + * Ports the logic from zeroeval-sdk/src/zeroeval/template.py + */ + +import { PromptRequestError } from '../errors'; + +/** Pattern for valid identifier names */ +const IDENTIFIER_RE = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + +/** Pattern for {{variable}} with optional whitespace */ +const VARIABLE_PATTERN = /\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}/g; + +/** Escape placeholders for \{{ and \}} */ +const ESC_L = '__ZE_ESC_L__'; +const ESC_R = '__ZE_ESC_R__'; + +export interface RenderOptions { + /** + * How to handle missing variables: + * - "error": Throw PromptRequestError + * - "leave": Leave placeholder unchanged + */ + missing: 'error' | 'leave'; +} + +/** + * Render a template by interpolating {{variable}} placeholders with values. + * + * Supports: + * - Escaped braces: \{{ and \}} are preserved + * - Whitespace in placeholders: {{ var }} works + * - Missing variable handling via options.missing + * + * @param content - Template string with {{variable}} placeholders + * @param variables - Object mapping variable names to values + * @param options - Rendering options (default: { missing: 'error' }) + * @returns Rendered string with variables interpolated + */ +export function renderTemplate( + content: string, + variables: Record, + options: RenderOptions = { missing: 'error' } +): string { + if (options.missing !== 'error' && options.missing !== 'leave') { + throw new Error("missing must be 'error' or 'leave'"); + } + + // Validate variable keys early + for (const key of Object.keys(variables)) { + if (!IDENTIFIER_RE.test(key)) { + throw new Error(`Invalid variable name: ${key}`); + } + } + + // Handle escaped braces: \{{ and \}} + let tmp = content.replace(/\\{\\{/g, ESC_L).replace(/\\}\\}/g, ESC_R); + // Also handle literal \{{ in the source + tmp = tmp.replace(/\\\{\{/g, ESC_L).replace(/\\\}\}/g, ESC_R); + + // Replace {{variable}} with values + const rendered = tmp.replace(VARIABLE_PATTERN, (match, name: string) => { + if (name in variables) { + return String(variables[name]); + } + if (options.missing === 'error') { + throw new PromptRequestError(`Missing variable: ${name}`, null); + } + return `{{${name}}}`; + }); + + // Restore escaped braces + return rendered.replace(new RegExp(ESC_L, 'g'), '{{').replace(new RegExp(ESC_R, 'g'), '}}'); +} + +/** + * Extract all variable names from a template. + * + * @param content - Template string with {{variable}} placeholders + * @returns Set of variable names found in the template + */ +export function extractVariables(content: string): Set { + const names = new Set(); + + // Temporarily remove escaped braces + let tmp = content.replace(/\\{\\{/g, '').replace(/\\}\\}/g, ''); + tmp = tmp.replace(/\\\{\{/g, '').replace(/\\\}\}/g, ''); + + // Find all variable placeholders + let match: RegExpExecArray | null; + const pattern = /\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}/g; + while ((match = pattern.exec(tmp)) !== null) { + names.add(match[1]); + } + + return names; +} diff --git a/tests/prompt/cache.test.ts b/tests/prompt/cache.test.ts new file mode 100644 index 0000000..1273156 --- /dev/null +++ b/tests/prompt/cache.test.ts @@ -0,0 +1,165 @@ +import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest'; +import { TTLCache } from '../../src/utils/cache'; + +describe('TTLCache', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + describe('basic operations', () => { + it('should set and get value', () => { + const cache = new TTLCache(); + cache.set('key', 'value'); + expect(cache.get('key')).toBe('value'); + }); + + it('should return undefined for missing key', () => { + const cache = new TTLCache(); + expect(cache.get('missing')).toBeUndefined(); + }); + + it('should overwrite existing value', () => { + const cache = new TTLCache(); + cache.set('key', 'value1'); + cache.set('key', 'value2'); + expect(cache.get('key')).toBe('value2'); + }); + + it('should delete key', () => { + const cache = new TTLCache(); + cache.set('key', 'value'); + cache.delete('key'); + expect(cache.get('key')).toBeUndefined(); + }); + + it('should clear all entries', () => { + const cache = new TTLCache(); + cache.set('key1', 'value1'); + cache.set('key2', 'value2'); + cache.clear(); + expect(cache.get('key1')).toBeUndefined(); + expect(cache.get('key2')).toBeUndefined(); + }); + + it('should report size', () => { + const cache = new TTLCache(); + expect(cache.size).toBe(0); + cache.set('key1', 'value1'); + expect(cache.size).toBe(1); + cache.set('key2', 'value2'); + expect(cache.size).toBe(2); + }); + }); + + describe('TTL expiration', () => { + it('should return value before TTL expires', () => { + const cache = new TTLCache({ ttlMs: 1000 }); + cache.set('key', 'value'); + + vi.advanceTimersByTime(500); + expect(cache.get('key')).toBe('value'); + }); + + it('should return undefined after TTL expires', () => { + const cache = new TTLCache({ ttlMs: 1000 }); + cache.set('key', 'value'); + + vi.advanceTimersByTime(1001); + expect(cache.get('key')).toBeUndefined(); + }); + + it('should use default 60s TTL', () => { + const cache = new TTLCache(); + cache.set('key', 'value'); + + vi.advanceTimersByTime(59000); + expect(cache.get('key')).toBe('value'); + + vi.advanceTimersByTime(2000); + expect(cache.get('key')).toBeUndefined(); + }); + }); + + describe('LRU eviction', () => { + it('should evict oldest entry when maxSize exceeded', () => { + const cache = new TTLCache({ maxSize: 2 }); + cache.set('key1', 'value1'); + cache.set('key2', 'value2'); + cache.set('key3', 'value3'); + + expect(cache.get('key1')).toBeUndefined(); + expect(cache.get('key2')).toBe('value2'); + expect(cache.get('key3')).toBe('value3'); + }); + + it('should update LRU order on get', () => { + const cache = new TTLCache({ maxSize: 2 }); + cache.set('key1', 'value1'); + cache.set('key2', 'value2'); + + // Access key1 to make it most recently used + cache.get('key1'); + + // Add key3, should evict key2 (least recently used) + cache.set('key3', 'value3'); + + expect(cache.get('key1')).toBe('value1'); + expect(cache.get('key2')).toBeUndefined(); + expect(cache.get('key3')).toBe('value3'); + }); + + it('should use default maxSize of 512', () => { + const cache = new TTLCache(); + for (let i = 0; i < 600; i++) { + cache.set(i, `value${i}`); + } + + // First entries should be evicted + expect(cache.get(0)).toBeUndefined(); + expect(cache.get(87)).toBeUndefined(); + + // Later entries should still exist + expect(cache.get(599)).toBe('value599'); + }); + }); + + describe('has method', () => { + it('should return true for existing non-expired key', () => { + const cache = new TTLCache(); + cache.set('key', 'value'); + expect(cache.has('key')).toBe(true); + }); + + it('should return false for missing key', () => { + const cache = new TTLCache(); + expect(cache.has('missing')).toBe(false); + }); + + it('should return false for expired key', () => { + const cache = new TTLCache({ ttlMs: 1000 }); + cache.set('key', 'value'); + vi.advanceTimersByTime(1001); + expect(cache.has('key')).toBe(false); + }); + + it('should NOT update LRU order (pure query)', () => { + const cache = new TTLCache({ maxSize: 2 }); + cache.set('key1', 'value1'); + cache.set('key2', 'value2'); + + // has() should NOT promote key1 to most-recently-used + cache.has('key1'); + + // Add key3, should evict key1 (still least recently used) + cache.set('key3', 'value3'); + + expect(cache.get('key1')).toBeUndefined(); + expect(cache.get('key2')).toBe('value2'); + expect(cache.get('key3')).toBe('value3'); + }); + }); +}); diff --git a/tests/prompt/errors.test.ts b/tests/prompt/errors.test.ts new file mode 100644 index 0000000..c5e31ce --- /dev/null +++ b/tests/prompt/errors.test.ts @@ -0,0 +1,77 @@ +import { describe, it, expect } from 'vitest'; +import { PromptNotFoundError, PromptRequestError } from '../../src/errors'; + +describe('Prompt Errors', () => { + describe('PromptNotFoundError', () => { + it('should create error with slug only', () => { + const error = new PromptNotFoundError('my-prompt'); + + expect(error.name).toBe('PromptNotFoundError'); + expect(error.message).toBe('Prompt not found: my-prompt'); + expect(error.slug).toBe('my-prompt'); + expect(error.version).toBeUndefined(); + expect(error.tag).toBeUndefined(); + }); + + it('should create error with version', () => { + const error = new PromptNotFoundError('my-prompt', 5); + + expect(error.message).toBe('Prompt not found: my-prompt v5'); + expect(error.version).toBe(5); + }); + + it('should create error with tag', () => { + const error = new PromptNotFoundError('my-prompt', undefined, 'latest'); + + expect(error.message).toBe('Prompt not found: my-prompt tag=latest'); + expect(error.tag).toBe('latest'); + }); + + it('should create error with version and tag', () => { + const error = new PromptNotFoundError('my-prompt', 3, 'production'); + + expect(error.message).toBe('Prompt not found: my-prompt v3 tag=production'); + expect(error.version).toBe(3); + expect(error.tag).toBe('production'); + }); + + it('should be instanceof Error', () => { + const error = new PromptNotFoundError('test'); + + expect(error).toBeInstanceOf(Error); + expect(error).toBeInstanceOf(PromptNotFoundError); + }); + }); + + describe('PromptRequestError', () => { + it('should create error with message and status', () => { + const error = new PromptRequestError('Request failed', 500); + + expect(error.name).toBe('PromptRequestError'); + expect(error.message).toBe('Request failed'); + expect(error.status).toBe(500); + expect(error.response).toBeUndefined(); + }); + + it('should create error with null status', () => { + const error = new PromptRequestError('Network error', null); + + expect(error.status).toBeNull(); + }); + + it('should create error with response', () => { + const responseBody = { error: 'Invalid request' }; + const error = new PromptRequestError('Bad request', 400, responseBody); + + expect(error.status).toBe(400); + expect(error.response).toEqual({ error: 'Invalid request' }); + }); + + it('should be instanceof Error', () => { + const error = new PromptRequestError('test', null); + + expect(error).toBeInstanceOf(Error); + expect(error).toBeInstanceOf(PromptRequestError); + }); + }); +}); diff --git a/tests/prompt/hash.test.ts b/tests/prompt/hash.test.ts new file mode 100644 index 0000000..4a705ac --- /dev/null +++ b/tests/prompt/hash.test.ts @@ -0,0 +1,85 @@ +import { describe, it, expect } from 'vitest'; +import { normalizePromptText, sha256Hex } from '../../src/utils/hash'; + +describe('hash utilities', () => { + describe('normalizePromptText', () => { + it('should normalize CRLF to LF', () => { + const input = 'line1\r\nline2\r\nline3'; + const result = normalizePromptText(input); + expect(result).toBe('line1\nline2\nline3'); + }); + + it('should normalize CR to LF', () => { + const input = 'line1\rline2\rline3'; + const result = normalizePromptText(input); + expect(result).toBe('line1\nline2\nline3'); + }); + + it('should strip trailing whitespace from each line', () => { + const input = 'line1 \nline2 \nline3'; + const result = normalizePromptText(input); + expect(result).toBe('line1\nline2\nline3'); + }); + + it('should preserve leading whitespace', () => { + const input = ' line1\n line2\nline3'; + const result = normalizePromptText(input); + expect(result).toBe('line1\n line2\nline3'); + }); + + it('should strip overall leading and trailing whitespace', () => { + const input = ' \n content \n '; + const result = normalizePromptText(input); + expect(result).toBe('content'); + }); + + it('should preserve {{variable}} tokens', () => { + const input = 'Hello {{name}}, your score is {{score}}.'; + const result = normalizePromptText(input); + expect(result).toBe('Hello {{name}}, your score is {{score}}.'); + }); + + it('should handle empty string', () => { + const result = normalizePromptText(''); + expect(result).toBe(''); + }); + + it('should convert non-string to string', () => { + const result = normalizePromptText(123 as unknown as string); + expect(result).toBe('123'); + }); + }); + + describe('sha256Hex', () => { + it('should return 64-character hex hash', async () => { + const result = await sha256Hex('test content'); + expect(result).toHaveLength(64); + expect(result).toMatch(/^[0-9a-f]{64}$/); + }); + + it('should return consistent hash for same input', async () => { + const result1 = await sha256Hex('hello world'); + const result2 = await sha256Hex('hello world'); + expect(result1).toBe(result2); + }); + + it('should return different hash for different input', async () => { + const result1 = await sha256Hex('content1'); + const result2 = await sha256Hex('content2'); + expect(result1).not.toBe(result2); + }); + + it('should normalize before hashing', async () => { + // Same content with different line endings should have same hash + const result1 = await sha256Hex('line1\nline2'); + const result2 = await sha256Hex('line1\r\nline2'); + expect(result1).toBe(result2); + }); + + it('should handle empty string', async () => { + const result = await sha256Hex(''); + expect(result).toHaveLength(64); + expect(result).toMatch(/^[0-9a-f]{64}$/); + }); + }); +}); diff --git a/tests/prompt/metadata.test.ts b/tests/prompt/metadata.test.ts new file mode 100644 index 0000000..44a7006 --- /dev/null +++ b/tests/prompt/metadata.test.ts @@ -0,0 +1,181 @@ +import { describe, it, expect } from 'vitest'; +import { decoratePrompt, extractZeroEvalMetadata } from '../../src/utils/metadata'; +import type { PromptMetadata } from '../../src/types/prompt'; + +describe('metadata utilities', () => { + describe('decoratePrompt', () => { + it('should decorate prompt with task metadata', () => { + const metadata: PromptMetadata = { task: 'test-task' }; + const result = decoratePrompt('Hello world', metadata); + + expect(result).toBe('{"task":"test-task"}Hello world'); + }); + + it('should include variables in metadata', () => { + const metadata: PromptMetadata = { + task: 'test-task', + variables: { name: 'John', role: 'admin' }, + }; + const result = decoratePrompt('Content', metadata); + + expect(result).toContain('"variables":{"name":"John","role":"admin"}'); + expect(result).toContain('"task":"test-task"'); + }); + + it('should include prompt_slug', () => { + const metadata: PromptMetadata = { + task: 'test-task', + prompt_slug: 'customer-support', + }; + const result = decoratePrompt('Content', metadata); + + expect(result).toContain('"prompt_slug":"customer-support"'); + }); + + it('should include prompt_version', () => { + const metadata: PromptMetadata = { + task: 'test-task', + prompt_version: 5, + }; + const result = decoratePrompt('Content', metadata); + + expect(result).toContain('"prompt_version":5'); + }); + + it('should include prompt_version_id', () => { + const metadata: PromptMetadata = { + task: 'test-task', + prompt_version_id: 'uuid-123-456', + }; + const result = decoratePrompt('Content', metadata); + + expect(result).toContain('"prompt_version_id":"uuid-123-456"'); + }); + + it('should include content_hash', () => { + const metadata: PromptMetadata = { + task: 'test-task', + content_hash: 'abc123def456', + }; + const result = decoratePrompt('Content', metadata); + + expect(result).toContain('"content_hash":"abc123def456"'); + }); + + it('should not include empty variables', () => { + const metadata: PromptMetadata = { + task: 'test-task', + variables: {}, + }; + const result = decoratePrompt('Content', metadata); + + expect(result).not.toContain('variables'); + }); + }); + + describe('extractZeroEvalMetadata', () => { + it('should extract metadata from decorated content', () => { + const content = '{"task":"test-task"}Hello world'; + const { metadata, cleanContent } = extractZeroEvalMetadata(content); + + expect(metadata).toEqual({ task: 'test-task' }); + expect(cleanContent).toBe('Hello world'); + }); + + it('should extract variables', () => { + const content = + '{"task":"test","variables":{"name":"John"}}Content'; + const { metadata } = extractZeroEvalMetadata(content); + + expect(metadata?.variables).toEqual({ name: 'John' }); + }); + + it('should extract optional fields', () => { + const content = + '{"task":"test","prompt_slug":"slug","prompt_version":3,"prompt_version_id":"id-123","content_hash":"hash"}Content'; + const { metadata } = extractZeroEvalMetadata(content); + + expect(metadata?.prompt_slug).toBe('slug'); + expect(metadata?.prompt_version).toBe(3); + expect(metadata?.prompt_version_id).toBe('id-123'); + expect(metadata?.content_hash).toBe('hash'); + }); + + it('should return null metadata for content without tags', () => { + const content = 'Plain content without tags'; + const { metadata, cleanContent } = extractZeroEvalMetadata(content); + + expect(metadata).toBeNull(); + expect(cleanContent).toBe('Plain content without tags'); + }); + + it('should return null metadata for invalid JSON', () => { + const content = '{invalid json}Content'; + const { metadata, cleanContent } = extractZeroEvalMetadata(content); + + expect(metadata).toBeNull(); + expect(cleanContent).toBe('{invalid json}Content'); + }); + + it('should return null metadata for non-object JSON', () => { + const content = '"string"Content'; + const { metadata, cleanContent } = extractZeroEvalMetadata(content); + + expect(metadata).toBeNull(); + expect(cleanContent).toBe('"string"Content'); + }); + + it('should return null metadata for array JSON', () => { + const content = '[1, 2, 3]Content'; + const { metadata, cleanContent } = extractZeroEvalMetadata(content); + + expect(metadata).toBeNull(); + expect(cleanContent).toBe('[1, 2, 3]Content'); + }); + + it('should handle multiline content', () => { + const content = + '{"task":"test"}Line 1\nLine 2\nLine 3'; + const { metadata, cleanContent } = extractZeroEvalMetadata(content); + + expect(metadata?.task).toBe('test'); + expect(cleanContent).toBe('Line 1\nLine 2\nLine 3'); + }); + + it('should only extract first occurrence of tags', () => { + const content = + '{"task":"first"}Content{"task":"second"}'; + const { metadata, cleanContent } = extractZeroEvalMetadata(content); + + expect(metadata?.task).toBe('first'); + expect(cleanContent).toBe('Content{"task":"second"}'); + }); + + it('should trim clean content', () => { + const content = '{"task":"test"} Content with spaces '; + const { cleanContent } = extractZeroEvalMetadata(content); + + expect(cleanContent).toBe('Content with spaces'); + }); + }); + + describe('roundtrip', () => { + it('should preserve metadata through decorate and extract', () => { + const originalMetadata: PromptMetadata = { + task: 'customer-support', + variables: { tone: 'friendly', product: 'Widget' }, + prompt_slug: 'support-v2', + prompt_version: 7, + prompt_version_id: 'uuid-abc-123', + content_hash: 'hash123', + }; + const originalContent = 'You are a helpful assistant.'; + + const decorated = decoratePrompt(originalContent, originalMetadata); + const { metadata, cleanContent } = extractZeroEvalMetadata(decorated); + + expect(metadata).toEqual(originalMetadata); + expect(cleanContent).toBe(originalContent); + }); + }); +}); diff --git a/tests/prompt/template.test.ts b/tests/prompt/template.test.ts new file mode 100644 index 0000000..7ae7ab0 --- /dev/null +++ b/tests/prompt/template.test.ts @@ -0,0 +1,109 @@ +import { describe, it, expect } from 'vitest'; +import { renderTemplate, extractVariables } from '../../src/utils/template'; +import { PromptRequestError } from '../../src/errors'; + +describe('template utilities', () => { + describe('renderTemplate', () => { + it('should interpolate single variable', () => { + const result = renderTemplate('Hello {{name}}!', { name: 'World' }); + expect(result).toBe('Hello World!'); + }); + + it('should interpolate multiple variables', () => { + const result = renderTemplate('{{greeting}} {{name}}!', { + greeting: 'Hello', + name: 'World', + }); + expect(result).toBe('Hello World!'); + }); + + it('should handle whitespace in variable placeholders', () => { + const result = renderTemplate('Hello {{ name }}!', { name: 'World' }); + expect(result).toBe('Hello World!'); + }); + + it('should handle variables with underscores', () => { + const result = renderTemplate('User: {{user_name}}', { user_name: 'John' }); + expect(result).toBe('User: John'); + }); + + it('should handle numeric values', () => { + const result = renderTemplate('Score: {{score}}', { score: 95 }); + expect(result).toBe('Score: 95'); + }); + + it('should handle boolean values', () => { + const result = renderTemplate('Active: {{active}}', { active: true }); + expect(result).toBe('Active: true'); + }); + + it('should throw on missing variable with error mode', () => { + expect(() => + renderTemplate('Hello {{name}}!', {}, { missing: 'error' }) + ).toThrow(PromptRequestError); + }); + + it('should leave placeholder on missing variable with leave mode', () => { + const result = renderTemplate('Hello {{name}}!', {}, { missing: 'leave' }); + expect(result).toBe('Hello {{name}}!'); + }); + + it('should preserve escaped braces', () => { + const result = renderTemplate('Use \\{{variable}} syntax', { variable: 'test' }); + expect(result).toBe('Use {{variable}} syntax'); + }); + + it('should handle empty variables object', () => { + const result = renderTemplate('No variables here', {}, { missing: 'leave' }); + expect(result).toBe('No variables here'); + }); + + it('should throw on invalid variable name', () => { + expect(() => renderTemplate('{{name}}', { '123invalid': 'value' })).toThrow( + 'Invalid variable name' + ); + }); + + it('should handle content with no variables', () => { + const result = renderTemplate('Plain text content', { name: 'unused' }); + expect(result).toBe('Plain text content'); + }); + }); + + describe('extractVariables', () => { + it('should extract single variable', () => { + const result = extractVariables('Hello {{name}}!'); + expect(result).toEqual(new Set(['name'])); + }); + + it('should extract multiple variables', () => { + const result = extractVariables('{{greeting}} {{name}}!'); + expect(result).toEqual(new Set(['greeting', 'name'])); + }); + + it('should handle duplicate variables', () => { + const result = extractVariables('{{name}} and {{name}} again'); + expect(result).toEqual(new Set(['name'])); + }); + + it('should ignore escaped braces', () => { + const result = extractVariables('\\{{escaped}} but {{real}}'); + expect(result).toEqual(new Set(['real'])); + }); + + it('should handle whitespace in placeholders', () => { + const result = extractVariables('{{ name }} and {{ other }}'); + expect(result).toEqual(new Set(['name', 'other'])); + }); + + it('should return empty set for no variables', () => { + const result = extractVariables('No variables here'); + expect(result).toEqual(new Set()); + }); + + it('should handle underscores in variable names', () => { + const result = extractVariables('{{user_name}} and {{_private}}'); + expect(result).toEqual(new Set(['user_name', '_private'])); + }); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index 4b7e3fa..7b560ed 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -34,22 +34,16 @@ export default defineConfig({ singleThread: true, // Important for testing AsyncLocalStorage }, }, - deps: { - optimizer: { - web: { - enabled: false, - }, - ssr: { - enabled: false, - }, + server: { + deps: { + external: [ + 'langchain', + '@langchain/core', + 'openai', + 'ai', + '@ai-sdk/openai', + ], }, - external: [ - 'langchain', - '@langchain/core', - 'openai', - 'ai', - '@ai-sdk/openai', - ], }, }, resolve: {