diff --git a/agent-server/nodejs/src/api-server.js b/agent-server/nodejs/src/api-server.js index e8d6e57f37..3773fa6357 100644 --- a/agent-server/nodejs/src/api-server.js +++ b/agent-server/nodejs/src/api-server.js @@ -548,8 +548,11 @@ class APIServer { await new Promise(resolve => setTimeout(resolve, waitTimeout)); } + // Extract tracing metadata for Langfuse integration + const tracingMetadata = requestBody.metadata || {}; + // Create a dynamic request for this request - const request = this.createDynamicRequestNested(requestBody.input, nestedModelConfig); + const request = this.createDynamicRequestNested(requestBody.input, nestedModelConfig, tracingMetadata); // Execute the request on the new tab's DevTools client logger.info('Executing request on new tab', { @@ -581,29 +584,35 @@ class APIServer { */ processNestedModelConfig(requestBody) { const defaults = this.configDefaults?.model || {}; + // Default LiteLLM endpoint from environment variable + const defaultLiteLLMEndpoint = process.env.LITELLM_ENDPOINT; // If nested format is provided, use it directly with fallbacks if (requestBody.model) { - // Extract endpoint from each model tier, with fallback chain: + // Helper to get endpoint with fallback chain: // 1. Try tier-specific endpoint (e.g., main_model.endpoint) // 2. Fall back to top-level endpoint (e.g., model.endpoint) - // 3. Fall back to undefined (will use env var later) - const mainEndpoint = requestBody.model.main_model?.endpoint || requestBody.model.endpoint; - const miniEndpoint = requestBody.model.mini_model?.endpoint || requestBody.model.endpoint; - const nanoEndpoint = requestBody.model.nano_model?.endpoint || requestBody.model.endpoint; + // 3. Fall back to LITELLM_ENDPOINT env var (for litellm provider) + const getEndpoint = (tierConfig) => { + const explicitEndpoint = tierConfig?.endpoint || requestBody.model.endpoint; + if (explicitEndpoint) return explicitEndpoint; + // Use env var default for litellm provider + if (tierConfig?.provider === 'litellm') return defaultLiteLLMEndpoint; + return undefined; + }; return { main_model: { ...this.extractModelTierConfig('main', requestBody.model.main_model, defaults), - endpoint: mainEndpoint + endpoint: getEndpoint(requestBody.model.main_model) }, mini_model: { ...this.extractModelTierConfig('mini', requestBody.model.mini_model, defaults), - endpoint: miniEndpoint + endpoint: getEndpoint(requestBody.model.mini_model) }, nano_model: { ...this.extractModelTierConfig('nano', requestBody.model.nano_model, defaults), - endpoint: nanoEndpoint + endpoint: getEndpoint(requestBody.model.nano_model) } }; } @@ -633,10 +642,15 @@ class APIServer { // If it's an object with provider/model/api_key, extract those fields if (typeof tierConfig === 'object' && tierConfig.provider) { + // Get API key with fallback for litellm provider + let apiKey = tierConfig.api_key; + if (!apiKey && tierConfig.provider === 'litellm') { + apiKey = process.env.LITELLM_API_KEY; + } return { provider: tierConfig.provider, model: tierConfig.model, - api_key: tierConfig.api_key + api_key: apiKey // endpoint will be added by caller }; } @@ -750,9 +764,10 @@ class APIServer { * Create a dynamic evaluation object with nested model configuration * @param {string|Array<{role: string, content: string}>} input - Input message (string) or conversation array (OpenAI Responses API format) * @param {import('./types/model-config').ModelConfig} nestedModelConfig - Model configuration + * @param {Object} tracingMetadata - Optional tracing metadata for Langfuse integration * @returns {import('./types/model-config').EvaluationRequest} Evaluation request object */ - createDynamicRequestNested(input, nestedModelConfig) { + createDynamicRequestNested(input, nestedModelConfig, tracingMetadata = {}) { const requestId = `api-req-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`; // Determine input format and structure accordingly @@ -792,7 +807,10 @@ class APIServer { tags: ['api', 'dynamic'], priority: 'high', source: 'api' - } + }, + // Tracing metadata for Langfuse integration + // Contains session_id, trace_id, eval_id, etc. from eval framework + tracing: tracingMetadata }; } diff --git a/agent-server/nodejs/src/lib/BrowserAgentServer.js b/agent-server/nodejs/src/lib/BrowserAgentServer.js index 6ac442712c..2d0c7a7458 100644 --- a/agent-server/nodejs/src/lib/BrowserAgentServer.js +++ b/agent-server/nodejs/src/lib/BrowserAgentServer.js @@ -730,11 +730,19 @@ export class BrowserAgentServer extends EventEmitter { metadata: { tags: request.metadata?.tags || [], retries: request.settings?.retry_policy?.max_retries || 0 - } + }, + // Forward tracing metadata for Langfuse session grouping + tracing: request.tracing || {} }, id: rpcId }; + logger.debug('RPC request prepared with tracing:', { + hasTracing: !!request.tracing, + tracingKeys: request.tracing ? Object.keys(request.tracing) : [], + sessionId: request.tracing?.session_id + }); + // Send RPC request const response = await connection.rpcClient.callMethod( connection.ws, diff --git a/front_end/panels/ai_chat/LLM/LLMClient.ts b/front_end/panels/ai_chat/LLM/LLMClient.ts index 59e1a30465..67b432697d 100644 --- a/front_end/panels/ai_chat/LLM/LLMClient.ts +++ b/front_end/panels/ai_chat/LLM/LLMClient.ts @@ -17,6 +17,7 @@ import { GenericOpenAIProvider } from './GenericOpenAIProvider.js'; import { CustomProviderManager } from '../core/CustomProviderManager.js'; import { LLMResponseParser } from './LLMResponseParser.js'; import { createLogger } from '../core/Logger.js'; +import { getCurrentTracingContext } from '../tracing/TracingConfig.js'; const logger = createLogger('LLMClient'); @@ -48,6 +49,7 @@ export interface LLMCallRequest { temperature?: number; retryConfig?: Partial; agentName?: string; // Name of the calling agent for provider-specific routing + tracingMetadata?: Record; // Explicit tracing metadata for Langfuse integration } /** @@ -207,6 +209,38 @@ export class LLMClient { options.agentName = (request as any).agentName; } + // Get tracing metadata - prefer explicit request metadata over global context + // This ensures metadata flows correctly even when async context is lost + let tracingMetadata = request.tracingMetadata; + + if (!tracingMetadata || Object.keys(tracingMetadata).length === 0) { + // Fall back to global tracing context + const tracingContext = getCurrentTracingContext(); + logger.info('LLMClient.call() - Checking tracing context (fallback):', { + hasContext: !!tracingContext, + hasMetadata: !!tracingContext?.metadata, + metadataKeys: tracingContext?.metadata ? Object.keys(tracingContext.metadata) : [], + sessionId: tracingContext?.metadata?.session_id, + traceId: tracingContext?.metadata?.trace_id + }); + if (tracingContext?.metadata && Object.keys(tracingContext.metadata).length > 0) { + tracingMetadata = tracingContext.metadata; + } + } else { + logger.info('LLMClient.call() - Using explicit tracingMetadata from request:', { + metadataKeys: Object.keys(tracingMetadata), + sessionId: tracingMetadata.session_id, + traceId: tracingMetadata.trace_id + }); + } + + if (tracingMetadata && Object.keys(tracingMetadata).length > 0) { + options.tracingMetadata = tracingMetadata; + logger.info('Passing tracing metadata to provider:', tracingMetadata); + } else { + logger.info('No tracing metadata available'); + } + return provider.callWithMessages(request.model, messages, options); } diff --git a/front_end/panels/ai_chat/LLM/LLMTypes.ts b/front_end/panels/ai_chat/LLM/LLMTypes.ts index b9197ff0c8..51dbb8df88 100644 --- a/front_end/panels/ai_chat/LLM/LLMTypes.ts +++ b/front_end/panels/ai_chat/LLM/LLMTypes.ts @@ -201,6 +201,14 @@ export interface LLMCallOptions { reasoningLevel?: 'low' | 'medium' | 'high'; // For O-series models retryConfig?: Partial; agentName?: string; // Name of the calling agent for provider-specific routing + // Tracing metadata for Langfuse integration via LiteLLM + tracingMetadata?: { + session_id?: string; + trace_id?: string; + generation_name?: string; + tags?: string[]; + [key: string]: any; + }; } /** diff --git a/front_end/panels/ai_chat/LLM/LiteLLMProvider.ts b/front_end/panels/ai_chat/LLM/LiteLLMProvider.ts index 1c58da7e08..8f5eeb45b1 100644 --- a/front_end/panels/ai_chat/LLM/LiteLLMProvider.ts +++ b/front_end/panels/ai_chat/LLM/LiteLLMProvider.ts @@ -209,6 +209,12 @@ export class LiteLLMProvider extends LLMBaseProvider { payloadBody.tool_choice = options.tool_choice; } + // Add tracing metadata for Langfuse integration + // LiteLLM forwards this to Langfuse callbacks + if (options?.tracingMetadata) { + payloadBody.metadata = options.tracingMetadata; + } + logger.info('Request payload:', payloadBody); const data = await this.makeAPIRequest(payloadBody); diff --git a/front_end/panels/ai_chat/agent_framework/AgentRunner.ts b/front_end/panels/ai_chat/agent_framework/AgentRunner.ts index 7e219f99e5..bf2971987b 100644 --- a/front_end/panels/ai_chat/agent_framework/AgentRunner.ts +++ b/front_end/panels/ai_chat/agent_framework/AgentRunner.ts @@ -749,6 +749,8 @@ export class AgentRunner { tools: toolSchemas, temperature: temperature ?? 0, agentName: agentName, // Pass agent identity for provider-specific routing + // Pass tracing metadata explicitly for Langfuse integration + tracingMetadata: tracingContext?.metadata, }); // Complete the generation observation diff --git a/front_end/panels/ai_chat/core/AgentNodes.ts b/front_end/panels/ai_chat/core/AgentNodes.ts index 7253289bc4..6ce3ff0919 100644 --- a/front_end/panels/ai_chat/core/AgentNodes.ts +++ b/front_end/panels/ai_chat/core/AgentNodes.ts @@ -239,6 +239,8 @@ export function createAgentNode(modelName: string, provider: LLMProvider, temper })), temperature: this.temperature, agentName: agentName, + // Pass tracing metadata explicitly from state context for Langfuse integration + tracingMetadata: state.context?.tracingContext?.metadata, }); // Parse the response diff --git a/front_end/panels/ai_chat/core/AgentService.ts b/front_end/panels/ai_chat/core/AgentService.ts index 5b62167933..01c3487c5c 100644 --- a/front_end/panels/ai_chat/core/AgentService.ts +++ b/front_end/panels/ai_chat/core/AgentService.ts @@ -276,15 +276,15 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{ }; } - // Default: provider requires apiKey - if (!apiKey) { + // Default: provider requires apiKey (unless in AUTOMATED_MODE where keys come dynamically) + if (!apiKey && !BUILD_CONFIG.AUTOMATED_MODE) { logger.warn(`Provider ${provider} requires API key`); return null; } return { provider, - apiKey + apiKey: apiKey || '' // Default to empty string for AUTOMATED_MODE }; } @@ -597,7 +597,9 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{ tracingContext: { sessionId: existingContext?.sessionId || this.#sessionId, traceId, - parentObservationId: parentObservationId + parentObservationId: parentObservationId, + // Forward metadata from evaluation context for Langfuse session grouping + metadata: existingContext?.metadata }, executionId: this.#executionId, abortSignal: this.#abortController?.signal, diff --git a/front_end/panels/ai_chat/evaluation/EvaluationAgent.ts b/front_end/panels/ai_chat/evaluation/EvaluationAgent.ts index b152fa2f4b..395d8a8141 100644 --- a/front_end/panels/ai_chat/evaluation/EvaluationAgent.ts +++ b/front_end/panels/ai_chat/evaluation/EvaluationAgent.ts @@ -321,13 +321,23 @@ export class EvaluationAgent { tool: params.tool }); - // Create a trace for this evaluation - const traceId = `eval-${params.evaluationId}-${Date.now()}`; - const sessionId = `eval-session-${Date.now()}`; - const tracingContext: TracingContext = { - traceId, + // Use tracing metadata from request params (from eval framework via api-server) + // The tracing field is now properly forwarded through BrowserAgentServer.executeRequest() + const requestTracing = params.tracing || {}; + logger.info('Tracing metadata received:', { + hasTracing: !!params.tracing, + tracingKeys: Object.keys(requestTracing), + sessionId: requestTracing.session_id, + traceId: requestTracing.trace_id + }); + const traceId = requestTracing.trace_id || `eval-${params.evaluationId}-${Date.now()}`; + const sessionId = requestTracing.session_id || `eval-session-${Date.now()}`; + const tracingContext: TracingContext = { + traceId, sessionId, - parentObservationId: undefined + parentObservationId: undefined, + // Include full tracing metadata for LLM calls + metadata: requestTracing }; try { diff --git a/front_end/panels/ai_chat/evaluation/EvaluationProtocol.ts b/front_end/panels/ai_chat/evaluation/EvaluationProtocol.ts index c21b24e43e..40ea10078e 100644 --- a/front_end/panels/ai_chat/evaluation/EvaluationProtocol.ts +++ b/front_end/panels/ai_chat/evaluation/EvaluationProtocol.ts @@ -91,6 +91,16 @@ export interface EvaluationParams { retries: number; priority?: 'low' | 'normal' | 'high'; }; + // Tracing metadata for Langfuse session grouping + tracing?: { + session_id?: string; + trace_id?: string; + eval_id?: string; + eval_name?: string; + category?: string; + tags?: string[]; + [key: string]: any; + }; } export interface EvaluationSuccessResponse { diff --git a/front_end/panels/ai_chat/evaluation/remote/EvaluationAgent.ts b/front_end/panels/ai_chat/evaluation/remote/EvaluationAgent.ts index 921aa32635..04f357fde3 100644 --- a/front_end/panels/ai_chat/evaluation/remote/EvaluationAgent.ts +++ b/front_end/panels/ai_chat/evaluation/remote/EvaluationAgent.ts @@ -426,13 +426,25 @@ export class EvaluationAgent { tool: params.tool }); - // Create a trace for this evaluation - const traceId = `eval-${params.evaluationId}-${Date.now()}`; - const sessionId = `eval-session-${Date.now()}`; - const tracingContext: TracingContext = { - traceId, + // Use tracing metadata from request params (from eval framework via api-server) + // The tracing field is now properly forwarded through BrowserAgentServer.executeRequest() + const requestTracing = params.tracing || {}; + logger.info('Tracing metadata received:', { + hasTracing: !!params.tracing, + tracingKeys: Object.keys(requestTracing), + sessionId: requestTracing.session_id, + traceId: requestTracing.trace_id + }); + + // Create a trace for this evaluation - use tracing from request if available + const traceId = requestTracing.trace_id || `eval-${params.evaluationId}-${Date.now()}`; + const sessionId = requestTracing.session_id || `eval-session-${Date.now()}`; + const tracingContext: TracingContext = { + traceId, sessionId, - parentObservationId: undefined + parentObservationId: undefined, + // Include full tracing metadata for LLM calls (session grouping in Langfuse) + metadata: requestTracing }; const orchestratorDescriptor = await this.orchestratorDescriptorPromise; diff --git a/front_end/panels/ai_chat/evaluation/remote/EvaluationProtocol.ts b/front_end/panels/ai_chat/evaluation/remote/EvaluationProtocol.ts index b84425a7a7..dbf100b509 100644 --- a/front_end/panels/ai_chat/evaluation/remote/EvaluationProtocol.ts +++ b/front_end/panels/ai_chat/evaluation/remote/EvaluationProtocol.ts @@ -104,6 +104,16 @@ export interface EvaluationParams { retries: number; priority?: 'low' | 'normal' | 'high'; }; + // Tracing metadata for Langfuse session grouping + tracing?: { + session_id?: string; + trace_id?: string; + eval_id?: string; + eval_name?: string; + category?: string; + tags?: string[]; + trace_name?: string; + }; } export interface EvaluationSuccessResponse { diff --git a/front_end/panels/ai_chat/tools/LLMTracingWrapper.ts b/front_end/panels/ai_chat/tools/LLMTracingWrapper.ts index 88122b5481..1143f19969 100644 --- a/front_end/panels/ai_chat/tools/LLMTracingWrapper.ts +++ b/front_end/panels/ai_chat/tools/LLMTracingWrapper.ts @@ -85,6 +85,8 @@ export async function callLLMWithTracing( messages: llmCallConfig.messages, systemPrompt: llmCallConfig.systemPrompt || '', temperature: llmCallConfig.temperature, + // Pass tracing metadata explicitly for Langfuse integration + tracingMetadata: tracingContext?.metadata, ...llmCallConfig.options }); diff --git a/front_end/panels/ai_chat/tracing/TracingProvider.ts b/front_end/panels/ai_chat/tracing/TracingProvider.ts index cd7f81b274..129eea56a5 100644 --- a/front_end/panels/ai_chat/tracing/TracingProvider.ts +++ b/front_end/panels/ai_chat/tracing/TracingProvider.ts @@ -18,6 +18,16 @@ export interface TracingContext { agentType: string; iterationCount?: number; }; + // Tracing metadata from eval framework for Langfuse integration + metadata?: { + session_id?: string; + trace_id?: string; + eval_id?: string; + eval_name?: string; + category?: string; + tags?: string[]; + [key: string]: any; + }; } export interface TraceMetadata {