Added tracing metadata for LiteLLM provider (#86)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

## Release Notes

* **New Features**
* Added comprehensive tracing and observability support for LLM calls
and agent evaluations, enabling enhanced session grouping and request
tracking.
* Implemented automatic tracing metadata propagation throughout the API
request and evaluation pipeline.

* **Improvements**
* Enhanced configuration resilience with automatic fallbacks for model
endpoints and credentials.

<sub>✏️ Tip: You can customize this high-level summary in your review
settings.</sub>

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Co-authored-by: Oleh Luchkiv <olesho@gmail.com>

Author: Daedaelius

agent-server/nodejs/src/api-server.js

@@ -548,8 +548,11 @@ class APIServer {
         await new Promise(resolve => setTimeout(resolve, waitTimeout));
       }
 
+      // Extract tracing metadata for Langfuse integration
+      const tracingMetadata = requestBody.metadata || {};
+
       // Create a dynamic request for this request
-      const request = this.createDynamicRequestNested(requestBody.input, nestedModelConfig);
+      const request = this.createDynamicRequestNested(requestBody.input, nestedModelConfig, tracingMetadata);
 
       // Execute the request on the new tab's DevTools client
       logger.info('Executing request on new tab', {
@@ -581,29 +584,35 @@ class APIServer {
    */
   processNestedModelConfig(requestBody) {
     const defaults = this.configDefaults?.model || {};
+    // Default LiteLLM endpoint from environment variable
+    const defaultLiteLLMEndpoint = process.env.LITELLM_ENDPOINT;
 
     // If nested format is provided, use it directly with fallbacks
     if (requestBody.model) {
-      // Extract endpoint from each model tier, with fallback chain:
+      // Helper to get endpoint with fallback chain:
       // 1. Try tier-specific endpoint (e.g., main_model.endpoint)
       // 2. Fall back to top-level endpoint (e.g., model.endpoint)
-      // 3. Fall back to undefined (will use env var later)
-      const mainEndpoint = requestBody.model.main_model?.endpoint || requestBody.model.endpoint;
-      const miniEndpoint = requestBody.model.mini_model?.endpoint || requestBody.model.endpoint;
-      const nanoEndpoint = requestBody.model.nano_model?.endpoint || requestBody.model.endpoint;
+      // 3. Fall back to LITELLM_ENDPOINT env var (for litellm provider)
+      const getEndpoint = (tierConfig) => {
+        const explicitEndpoint = tierConfig?.endpoint || requestBody.model.endpoint;
+        if (explicitEndpoint) return explicitEndpoint;
+        // Use env var default for litellm provider
+        if (tierConfig?.provider === 'litellm') return defaultLiteLLMEndpoint;
+        return undefined;
+      };
 
       return {
         main_model: {
           ...this.extractModelTierConfig('main', requestBody.model.main_model, defaults),
-          endpoint: mainEndpoint
+          endpoint: getEndpoint(requestBody.model.main_model)
         },
         mini_model: {
           ...this.extractModelTierConfig('mini', requestBody.model.mini_model, defaults),
-          endpoint: miniEndpoint
+          endpoint: getEndpoint(requestBody.model.mini_model)
         },
         nano_model: {
           ...this.extractModelTierConfig('nano', requestBody.model.nano_model, defaults),
-          endpoint: nanoEndpoint
+          endpoint: getEndpoint(requestBody.model.nano_model)
         }
       };
     }
@@ -633,10 +642,15 @@ class APIServer {
 
     // If it's an object with provider/model/api_key, extract those fields
     if (typeof tierConfig === 'object' && tierConfig.provider) {
+      // Get API key with fallback for litellm provider
+      let apiKey = tierConfig.api_key;
+      if (!apiKey && tierConfig.provider === 'litellm') {
+        apiKey = process.env.LITELLM_API_KEY;
+      }
       return {
         provider: tierConfig.provider,
         model: tierConfig.model,
-        api_key: tierConfig.api_key
+        api_key: apiKey
         // endpoint will be added by caller
       };
     }
@@ -750,9 +764,10 @@ class APIServer {
    * Create a dynamic evaluation object with nested model configuration
    * @param {string|Array<{role: string, content: string}>} input - Input message (string) or conversation array (OpenAI Responses API format)
    * @param {import('./types/model-config').ModelConfig} nestedModelConfig - Model configuration
+   * @param {Object} tracingMetadata - Optional tracing metadata for Langfuse integration
    * @returns {import('./types/model-config').EvaluationRequest} Evaluation request object
    */
-  createDynamicRequestNested(input, nestedModelConfig) {
+  createDynamicRequestNested(input, nestedModelConfig, tracingMetadata = {}) {
     const requestId = `api-req-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`;
 
     // Determine input format and structure accordingly
@@ -792,7 +807,10 @@ class APIServer {
         tags: ['api', 'dynamic'],
         priority: 'high',
         source: 'api'
-      }
+      },
+      // Tracing metadata for Langfuse integration
+      // Contains session_id, trace_id, eval_id, etc. from eval framework
+      tracing: tracingMetadata
     };
   }
 

agent-server/nodejs/src/lib/BrowserAgentServer.js

@@ -730,11 +730,19 @@ export class BrowserAgentServer extends EventEmitter {
           metadata: {
             tags: request.metadata?.tags || [],
             retries: request.settings?.retry_policy?.max_retries || 0
-          }
+          },
+          // Forward tracing metadata for Langfuse session grouping
+          tracing: request.tracing || {}
         },
         id: rpcId
       };
 
+      logger.debug('RPC request prepared with tracing:', {
+        hasTracing: !!request.tracing,
+        tracingKeys: request.tracing ? Object.keys(request.tracing) : [],
+        sessionId: request.tracing?.session_id
+      });
+
       // Send RPC request
       const response = await connection.rpcClient.callMethod(
         connection.ws,

front_end/panels/ai_chat/LLM/LLMClient.ts

@@ -17,6 +17,7 @@ import { GenericOpenAIProvider } from './GenericOpenAIProvider.js';
 import { CustomProviderManager } from '../core/CustomProviderManager.js';
 import { LLMResponseParser } from './LLMResponseParser.js';
 import { createLogger } from '../core/Logger.js';
+import { getCurrentTracingContext } from '../tracing/TracingConfig.js';
 
 const logger = createLogger('LLMClient');
 
@@ -48,6 +49,7 @@ export interface LLMCallRequest {
   temperature?: number;
   retryConfig?: Partial<RetryConfig>;
   agentName?: string; // Name of the calling agent for provider-specific routing
+  tracingMetadata?: Record<string, any>; // Explicit tracing metadata for Langfuse integration
 }
 
 /**
@@ -207,6 +209,38 @@ export class LLMClient {
       options.agentName = (request as any).agentName;
     }
 
+    // Get tracing metadata - prefer explicit request metadata over global context
+    // This ensures metadata flows correctly even when async context is lost
+    let tracingMetadata = request.tracingMetadata;
+
+    if (!tracingMetadata || Object.keys(tracingMetadata).length === 0) {
+      // Fall back to global tracing context
+      const tracingContext = getCurrentTracingContext();
+      logger.info('LLMClient.call() - Checking tracing context (fallback):', {
+        hasContext: !!tracingContext,
+        hasMetadata: !!tracingContext?.metadata,
+        metadataKeys: tracingContext?.metadata ? Object.keys(tracingContext.metadata) : [],
+        sessionId: tracingContext?.metadata?.session_id,
+        traceId: tracingContext?.metadata?.trace_id
+      });
+      if (tracingContext?.metadata && Object.keys(tracingContext.metadata).length > 0) {
+        tracingMetadata = tracingContext.metadata;
+      }
+    } else {
+      logger.info('LLMClient.call() - Using explicit tracingMetadata from request:', {
+        metadataKeys: Object.keys(tracingMetadata),
+        sessionId: tracingMetadata.session_id,
+        traceId: tracingMetadata.trace_id
+      });
+    }
+
+    if (tracingMetadata && Object.keys(tracingMetadata).length > 0) {
+      options.tracingMetadata = tracingMetadata;
+      logger.info('Passing tracing metadata to provider:', tracingMetadata);
+    } else {
+      logger.info('No tracing metadata available');
+    }
+
     return provider.callWithMessages(request.model, messages, options);
   }
 

front_end/panels/ai_chat/LLM/LLMTypes.ts

@@ -201,6 +201,14 @@ export interface LLMCallOptions {
   reasoningLevel?: 'low' | 'medium' | 'high'; // For O-series models
   retryConfig?: Partial<RetryConfig>;
   agentName?: string; // Name of the calling agent for provider-specific routing
+  // Tracing metadata for Langfuse integration via LiteLLM
+  tracingMetadata?: {
+    session_id?: string;
+    trace_id?: string;
+    generation_name?: string;
+    tags?: string[];
+    [key: string]: any;
+  };
 }
 
 /**

front_end/panels/ai_chat/LLM/LiteLLMProvider.ts

@@ -209,6 +209,12 @@ export class LiteLLMProvider extends LLMBaseProvider {
         payloadBody.tool_choice = options.tool_choice;
       }
 
+      // Add tracing metadata for Langfuse integration
+      // LiteLLM forwards this to Langfuse callbacks
+      if (options?.tracingMetadata) {
+        payloadBody.metadata = options.tracingMetadata;
+      }
+
       logger.info('Request payload:', payloadBody);
 
       const data = await this.makeAPIRequest(payloadBody);

front_end/panels/ai_chat/agent_framework/AgentRunner.ts

@@ -749,6 +749,8 @@ export class AgentRunner {
           tools: toolSchemas,
           temperature: temperature ?? 0,
           agentName: agentName,  // Pass agent identity for provider-specific routing
+          // Pass tracing metadata explicitly for Langfuse integration
+          tracingMetadata: tracingContext?.metadata,
         });
 
         // Complete the generation observation

front_end/panels/ai_chat/core/AgentNodes.ts

@@ -239,6 +239,8 @@ export function createAgentNode(modelName: string, provider: LLMProvider, temper
               })),
               temperature: this.temperature,
               agentName: agentName,
+              // Pass tracing metadata explicitly from state context for Langfuse integration
+              tracingMetadata: state.context?.tracingContext?.metadata,
             });
 
             // Parse the response

front_end/panels/ai_chat/core/AgentService.ts

@@ -276,15 +276,15 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{
       };
     }
 
-    // Default: provider requires apiKey
-    if (!apiKey) {
+    // Default: provider requires apiKey (unless in AUTOMATED_MODE where keys come dynamically)
+    if (!apiKey && !BUILD_CONFIG.AUTOMATED_MODE) {
       logger.warn(`Provider ${provider} requires API key`);
       return null;
     }
 
     return {
       provider,
-      apiKey
+      apiKey: apiKey || ''  // Default to empty string for AUTOMATED_MODE
     };
   }
 
@@ -597,7 +597,9 @@ export class AgentService extends Common.ObjectWrapper.ObjectWrapper<{
           tracingContext: {
             sessionId: existingContext?.sessionId || this.#sessionId,
             traceId,
-            parentObservationId: parentObservationId
+            parentObservationId: parentObservationId,
+            // Forward metadata from evaluation context for Langfuse session grouping
+            metadata: existingContext?.metadata
           },
           executionId: this.#executionId,
           abortSignal: this.#abortController?.signal,

front_end/panels/ai_chat/evaluation/EvaluationAgent.ts

@@ -321,13 +321,23 @@ export class EvaluationAgent {
       tool: params.tool
     });
 
-    // Create a trace for this evaluation
-    const traceId = `eval-${params.evaluationId}-${Date.now()}`;
-    const sessionId = `eval-session-${Date.now()}`;
-    const tracingContext: TracingContext = { 
-      traceId, 
+    // Use tracing metadata from request params (from eval framework via api-server)
+    // The tracing field is now properly forwarded through BrowserAgentServer.executeRequest()
+    const requestTracing = params.tracing || {};
+    logger.info('Tracing metadata received:', {
+      hasTracing: !!params.tracing,
+      tracingKeys: Object.keys(requestTracing),
+      sessionId: requestTracing.session_id,
+      traceId: requestTracing.trace_id
+    });
+    const traceId = requestTracing.trace_id || `eval-${params.evaluationId}-${Date.now()}`;
+    const sessionId = requestTracing.session_id || `eval-session-${Date.now()}`;
+    const tracingContext: TracingContext = {
+      traceId,
       sessionId,
-      parentObservationId: undefined 
+      parentObservationId: undefined,
+      // Include full tracing metadata for LLM calls
+      metadata: requestTracing
     };
 
     try {

front_end/panels/ai_chat/evaluation/EvaluationProtocol.ts

@@ -91,6 +91,16 @@ export interface EvaluationParams {
     retries: number;
     priority?: 'low' | 'normal' | 'high';
   };
+  // Tracing metadata for Langfuse session grouping
+  tracing?: {
+    session_id?: string;
+    trace_id?: string;
+    eval_id?: string;
+    eval_name?: string;
+    category?: string;
+    tags?: string[];
+    [key: string]: any;
+  };
 }
 
 export interface EvaluationSuccessResponse {