diff --git a/docs/pyagentspec/source/agentspec/index.rst b/docs/pyagentspec/source/agentspec/index.rst index 0fb4c80a..9cb6cc5d 100644 --- a/docs/pyagentspec/source/agentspec/index.rst +++ b/docs/pyagentspec/source/agentspec/index.rst @@ -28,3 +28,4 @@ with all the latest updates at :ref:`this link`. Introduction, motivation & vision Language specification (v25.4.1) Positioning in the agentic ecosystem + Tracing diff --git a/docs/pyagentspec/source/agentspec/json_spec/agentspec_json_spec_26_1_0.json b/docs/pyagentspec/source/agentspec/json_spec/agentspec_json_spec_26_1_0.json index eb561576..978a28d7 100644 --- a/docs/pyagentspec/source/agentspec/json_spec/agentspec_json_spec_26_1_0.json +++ b/docs/pyagentspec/source/agentspec/json_spec/agentspec_json_spec_26_1_0.json @@ -3669,6 +3669,18 @@ "$ref": "#/$defs/OpenAIAPIType", "default": "chat_completions" }, + "api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Api Key" + }, "$referenced_components": { "$ref": "#/$defs/ReferencedComponents" }, diff --git a/docs/pyagentspec/source/agentspec/language_spec_nightly.rst b/docs/pyagentspec/source/agentspec/language_spec_nightly.rst index a68cdef1..e7762245 100644 --- a/docs/pyagentspec/source/agentspec/language_spec_nightly.rst +++ b/docs/pyagentspec/source/agentspec/language_spec_nightly.rst @@ -649,13 +649,15 @@ OpenAI This class of LLMs refers to the models offered by `OpenAI `_. Similar to :ref:`OpenAI Compatible LLMs `, you can also configure the ``api_type`` parameter, which takes one of 2 string values, namely ``chat_completions`` or ``responses``. -By default, the API type is set to chat completions. +By default, the API type is set to chat completions. Additionally, an optional +``api_key`` can be set for the remote LLM model. .. code-block:: python class OpenAiConfig(LlmConfig): model_id: str api_type: Literal["chat_completions", "responses"] = "chat_completions" + api_key: SensitiveField[Optional[str]] = None OCI GenAI ^^^^^^^^^ @@ -2365,6 +2367,8 @@ See all the fields below that are considered sensitive fields: +==================================+====================+ | OpenAiCompatibleConfig | api_key | +----------------------------------+--------------------+ +| OpenAiConfig | api_key | ++----------------------------------+--------------------+ | OciClientConfigWithSecurityToken | auth_file_location | +----------------------------------+--------------------+ | OciClientConfigWithApiKey | auth_file_location | diff --git a/docs/pyagentspec/source/agentspec/tracing.rst b/docs/pyagentspec/source/agentspec/tracing.rst new file mode 100644 index 00000000..5ea18ac8 --- /dev/null +++ b/docs/pyagentspec/source/agentspec/tracing.rst @@ -0,0 +1,1186 @@ +.. _agentspec_tracing: + +===================================================== +Open Agent Specification Tracing (Agent Spec Tracing) +===================================================== + +Overview +======== + +Open Agent Specification Tracing (short: Agent Spec Tracing) is an extension of +Agent Spec that standardizes how agent and flow executions emit traces. +It defines a unified, implementation-agnostic semantic for: + +- Events: structured, point-in-time facts. +- Spans: time-bounded execution contexts that group events. +- Traces: trees of spans that represent an end-to-end execution. +- SpanProcessors: pluggable hooks to consume spans and events (e.g., export to UIs, tracing backends, or logs). + +Agent Spec Tracing enables: + +- Runtime adapters to emit consistent traces across different frameworks. +- Consumers (observability backends, UIs, developer tooling) to ingest one standardized format regardless of the producer. + +Agent Spec Tracing aligns with widely used observability concepts (e.g., +OpenTelemetry), while grounding definitions in Agent Spec components and +semantics. It specifies what spans and events to emit, when to emit them, and +which attributes to include, including which attributes are sensitive. + +Scope and goals +--------------- + +- Provide a canonical list of span and event types for Agent Spec runtimes. +- Define lifecycle and attribute schema for each span/event. +- Identify sensitive fields and how they should be handled. +- Provide a minimal API surface for producers and consumers. +- Remain neutral to storage/transport (Telemetry, UIs, files, etc.). + +Core Concepts +============= + +Event +----- + +An Event is an atomic episode that occurs at a specific time instant. It always belongs to exactly one Span. + +Events have a definition similar to the Agent Spec Components, and they have the same descriptive fields: +``id``, ``name``, ``description``, ``type``, and ``metadata``. +Additionally, they require a ``timestamp`` that defines when the event occurred, and extensions of this class can +add more attributes based on the event they represent, aimed at preserving all the relevant information +related to the event being recorded. +Events can also have Sensitive fields, that are declared and must be handled per Agent Spec guidelines. + +.. code-block:: python + + class Event: + id: str # Unique identifier for the event. Typically generated from component type plus a unique token. + type: str # Concrete type specifier for this event + name: str # Name of the event, if applicable + description: str # Description of the event, if applicable + metadata: Dictionary[str, Any] # Additional metadata that could be used for extra information + timestamp: int # nanoseconds since epoch + +- timestamp: time of occurrence (ns). Producers should use monotonic clocks + where possible and/or convert to wall-clock ns as configured by the runtime. + +Agent Spec Tracing defines a set of Event types with specific attributes, that you can find in the following sections. + + +Span +---- + +A Span defines a time-bounded execution context. Each Span: + +- starts at start_time (ns), ends at end_time (ns), end_time can be null if not closed. +- can contain zero or more Events. +- can be nested (child span has a parent span). + + +Also Spans have a definition similar to the Agent Spec Components, and they share the same descriptive fields: +``id``, ``name``, ``description``, ``type``, and ``metadata``. +Extensions of this Span can add more attributes based on the Span they represent. +Attributes on a Span typically reflect configuration that applies to the whole +duration of the Span (e.g., the Agent being executed, the LLM config, etc.). +Spans can also have Sensitive fields, that are declared and must be handled per Agent Spec guidelines. + +Besides these attributes, Spans MUST also implement the following interface: + +.. code-block:: python + + class Span: + id: str # Unique identifier for the span. Typically generated from component type plus a unique token. + type: str # Concrete type specifier for this span. + name: str # Name of the span, if applicable + description: str # Description of the span, if applicable + metadata: Dictionary[str, Any] # Additional metadata that could be used for extra information + start_time: int + end_time: Optional[int] + events: List[Event] + + def start(self) -> None: ... + + def end(self) -> None: ... + + def add_event(self, event: Event) -> None: ... + + +- ``start``: called when the span starts. +- ``shutdown``: called when the span ends. +- ``add_event``: called when an event has to be added to this Span. It MUST append the event in the ``events`` attribute. + +Lifecycle rules: + +- Spans MUST have start_time when started, while end_time is set on end. +- Events added to a Span MUST have timestamps within [start_time, end_time], whenever end_time is known. +- Spans MAY contain child spans. Implementations SHOULD propagate correlation context so consumers can rebuild the tree. + +Agent Spec Tracing defines a set of Span types with specific attributes, that you can find in the following sections. + +SpanProcessor +------------- + +A SpanProcessor receives callbacks when Spans start/end and when Events are added. +Processors are meant to consume the Agent Spec traces (spans, events) emitted by the runtime adapter +during the execution. They can be used to export traces to third parties consumers (e.g., to OpenTelemetry, files, UIs). + +A SpanProcessor MUST implement the following interface. + +.. code-block:: python + + class SpanProcessor(ABC): + + def on_start(self, span: Span) -> None: ... + + def on_end(self, span: "Span") -> None: ... + + def on_event(self, event: Event, span: Span) -> None: ... + + def startup(self) -> None: ... + + def shutdown(self) -> None: ... + +- ``startup``: called when an Agent Spec Tracing session starts. +- ``shutdown``: called when an Agent Spec Tracing session ends. +- ``on_start``: called when a Span starts. +- ``on_end``: called when a Span ends. +- ``on_event``: called when an Event is added to a Span. + +Trace +----- + +A Trace groups all spans and events that belong to the same top-level assistant execution. +It is the root where all the SpanProcessors that must be active during the assistant execution are declared. + +- Opening a Trace SHOULD call ``SpanProcessor.startup()`` on all configured processors. +- Closing a Trace SHOULD call ``SpanProcessor.shutdown()`` on all configured processors. + +Standard Span Types +=================== + +This first version defines the following span types. +All spans inherit the attributes of the base ``Span`` class. +The attributes listed here are additional and span-specific. +Fields marked sensitive MUST be handled as described in Security Considerations. + +LlmGenerationSpan +----------------- + +Covers the whole LLM generation process. + +- Starts: when the LLM generation request is received and the LLM call is + performed. +- Ends: when the LLM output has been generated and is ready to be processed. + +Attributes: + +.. list-table:: + :header-rows: 1 + :widths: 20 50 12 8 10 + + * - Name + - Description + - Type + - Default + - Sensitive + * - llm_config + - The LlmConfig performing the generation + - LlmConfig + - - + - no + +ToolExecutionSpan +----------------- + +Covers a tool execution (excluding client-side tools executed by the UI/client). + +- Starts: when tool execution starts. +- Ends: when the tool execution completes and the result is ready to be processed. + +Attributes: + +.. list-table:: + :header-rows: 1 + :widths: 20 50 12 8 10 + + * - Name + - Description + - Type + - Default + - Sensitive + * - tool + - The Tool being executed + - Tool + - - + - no + +AgentExecutionSpan +------------------ + +Represents the execution of an Agent. May be nested for sub-agents. + +- Starts: when the agent execution starts. +- Ends: when the agent execution completes and outputs are ready to process. + +Attributes: + +.. list-table:: + :header-rows: 1 + :widths: 20 50 12 8 10 + + * - Name + - Description + - Type + - Default + - Sensitive + * - agent + - The Agent being executed + - Agent + - - + - no + +SwarmExecutionSpan +------------------ + +Specialization of AgentExecutionSpan for a Swarm Component. + +- Starts: when swarm execution starts. +- Ends: when swarm execution completes and outputs are ready. + +Attributes: + +.. list-table:: + :header-rows: 1 + :widths: 20 50 12 8 10 + + * - Name + - Description + - Type + - Default + - Sensitive + * - swarm + - The Swarm being executed + - Swarm + - - + - no + +ManagerWorkersExecutionSpan +--------------------------- + +Specialization of AgentExecutionSpan for a Manager-Workers pattern. + +- Starts: when Manager-Workers execution starts. +- Ends: when the execution completes and outputs are ready. + +Attributes: + +.. list-table:: + :header-rows: 1 + :widths: 20 50 12 8 10 + + * - Name + - Description + - Type + - Default + - Sensitive + * - managerworkers + - The ManagerWorkers being executed + - ManagerWorkers + - - + - no + +FlowExecutionSpan +----------------- + +Covers the execution of a Flow. + +- Starts: when the Flow's StartNode execution starts. +- Ends: when one of the Flow's EndNode executions finishes. + +Attributes: + +.. list-table:: + :header-rows: 1 + :widths: 20 50 12 8 10 + + * - Name + - Description + - Type + - Default + - Sensitive + * - flow + - The Flow being executed + - Flow + - - + - no + +NodeExecutionSpan +----------------- + +Covers the execution of a single Node within a Flow. + +- Starts: when the Node execution starts on the given inputs. +- Ends: when the Node execution ends and outputs are ready. + +Attributes: + +.. list-table:: + :header-rows: 1 + :widths: 20 50 12 8 10 + + * - Name + - Description + - Type + - Default + - Sensitive + * - node + - The Node being executed + - Node + - - + - no + + +Standard Event Types +==================== + +All events are inherit the attributes defined in the base ``Event`` class. +The following events define the default set for Agent Spec Tracing. +For each, we specify when it is emitted and which attributes it carries. +Fields marked sensitive MUST be handled as described in Security Considerations. + +LLM events +---------- + +LlmGenerationRequest +^^^^^^^^^^^^^^^^^^^^ + +An LLM generation request was received. Emitted when an LlmGenerationSpan starts. + +Attributes: + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - llm_config + - The LlmConfig performing the generation + - LlmConfig + - - + - no + * - request_id + - Identifier of the generation request + - str + - - + - no + * - llm_generation_config + - The LLM generation parameters used for this call + - Optional[LlmGenerationConfig] + - null + - no + * - prompt + - Prompt that will be sent to the LLM; a list of Message with at least content and role, optionally sender + - List[Message] + - - + - yes + * - tools + - Tools sent as part of the generation request + - Optional[List[Tool]] + - null + - no + +The ``Message`` model should be implemented as + +.. code-block:: python + + class Message(BaseModel): + """Model used to specify LLM message details in events and spans""" + + id: Optional[str] = None + "Identifier of the message" + + content: str + "Content of the message" + + sender: Optional[str] = None + "Sender of the message" + + role: str + "Role of the sender of the message. Typically 'user', 'assistant', or 'system'" + + +LlmGenerationResponse +^^^^^^^^^^^^^^^^^^^^^ + +An LLM response was received. Emitted when an LlmGenerationSpan ends. + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - llm_config + - The LlmConfig performing the generation + - LlmConfig + - - + - no + * - request_id + - Identifier of the generation request + - str + - - + - no + * - tool_calls + - Tool calls returned by the LLM. Each tool call should contain a ``call_id`` identifier, the ``tool_name``, + and the ``arguments`` with which the tool is being called as a string in JSON format. + - List[ToolCall] + - - + - yes + * - completion_id + - Identifier of the completion related to this response + - Optional[str] + - null + - no + * - content + - The content of the response (assistant message text) + - str + - - + - yes + +The ``ToolCall`` model should be implemented as + +.. code-block:: python + + class ToolCall(BaseModel): + """Model for an LLM tool call.""" + + call_id: str + "Identifier of the tool call" + + tool_name: str + "The name of the tool that should be called" + + arguments: str + "The values of the arguments that should be passed to the tool, in JSON format" + + +LlmGenerationStreamingChunkReceived +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A streamed chunk was received during LLM generation. + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - llm_config + - The LlmConfig performing the generation + - LlmConfig + - - + - no + * - request_id + - Identifier of the generation request + - str + - - + - no + * - tool_calls + - Tool calls chunked by the LLM. Each tool call should contain a ``call_id`` identifier, the ``tool_name``, + and the ``arguments`` with which the tool is being called as a string in JSON format. The content of arguments + should be considered as the delta compared to the last chunk received. + - List[ToolCall] + - - + - yes + * - completion_id + - Identifier of the parent completion (message or tool call) this chunk belongs to + - Optional[str] + - null + - no + * - content + - The chunk content. This is the delta compared to the last chunk received. + - str + - - + - yes + +Tool events +----------- + +ToolExecutionRequest +^^^^^^^^^^^^^^^^^^^^ + +A tool execution request is received. Emitted when a ToolExecutionSpan starts (or a client tool is requested). + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - tool + - The Tool being executed + - Tool + - - + - no + * - request_id + - Identifier of the tool execution request + - str + - - + - no + * - inputs + - Input values for the tool (one per input property) + - dict[str, any] + - - + - yes + +ToolExecutionResponse +^^^^^^^^^^^^^^^^^^^^^ + +A tool execution finishes and a result is received. Emitted when a ToolExecutionSpan ends (or a client tool result is received). + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - tool + - The Tool being executed + - Tool + - - + - no + * - request_id + - Identifier of the tool execution request + - str + - - + - no + * - output + - Return value produced by the tool (one per output property) + - dict[str, any] + - - + - yes + +ToolConfirmationRequest +^^^^^^^^^^^^^^^^^^^^^^^ + +A tool confirmation is requested (e.g., human-in-the-loop approval before execution). + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - tool + - The Tool being executed + - Tool + - - + - no + * - tool_execution_request_id + - Identifier of the related tool execution request + - str + - - + - no + * - request_id + - Identifier of this confirmation request + - str + - - + - no + +ToolConfirmationResponse +^^^^^^^^^^^^^^^^^^^^^^^^ + +A tool confirmation response is received. + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - tool + - The Tool being executed + - Tool + - - + - no + * - tool_execution_request_id + - Identifier of the related tool execution request + - str + - - + - no + * - request_id + - Identifier of the confirmation request + - str + - - + - no + * - execution_confirmed + - Whether execution was confirmed + - bool + - - + - no + +AgenticComponent events +----------------------- + +AgentExecutionStart +^^^^^^^^^^^^^^^^^^^ + +Emitted when an AgentExecutionSpan starts. + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - agent + - The Agent being executed + - Agent + - - + - no + * - inputs + - Inputs used for the agent execution (one per input property) + - dict[str, any] + - - + - yes + +AgentExecutionEnd +^^^^^^^^^^^^^^^^^ + +Emitted when an AgentExecutionSpan ends. + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - agent + - The Agent being executed + - Agent + - - + - no + * - outputs + - Outputs produced by the agent (one per output property) + - dict[str, any] + - - + - yes + +ManagerWorkersExecutionStart +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Emitted when a ManagerWorkersExecutionSpan starts. + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - managerworkers + - The ManagerWorkers being executed + - ManagerWorkers + - - + - no + * - inputs + - Inputs used for execution (one per input property) + - dict[str, any] + - - + - yes + +ManagerWorkersExecutionEnd +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Emitted when a ManagerWorkersExecutionSpan ends. + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - managerworkers + - The ManagerWorkers being executed + - ManagerWorkers + - - + - no + * - outputs + - Outputs produced (one per output property) + - dict[str, any] + - - + - yes + +SwarmExecutionStart +^^^^^^^^^^^^^^^^^^^ + +Emitted when a SwarmExecutionSpan starts. + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - swarm + - The Swarm being executed + - Swarm + - - + - no + * - inputs + - Inputs used for the swarm execution (one per input property) + - dict[str, any] + - - + - yes + +SwarmExecutionEnd +^^^^^^^^^^^^^^^^^ + +Emitted when a SwarmExecutionSpan ends. + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - swarm + - The Swarm being executed + - Swarm + - - + - no + * - outputs + - Outputs produced (one per output property) + - dict[str, any] + - - + - yes + +Flow events +----------- + +FlowExecutionStart +^^^^^^^^^^^^^^^^^^ + +Emitted when a FlowExecutionSpan starts. + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - flow + - The Flow being executed + - Flow + - - + - no + * - inputs + - Inputs used by the flow (one per StartNode input property) + - dict[str, any] + - - + - yes + +FlowExecutionEnd +^^^^^^^^^^^^^^^^ + +Emitted when a FlowExecutionSpan ends. + +.. list-table:: + :header-rows: 1 + :widths: 22 38 20 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - flow + - The Flow being executed + - Flow + - - + - no + * - outputs + - Outputs produced by the flow (one per flow output property) + - dict[str, any] + - - + - yes + * - branch_selected + - Exit branch selected at the end of the Flow + - str + - - + - no + +NodeExecutionStart +^^^^^^^^^^^^^^^^^^ + +Emitted when a NodeExecutionSpan starts. + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - node + - The Node being executed + - Node + - - + - no + * - inputs + - Inputs used by the node (one per node input property) + - dict[str, any] + - - + - yes + +NodeExecutionEnd +^^^^^^^^^^^^^^^^ + +Emitted when a NodeExecutionSpan ends. + +.. list-table:: + :header-rows: 1 + :widths: 22 38 20 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - node + - The Node being executed + - Node + - - + - no + * - outputs + - Outputs produced by the node (one per node output property) + - dict[str, any] + - - + - yes + * - branch_selected + - Exit branch selected at the end of the Node + - str + - - + - no + +Conversation and control events +------------------------------- + +ConversationMessageAdded +^^^^^^^^^^^^^^^^^^^^^^^^ + +A message was added to the conversation. + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - message + - The message added; must contain at least content and role, optionally sender + - Message + - - + - yes + +ExceptionRaised +^^^^^^^^^^^^^^^ + +An exception occurred during execution. + +.. list-table:: + :header-rows: 1 + :widths: 24 50 16 10 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - exception_type + - Type of the exception + - str + - - + - no + * - exception_message + - Exception message + - str + - - + - yes + * - exception_stacktrace + - Stacktrace of the exception, if available + - Optional[str] + - null + - yes + +HumanInTheLoopRequest +^^^^^^^^^^^^^^^^^^^^^ + +A human-in-the-loop intervention is required; execution is interrupted until a response. + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - request_id + - Identifier of the human-in-the-loop request + - str + - - + - no + * - content + - Request content forwarded to the user + - dict[str, any] + - {} (empty object) + - yes + +HumanInTheLoopResponse +^^^^^^^^^^^^^^^^^^^^^^ + +A HITL response is received; execution resumes. + +.. list-table:: + :header-rows: 1 + :widths: 22 48 18 12 10 + + * - Name + - Description + - Type + - Default/Optional + - Sensitive + * - request_id + - Identifier of the HITL request + - str + - - + - no + * - content + - Response content received from the user + - dict[str, any] + - {} (empty object) + - yes + + +Deterministic identifiers and correlation +----------------------------------------- + +Some events define correlation identifiers to allow consumers to link request/response and other events. + +For example: + +- request_id: unique identifier of a single LLM or Tool execution request within a Span. +- completion_id: identifier of a completion (LLM message or tool-call) which may receive streaming chunks. +- tool_execution_request_id: identifier of a tool execution for confirmation. + +Runtimes SHOULD ensure uniqueness within a Span and consistency across all related events. + + +PyAgentSpecTracing (Python materialization) +=========================================== + +The ``pyagentspec.tracing`` subpackage of ``pyagentspec`` provides convenient Pydantic-based models and interfaces so that: + +- Producers (adapters, runtimes) can emit spans/events according to Agent Spec Tracing standards. +- Consumers (exporters, UIs) can receive and consume them via SpanProcessors. + +Emitting traces (producer example) +---------------------------------- + +Here's an example of how adapters can emit traces (i.e., start and close Spans, emit Events) +extracted from the AgentNode implementation of the LangGraph's adapter in ``pyagentspec==26.1.0``. + +.. code-block:: python + + with AgentExecutionSpan(name=f"AgentExecution - {agentspec_agent.name}", agent=agentspec_agent) as span: + span.add_event(AgentExecutionStart(inputs=inputs)) + result = agent.invoke(inputs, config) + outputs = result.outputs if hasattr(result, "outputs") else {} + span.add_event(AgentExecutionEnd(outputs=outputs)) + +Consuming traces (consumer example) +----------------------------------- + +.. code-block:: python + + class OpenTelemetrySpanProcessor(SpanProcessor): + + def __init__(self, sdk_processor: OtelSdkSpanProcessor): + self._sdk_processor = sdk_processor + + def on_start(self, span: "Span") -> None: + otel_span = self._to_otel_span(span) + otel_span.start(start_time=span.start_time) + self._sdk_processor.on_start(span=otel_span) + + def on_end(self, span: "Span") -> None: + otel_span = self._to_otel_span(span) + otel_span.end(end_time=span.end_time) + self._sdk_processor.on_end(span=otel_span) + + def on_event(self, event: Event, span: Span) -> None: + # Other processors may use this hook to stream events + pass + + def startup(self) -> None: + ... + + def shutdown(self) -> None: + self._sdk_processor.shutdown + +Interoperability examples +------------------------- + +Tracing with LangGraph + +.. code-block:: python + + from pyagentspec.adapters.langgraph import AgentSpecLoader + from openinference_spanprocessor import ArizePhoenixSpanProcessor + # Assuming this package implements a SpanProcessor that takes the Agent Spec Traces and sends them to a Phoenix Arize server + + agent_json = read_json_file("my/agentspec/agent.json") + processor = ArizePhoenixSpanProcessor(mask_sensitive_information=False, project_name="agentspec-tracing-test") + + with Trace(name="agentspec_langgraph_demo", span_processors=[processor]) as trace: + agent = AgentSpecLoader().load_json(agent_json) + result = agent.invoke({"inputs": {}, "messages": []}) + +Tracing with WayFlow + +.. code-block:: python + + from wayflowcore.agentspec import AgentSpecLoader + from wayflowcore.agentspec.tracing import AgentSpecTracingEventListener + from wayflowcore.events.eventlistener import register_event_listeners + from openinference_spanprocessor import ArizePhoenixSpanProcessor + + agent_json = read_json_file("my/agentspec/agent.json") + processor = ArizePhoenixSpanProcessor(mask_sensitive_information=False, project_name="agentspec-tracing-test") + + with register_event_listeners([AgentSpecTracingEventListener()]): + with Trace(name="agentspec_wayflow_demo", span_processors=[processor]) as trace: + agent = AgentSpecLoader().load_json(agent_json) + conversation = agent.start_conversation() + status = conversation.execute() + + +Security Considerations +======================= + +Agent Spec Tracing inherits all security requirements from Agent Spec (see :doc:`../security`). +Additionally, tracing frequently includes potentially sensitive information (PII), including, but not limited to: + +- LLM prompts and generated content +- Tool inputs/outputs +- Exception messages and stacktraces +- Conversation messages + +Implementing a SpanProcessor +---------------------------- + +Key points: + +* **Async / non-blocking** - keep the span processor off the critical path to avoid impacting agent's performance. +* **Robust error handling** - never raise from span processor methods; drop or queue on failure. +* **Back-pressure** - apply rate limits, size limits, or batch Spans and Events to avoid DoS on the collector. + +Sensitive fields +---------------- + +Each event table above flags attributes that are considered sensitive. +Producers SHOULD mark and/or emit them using Agent Spec's Sensitive Field mechanism where +applicable; consumers (SpanProcessors) SHOULD: + +- Mask or omit sensitive fields by default when exporting traces. +- Provide an explicit configuration to unmask for trusted environments. +- Avoid logging or exporting sensitive data inadvertently. + +Guidelines: + +- Attribute-level masking: redact entire values or apply strongly irreversible + masking (e.g., replace content with fixed placeholders or hashes as policy dictates). +- Downstream mappers (e.g., OpenTelemetry exporters) MUST NOT downgrade masking guarantees. +- When masking affects correlation (e.g., truncating request_id), preserve minimal + non-sensitive identifiers for linkage. + +Design Notes and Best Practices +=============================== + +- Event emission ordering: Within a span, events SHOULD be in timestamp order. +- Time units: Use nanoseconds since epoch for timestamps and start/end times + for consistency with common tracing systems. +- Nesting: Prefer nesting spans to represent sub-operations (e.g., NodeExecutionSpan + under FlowExecutionSpan, ToolExecutionSpan under AgentExecutionSpan). +- Exceptions: Emit ExceptionRaised with type/message/stacktrace and consider adding + it before ending the current span or on a dedicated error span, depending on runtime design. + + +FAQ and Open Questions +====================== + +Naming alignment with observability +----------------------------------- + +This specification uses tracing terminology common in OpenTelemetry (Trace, +Span, Event, SpanProcessor) to leverage community familiarity. + +Environment context +------------------- + +This version focuses on agentic execution tracing. Future versions may add +execution-environment spans or include environment metadata on Trace. + +References and Cross-links +========================== + +- Agent Spec language specification: :doc:`language_spec_nightly` +- Security guidelines: :doc:`../security` diff --git a/docs/pyagentspec/source/code_examples/howto_llm_from_different_providers.py b/docs/pyagentspec/source/code_examples/howto_llm_from_different_providers.py index 001db9e7..d625c90c 100644 --- a/docs/pyagentspec/source/code_examples/howto_llm_from_different_providers.py +++ b/docs/pyagentspec/source/code_examples/howto_llm_from_different_providers.py @@ -50,6 +50,7 @@ model_id="llama-4-maverick", url="http://url.to.my.vllm.server/llama4mav", api_type=OpenAIAPIType.RESPONSES, + api_key="optional_api_key", default_generation_parameters=generation_config, ) # .. openaicompatible-end @@ -77,6 +78,7 @@ name="openai-gpt-5", model_id="gpt-5", default_generation_parameters=generation_config, + api_key="optional_api_key", ) # .. openai-end diff --git a/docs/pyagentspec/source/howtoguides/howto_llm_from_different_providers.rst b/docs/pyagentspec/source/howtoguides/howto_llm_from_different_providers.rst index 5bc696a4..6d3a17a0 100644 --- a/docs/pyagentspec/source/howtoguides/howto_llm_from_different_providers.rst +++ b/docs/pyagentspec/source/howtoguides/howto_llm_from_different_providers.rst @@ -143,6 +143,10 @@ You can refer to one of those models by using the ``OpenAiConfig`` Component. The API type that should be used. Can be either ``chat_completions`` or ``responses``. +.. option:: api_key: str, null + + An optional api key for the authentication with the OpenAI endpoint. + .. option:: default_generation_parameters: dict, null Default parameters for text generation with this model. @@ -180,6 +184,10 @@ The ``OpenAiCompatibleConfig`` allows users to use this type of models in their The API type that should be used. Can be either ``chat_completions`` or ``responses``. +.. option:: api_key: str, null + + An optional api key if the remote server requires it. + .. option:: default_generation_parameters: dict, null Default parameters for text generation with this model. diff --git a/pyagentspec/requirements-dev.txt b/pyagentspec/requirements-dev.txt index 4e220279..f1ee3eb8 100644 --- a/pyagentspec/requirements-dev.txt +++ b/pyagentspec/requirements-dev.txt @@ -29,6 +29,7 @@ sphinxcontrib-htmlhelp<2.0.5 sphinxcontrib.serializinghtml<1.1.10 sphinxcontrib.qthelp<1.0.7 sphinx_toolbox<=3.8.0 +sphinx_design==0.6.1 -c constraints/constraints.txt # Frameworks do not support all python versions up to 3.14 yet diff --git a/pyagentspec/src/pyagentspec/adapters/crewai/_crewaiconverter.py b/pyagentspec/src/pyagentspec/adapters/crewai/_crewaiconverter.py index 13144590..9408ec29 100644 --- a/pyagentspec/src/pyagentspec/adapters/crewai/_crewaiconverter.py +++ b/pyagentspec/src/pyagentspec/adapters/crewai/_crewaiconverter.py @@ -17,6 +17,7 @@ CrewAIServerToolType, CrewAITool, ) +from pyagentspec.adapters.crewai.tracing import CrewAIAgentWithTracing from pyagentspec.agent import Agent as AgentSpecAgent from pyagentspec.component import Component as AgentSpecComponent from pyagentspec.llms import LlmConfig as AgentSpecLlmConfig @@ -81,6 +82,11 @@ def _create_pydantic_model_from_properties( class AgentSpecToCrewAIConverter: + def __init__(self, enable_agentspec_tracing: bool = True) -> None: + self.enable_agentspec_tracing = enable_agentspec_tracing + self._is_root_call: bool = True + self._obj_id_to_agentspec_component: Dict[int, AgentSpecComponent] = {} + def convert( self, agentspec_component: AgentSpecComponent, @@ -91,35 +97,58 @@ def convert( if converted_components is None: converted_components = {} - if agentspec_component.id in converted_components: - return converted_components[agentspec_component.id] + if self._is_root_call: + # Reset the obj id -> agentspec component mapping + self._obj_id_to_agentspec_component = {} - # If we did not find the object, we create it, and we record it in the referenced_objects registry - crewai_component: Any - if isinstance(agentspec_component, AgentSpecLlmConfig): - crewai_component = self._llm_convert_to_crewai( - agentspec_component, tool_registry, converted_components - ) - elif isinstance(agentspec_component, AgentSpecAgent): - crewai_component = self._agent_convert_to_crewai( - agentspec_component, tool_registry, converted_components - ) - elif isinstance(agentspec_component, AgentSpecTool): - crewai_component = self._tool_convert_to_crewai( - agentspec_component, tool_registry, converted_components - ) - elif isinstance(agentspec_component, AgentSpecComponent): - raise NotImplementedError( - f"The AgentSpec Component type '{agentspec_component.__class__.__name__}' is not yet supported " - f"for conversion. Please contact the AgentSpec team." - ) - else: - raise TypeError( - f"Expected object of type 'pyagentspec.component.Component'," - f" but got {type(agentspec_component)} instead" + is_root_call = self._is_root_call + self._is_root_call = False + + if agentspec_component.id not in converted_components: + # If we did not find the object, we create it, and we record it in the referenced_objects registry + crewai_component: Any + if isinstance(agentspec_component, AgentSpecLlmConfig): + crewai_component = self._llm_convert_to_crewai( + agentspec_component, tool_registry, converted_components + ) + elif isinstance(agentspec_component, AgentSpecAgent): + crewai_component = self._agent_convert_to_crewai( + agentspec_component, tool_registry, converted_components + ) + elif isinstance(agentspec_component, AgentSpecTool): + crewai_component = self._tool_convert_to_crewai( + agentspec_component, tool_registry, converted_components + ) + elif isinstance(agentspec_component, AgentSpecComponent): + raise NotImplementedError( + f"The AgentSpec Component type '{agentspec_component.__class__.__name__}' is not yet supported " + f"for conversion. Please contact the AgentSpec team." + ) + else: + raise TypeError( + f"Expected object of type 'pyagentspec.component.Component'," + f" but got {type(agentspec_component)} instead" + ) + converted_components[agentspec_component.id] = crewai_component + + converted_crewai_component = converted_components[agentspec_component.id] + self._obj_id_to_agentspec_component[id(converted_crewai_component)] = agentspec_component + + if ( + is_root_call + and self.enable_agentspec_tracing + and isinstance(converted_crewai_component, CrewAIAgentWithTracing) + ): + # If the root component is an agent to which we can attach an agent spec listener, + # we monkey patch the root CrewAI component to attach the event listener for Agent Spec + from pyagentspec.adapters.crewai.tracing import AgentSpecEventListener + + converted_crewai_component._agentspec_event_listener = AgentSpecEventListener( + agentspec_components=self._obj_id_to_agentspec_component ) - converted_components[agentspec_component.id] = crewai_component - return converted_components[agentspec_component.id] + + self._is_root_call = is_root_call + return converted_crewai_component def _llm_convert_to_crewai( self, @@ -130,7 +159,9 @@ def _llm_convert_to_crewai( def parse_url(url: str) -> str: url = url.strip() - if not url.endswith("/v1"): + if url.endswith("/completions"): + return url + if not url.endswith("/v1") and not url.endswith("/litellm"): url += "/v1" if not url.startswith("http"): url = "http://" + url @@ -252,7 +283,7 @@ def _agent_convert_to_crewai( tool_registry: Dict[str, CrewAIServerToolType], converted_components: Optional[Dict[str, Any]] = None, ) -> CrewAIAgent: - return CrewAIAgent( + crewai_agent = CrewAIAgentWithTracing( # We interpret the name as the `role` of the agent in CrewAI, # the description as the `backstory`, and the system prompt as the `goal`, as they are all required # This interpretation comes from the analysis of CrewAI Agent definition examples @@ -271,3 +302,7 @@ def _agent_convert_to_crewai( for tool in agentspec_agent.tools ], ) + if not agentspec_agent.metadata: + agentspec_agent.metadata = {} + agentspec_agent.metadata["__crewai_agent_id__"] = str(crewai_agent.id) + return crewai_agent diff --git a/pyagentspec/src/pyagentspec/adapters/crewai/_types.py b/pyagentspec/src/pyagentspec/adapters/crewai/_types.py index d23a58f9..38202371 100644 --- a/pyagentspec/src/pyagentspec/adapters/crewai/_types.py +++ b/pyagentspec/src/pyagentspec/adapters/crewai/_types.py @@ -16,6 +16,30 @@ from crewai import LLM as CrewAILlm from crewai import Agent as CrewAIAgent from crewai import Flow as CrewAIFlow + from crewai.events.base_event_listener import BaseEventListener as CrewAIBaseEventListener + from crewai.events.base_events import BaseEvent as CrewAIBaseEvent + from crewai.events.event_bus import CrewAIEventsBus, crewai_event_bus + from crewai.events.types.agent_events import ( + AgentExecutionCompletedEvent as CrewAIAgentExecutionCompletedEvent, + ) + from crewai.events.types.agent_events import ( + AgentExecutionStartedEvent as CrewAIAgentExecutionStartedEvent, + ) + from crewai.events.types.agent_events import ( + LiteAgentExecutionCompletedEvent as CrewAILiteAgentExecutionCompletedEvent, + ) + from crewai.events.types.agent_events import ( + LiteAgentExecutionStartedEvent as CrewAILiteAgentExecutionStartedEvent, + ) + from crewai.events.types.llm_events import LLMCallCompletedEvent as CrewAILLMCallCompletedEvent + from crewai.events.types.llm_events import LLMCallStartedEvent as CrewAILLMCallStartedEvent + from crewai.events.types.llm_events import LLMStreamChunkEvent as CrewAILLMStreamChunkEvent + from crewai.events.types.tool_usage_events import ( + ToolUsageFinishedEvent as CrewAIToolUsageFinishedEvent, + ) + from crewai.events.types.tool_usage_events import ( + ToolUsageStartedEvent as CrewAIToolUsageStartedEvent, + ) from crewai.tools import BaseTool as CrewAIBaseTool from crewai.tools.base_tool import Tool as CrewAITool from crewai.tools.structured_tool import CrewStructuredTool as CrewAIStructuredTool @@ -28,11 +52,39 @@ CrewAIBaseTool = LazyLoader("crewai.tools").BaseTool CrewAITool = LazyLoader("crewai.tools.base_tool").Tool CrewAIStructuredTool = LazyLoader("crewai.tools.structured_tool").CrewStructuredTool + CrewAIBaseEventListener = LazyLoader("crewai.events.base_event_listener").BaseEventListener + CrewAIEventsBus = LazyLoader("crewai.events.event_bus").CrewAIEventsBus + crewai_event_bus = LazyLoader("crewai.events.event_bus").crewai_event_bus + crewai = LazyLoader("crewai") + CrewAIAgentExecutionStartedEvent = LazyLoader( + "crewai.events.types.agent_events" + ).AgentExecutionStartedEvent + CrewAIAgentExecutionCompletedEvent = LazyLoader( + "crewai.events.types.agent_events" + ).AgentExecutionCompletedEvent + CrewAILiteAgentExecutionStartedEvent = LazyLoader( + "crewai.events.types.agent_events" + ).LiteAgentExecutionStartedEvent + CrewAILiteAgentExecutionCompletedEvent = LazyLoader( + "crewai.events.types.agent_events" + ).LiteAgentExecutionCompletedEvent + CrewAILLMCallCompletedEvent = LazyLoader("crewai.events.types.llm_events").LLMCallCompletedEvent + CrewAIBaseEvent = LazyLoader("crewai.events.base_events").BaseEvent + CrewAILLMCallStartedEvent = LazyLoader("crewai.events.types.llm_events").LLMCallStartedEvent + CrewAILLMStreamChunkEvent = LazyLoader("crewai.events.types.llm_events").LLMStreamChunkEvent + CrewAIToolUsageFinishedEvent = LazyLoader( + "crewai.events.types.tool_usage_events" + ).ToolUsageFinishedEvent + CrewAIToolUsageStartedEvent = LazyLoader( + "crewai.events.types.tool_usage_events" + ).ToolUsageStartedEvent CrewAIComponent = Union[CrewAIAgent, CrewAIFlow[Any]] CrewAIServerToolType = Union[CrewAITool, Callable[..., Any]] __all__ = [ + "crewai", + "crewai_event_bus", "CrewAILlm", "CrewAIAgent", "CrewAIFlow", @@ -41,4 +93,16 @@ "CrewAIStructuredTool", "CrewAIComponent", "CrewAIServerToolType", + "CrewAIBaseEvent", + "CrewAIBaseEventListener", + "CrewAILLMCallCompletedEvent", + "CrewAILLMCallStartedEvent", + "CrewAILLMStreamChunkEvent", + "CrewAIToolUsageStartedEvent", + "CrewAIToolUsageFinishedEvent", + "CrewAIEventsBus", + "CrewAIAgentExecutionStartedEvent", + "CrewAIAgentExecutionCompletedEvent", + "CrewAILiteAgentExecutionStartedEvent", + "CrewAILiteAgentExecutionCompletedEvent", ] diff --git a/pyagentspec/src/pyagentspec/adapters/crewai/agentspecloader.py b/pyagentspec/src/pyagentspec/adapters/crewai/agentspecloader.py index 8180db0a..75d385f3 100644 --- a/pyagentspec/src/pyagentspec/adapters/crewai/agentspecloader.py +++ b/pyagentspec/src/pyagentspec/adapters/crewai/agentspecloader.py @@ -38,6 +38,7 @@ def __init__( """ self.tool_registry = tool_registry or {} self.plugins = plugins + self._enable_agentspec_tracing = True def load_yaml(self, serialized_assistant: str) -> CrewAIComponent: """ @@ -81,5 +82,7 @@ def load_component(self, agentspec_component: AgentSpecComponent) -> CrewAICompo """ return cast( CrewAIComponent, - AgentSpecToCrewAIConverter().convert(agentspec_component, self.tool_registry), + AgentSpecToCrewAIConverter( + enable_agentspec_tracing=self._enable_agentspec_tracing, + ).convert(agentspec_component, self.tool_registry), ) diff --git a/pyagentspec/src/pyagentspec/adapters/crewai/tracing.py b/pyagentspec/src/pyagentspec/adapters/crewai/tracing.py new file mode 100644 index 00000000..f458f110 --- /dev/null +++ b/pyagentspec/src/pyagentspec/adapters/crewai/tracing.py @@ -0,0 +1,488 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +import json +import threading +import time +import uuid +from contextlib import contextmanager +from typing import Any, Dict, Generator, List, Optional, Type, cast + +from pydantic import PrivateAttr + +from pyagentspec import Agent as AgentSpecAgent +from pyagentspec import Component as AgentSpecComponent +from pyagentspec.adapters.crewai._types import ( + CrewAIAgent, + CrewAIAgentExecutionCompletedEvent, + CrewAIAgentExecutionStartedEvent, + CrewAIBaseEvent, + CrewAIBaseEventListener, + CrewAIEventsBus, + CrewAILiteAgentExecutionCompletedEvent, + CrewAILiteAgentExecutionStartedEvent, + CrewAILLMCallCompletedEvent, + CrewAILLMCallStartedEvent, + CrewAILLMStreamChunkEvent, + CrewAIToolUsageFinishedEvent, + CrewAIToolUsageStartedEvent, +) +from pyagentspec.llms import LlmConfig as AgentSpecLlmConfig +from pyagentspec.llms import OpenAiCompatibleConfig as AgentSpecOpenAiCompatibleConfig +from pyagentspec.llms import OpenAiConfig as AgentSpecOpenAiConfig +from pyagentspec.tools import Tool as AgentSpecTool +from pyagentspec.tracing.events import ( + AgentExecutionEnd, + AgentExecutionStart, + LlmGenerationChunkReceived, + LlmGenerationRequest, + LlmGenerationResponse, + ToolExecutionRequest, + ToolExecutionResponse, +) +from pyagentspec.tracing.messages.message import Message as AgentSpecMessage +from pyagentspec.tracing.spans import AgentExecutionSpan, LlmGenerationSpan, Span, ToolExecutionSpan +from pyagentspec.tracing.spans.span import ( + _ACTIVE_SPAN_STACK, + get_active_span_stack, + get_current_span, +) + + +def _get_closest_span_of_given_type(agentspec_span_type: Type[Span]) -> Optional[Span]: + return next( + (span for span in get_active_span_stack()[::-1] if isinstance(span, agentspec_span_type)), + None, + ) + + +def _ensure_dict(obj: Any) -> Dict[str, Any]: + """Ensure that an object is a dict, if it is not, transform it into one.""" + if isinstance(obj, dict): + return obj + if isinstance(obj, str): + stripped = obj.strip() + if stripped.startswith("{") or stripped.startswith("["): + try: + parsed = json.loads(stripped) + if isinstance(parsed, list): + return {"value": parsed} + if isinstance(parsed, dict): + return parsed + except Exception: + return {"value": obj} + return {"value": obj} + return {"value": str(obj)} + + +class AgentSpecEventListener: + + def __init__(self, agentspec_components: Dict[int, AgentSpecComponent]) -> None: + super().__init__() + self.agentspec_components = agentspec_components + self._event_listener: Optional[_CrewAiEventListener] = None + self.scoped_handlers_context_generator: Optional[Generator[None, Any, None]] = None + self._events_flush_timeout: float = 2.0 + + @contextmanager + def record_listener(self) -> Generator[None, Any, None]: + from crewai.events import crewai_event_bus + + with crewai_event_bus.scoped_handlers(): + self._event_listener = _CrewAiEventListener(self.agentspec_components) + yield + # Before getting out, we ensure that the events have all been handled + # We first wait a little to make the handlers start before we continue with this code + time.sleep(0.1) + start_time = time.time() + while ( + len(self._event_listener._events_list) > 0 + and start_time + self._events_flush_timeout > time.time() + ): + time.sleep(0.1) + self._event_listener = None + + +class _CrewAiEventListener(CrewAIBaseEventListener): + """Bridges CrewAI streaming and tool events to Agent Spec Tracing""" + + def __init__(self, agentspec_components: Dict[int, AgentSpecComponent]) -> None: + super().__init__() + self.agentspec_components = agentspec_components + self.llm_configs_map: Dict[str, AgentSpecLlmConfig] = { + llm.model_id: llm + for llm in agentspec_components.values() + if isinstance(llm, (AgentSpecOpenAiConfig, AgentSpecOpenAiCompatibleConfig)) + } + self.tools_map: Dict[str, AgentSpecTool] = { + tool.name: tool + for tool in agentspec_components.values() + if isinstance(tool, AgentSpecTool) + } + self.agents_map: Dict[str, AgentSpecAgent] = { + (agent.metadata or {}).get("__crewai_agent_id__", str(agent_obj_id)): agent + for agent_obj_id, agent in agentspec_components.items() + if isinstance(agent, AgentSpecAgent) + } + # We keep a registry of conversions, so that we do not repeat the conversion for the same object twice + self.agentspec_spans_registry: Dict[str, Span] = {} + # Correlation helpers + self._agent_fingerprint_to_last_msg: Dict[str, str] = {} + # Track active tool execution spans by CrewAI agent_key + self._tool_span_by_agent_key: Dict[str, ToolExecutionSpan] = {} + # Track active agent execution spans by CrewAI agent_key + self._agent_span_by_agent_key: Dict[str, AgentExecutionSpan] = {} + # Per-agent_key tool_call_id and parent message id for correlation + self._tool_call_id_by_agent_key: Dict[str, str] = {} + self._parent_msg_by_agent_key: Dict[str, Optional[str]] = {} + # This is a reference to the parent span stack, it is needed because it must be shared + # when dealing with events, otherwise the changes to the stack performed in there, + # like span start or end, are not persisted + self._parent_context = _ACTIVE_SPAN_STACK.get() + # Events are raised and handled sometimes concurrently (especially end of previous span and start of new one), + # which makes it hard to handle the nested structure of spans + # See the `_add_event_and_handle_events_list` method for more information. + # This lock is used to manage the event list with a single thread at a time + self._lock = threading.Lock() + # This list contains all the pending events that could not be handled properly yet + self._events_list: List[CrewAIBaseEvent] = [] + + def _get_agentspec_component_from_crewai_object(self, crewai_obj: Any) -> AgentSpecComponent: + return self.agentspec_components[id(crewai_obj)] + + @contextmanager + def _parent_span_stack(self) -> Generator[None, Any, None]: + """ + Context manager that sets the span stack of the root context in the current context. + It is used because events are handled in async "threads" that have a different context, + so changes to the span stack performed in there would not be persisted and propagated to the parent context. + This way we centralize the context in this object and propagate/persist the changes across all the event handlers. + """ + _ACTIVE_SPAN_STACK.set(self._parent_context) + yield + self._parent_context = _ACTIVE_SPAN_STACK.get() + + def _handle_event(self, event: CrewAIBaseEvent) -> bool: + """ + Deal with the occurrence of the given event. + Returns True if the event is properly handled, False if the event cannot be handled. + """ + span: Span + match event: + case CrewAILiteAgentExecutionStartedEvent() | CrewAIAgentExecutionStartedEvent(): + if isinstance(event, CrewAILiteAgentExecutionStartedEvent): + agent_key = str(event.agent_info.get("id")) + else: + agent_key = str(event.agent.id) + agent = self.agents_map.get(agent_key) + if agent is None: + return False + span = AgentExecutionSpan(agent=agent) + span.start() + span.add_event(AgentExecutionStart(agent=agent, inputs={})) + self._agent_span_by_agent_key[agent_key] = span + return True + case CrewAILiteAgentExecutionCompletedEvent() | CrewAIAgentExecutionCompletedEvent(): + if not isinstance(get_current_span(), AgentExecutionSpan): + return False + if isinstance(event, CrewAILiteAgentExecutionCompletedEvent): + agent_key = str(event.agent_info.get("id")) + else: + agent_key = str(event.agent.id) + agent = self.agents_map.get(agent_key) + if agent is None: + return False + span = self._agent_span_by_agent_key[agent_key] + span.add_event( + AgentExecutionEnd( + agent=agent, + outputs={"output": event.output} if hasattr(event, "output") else {}, + ) + ) + span.end() + self._agent_span_by_agent_key.pop(agent_key, None) + return True + case CrewAILLMCallStartedEvent(): + if not isinstance(get_current_span(), AgentExecutionSpan): + return False + messages = event.messages or [] + if isinstance(messages, str): + messages = [{"content": messages}] + run_id = self._compute_chat_history_hash(messages) + model_id = self._sanitize_model_id(event.model or "") + # model_id should match an entry in the config map + llm_cfg = self.llm_configs_map.get(model_id) + if llm_cfg is None and "/" in model_id: + # Try last token as a fallback (provider differences) + llm_cfg = self.llm_configs_map.get(model_id.split("/")[-1]) + if llm_cfg is None: + raise RuntimeError( + f"Unable to find the Agent Spec LlmConfig during tracing: `{model_id}`" + ) + span = LlmGenerationSpan(id=run_id, llm_config=llm_cfg) + span.start() + span.add_event( + LlmGenerationRequest( + llm_config=span.llm_config, + llm_generation_config=span.llm_config.default_generation_parameters, + prompt=[ + AgentSpecMessage( + content=m["content"], + role=m["role"], + ) + for m in messages + ], + tools=list(self.tools_map.values()), + request_id=run_id, + ) + ) + self.agentspec_spans_registry[run_id] = span + return True + case CrewAILLMCallCompletedEvent(): + if not isinstance(get_current_span(), LlmGenerationSpan): + return False + messages = event.messages or [] + if isinstance(messages, str): + messages = [{"content": messages}] + run_id = self._compute_chat_history_hash(messages) + span = cast(LlmGenerationSpan, self.agentspec_spans_registry[run_id]) + span.add_event( + LlmGenerationResponse( + llm_config=span.llm_config, + completion_id=run_id, + content=event.response, + tool_calls=[], + request_id=run_id, + ) + ) + span.end() + self.agentspec_spans_registry.pop(run_id, None) + return True + case CrewAILLMStreamChunkEvent(): + current_span = _get_closest_span_of_given_type(LlmGenerationSpan) + if isinstance(current_span, LlmGenerationSpan): + current_span.add_event( + LlmGenerationChunkReceived( + llm_config=current_span.llm_config, + completion_id=current_span.id, + content=event.chunk, + tool_calls=[], + request_id=current_span.id, + ) + ) + return True + case CrewAIToolUsageStartedEvent(): + tool_name = event.tool_name + tool_args = event.tool_args + agent_key = event.agent_key or "" + # Correlate to current assistant message via agent fingerprint + parent_msg_id = None + if event.source_fingerprint: + parent_msg_id = self._agent_fingerprint_to_last_msg.get( + event.source_fingerprint + ) + + # Resolve tool object and create a ToolExecutionSpan + tool = self.tools_map.get(tool_name) + if tool is None: + return False + tool_span = ToolExecutionSpan(name=f"ToolExecution - {tool_name}", tool=tool) + tool_span.start() + self._tool_span_by_agent_key[agent_key] = tool_span + + # Ensure a tool_call_id for later correlation (no streaming support → always synthesize) + tool_call_id = str(uuid.uuid4()) + self._tool_call_id_by_agent_key[agent_key] = tool_call_id + self._parent_msg_by_agent_key[agent_key] = parent_msg_id + + inputs = _ensure_dict(tool_args) + tool_span.add_event( + ToolExecutionRequest( + tool=tool, + inputs=inputs, + request_id=tool_call_id, + ) + ) + return True + case CrewAIToolUsageFinishedEvent(): + if not isinstance(get_current_span(), ToolExecutionSpan): + return False + + outputs = event.output + agent_key = event.agent_key or "" + + tool_span = self._tool_span_by_agent_key[agent_key] + tool_call_id = self._tool_call_id_by_agent_key[agent_key] + if tool_span is None: + return False + + tool_span.add_event( + ToolExecutionResponse( + request_id=tool_call_id, + tool=tool_span.tool, + outputs=_ensure_dict(outputs), + ) + ) + tool_span.end() + + # Cleanup + self._tool_span_by_agent_key.pop(agent_key, None) + self._tool_call_id_by_agent_key.pop(agent_key, None) + self._parent_msg_by_agent_key.pop(agent_key, None) + + return True + return False + + def _add_event_and_handle_events_list(self, new_event: CrewAIBaseEvent) -> None: + """ + The goal of this method is to add the given event to the events list, and then try to handle + all the events in the _events_list. The reason why we need this is that the order in which some + events are emitted/handled in CrewAI is arbitrary. For example, the llm generation end and the consequent + agent execution end events are emitted at the same time, and since event handlers are executed concurrently, + there's no guarantee on the order in which those events are handled. From an Agent Spec Tracing perspective, + instead, we need to have a precise order in order to open and close spans properly, according to the span stack. + + In order to recreate this order manually, we adopt the following solution. + When an event is emitted by CrewAI, we simply add it to the list of events that should be handled. + Then we try to handle all the events in the list. The idea is that: + - If an event cannot be handled (e.g., because it's not in the correct span), it stays in the events list. + This means that another event has to happen in order to unlock this event to be handled. When that event will happen, + it will unlock this event from being handled, and that will happen. + - If the event can be handled, it is handled and popped from the list. This event being handled might unlock another event, + that will be handled as well, and so on until no event can be handled anymore, or the events list is empty. + """ + with self._lock: + # We first add the new event to the list of events to be handled. + # We use the lock to avoid changing the list that is already being modified by some other event handling + self._events_list.append(new_event) + with self._lock: + # We now take the lock again and try to handle all the events we can + events_correctly_handled = 1 + while events_correctly_handled > 0 and len(self._events_list) > 0: + event_indices_to_remove = [] + # We go over the list of events that are waiting for being handled + for i, event in enumerate(self._events_list): + # We need to ensure that we are using the right span stack contextvar + with self._parent_span_stack(): + # The events that get correctly handled, will be removed from the list, the others stay + if self._handle_event(event): + event_indices_to_remove.append(i) + events_correctly_handled = len(event_indices_to_remove) + # Remove the handled events from the list + for offset, event_index in enumerate(sorted(event_indices_to_remove)): + self._events_list.pop(event_index - offset) + + def setup_listeners(self, crewai_event_bus: CrewAIEventsBus) -> None: + """Register handlers on the global CrewAI event bus.""" + + @crewai_event_bus.on(CrewAILiteAgentExecutionStartedEvent) + def on_lite_agent_execution_started( + source: Any, event: CrewAILiteAgentExecutionStartedEvent + ) -> None: + self._add_event_and_handle_events_list(event) + + @crewai_event_bus.on(CrewAILiteAgentExecutionCompletedEvent) + def on_lite_agent_execution_finished( + source: Any, event: CrewAILiteAgentExecutionCompletedEvent + ) -> None: + self._add_event_and_handle_events_list(event) + + @crewai_event_bus.on(CrewAIAgentExecutionStartedEvent) + def on_agent_execution_started( + source: Any, event: CrewAIAgentExecutionStartedEvent + ) -> None: + self._add_event_and_handle_events_list(event) + + @crewai_event_bus.on(CrewAIAgentExecutionCompletedEvent) + def on_agent_execution_finished( + source: Any, event: CrewAIAgentExecutionCompletedEvent + ) -> None: + self._add_event_and_handle_events_list(event) + + @crewai_event_bus.on(CrewAILLMCallStartedEvent) + def on_llm_call_started(source: Any, event: CrewAILLMCallStartedEvent) -> None: + self._add_event_and_handle_events_list(event) + + @crewai_event_bus.on(CrewAILLMCallCompletedEvent) + def on_llm_call_completed(source: Any, event: CrewAILLMCallCompletedEvent) -> None: + self._add_event_and_handle_events_list(event) + + @crewai_event_bus.on(CrewAILLMStreamChunkEvent) + def on_llm_call_chunk(source: Any, event: CrewAILLMStreamChunkEvent) -> None: + self._add_event_and_handle_events_list(event) + + @crewai_event_bus.on(CrewAIToolUsageStartedEvent) + def on_tool_usage_started(source: Any, event: CrewAIToolUsageStartedEvent) -> None: + self._add_event_and_handle_events_list(event) + + @crewai_event_bus.on(CrewAIToolUsageFinishedEvent) + def on_tool_usage_finished(source: Any, event: CrewAIToolUsageFinishedEvent) -> None: + self._add_event_and_handle_events_list(event) + + @staticmethod + def _sanitize_model_id(model_id: str) -> str: + model_parts = model_id.split("/") if model_id else [] + if len(model_parts) > 1: + # Since CrewAI relies on LiteLLM, it contains the model provider at the start of the model id + # That is removed in Agent Spec conversion, so we must remove it from here too + return "/".join(model_parts[1:]) + return model_id + + @staticmethod + def _compute_chat_history_hash(messages: List[Dict[str, Any]]) -> str: + """Compute a stable UUID based on the list of messages. + + We only allow messages with role/content fields and roles in + {system,user,assistant} to mirror the frontend inputs. + """ + normalized = [ + { + "role": m["role"], + "content": str(m["content"]).replace("\r\n", "\n").replace("\r", "\n"), + } + for m in messages + ] + payload = json.dumps(normalized, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + return str(uuid.uuid5(uuid.NAMESPACE_URL, payload)) + + +class CrewAIAgentWithTracing(CrewAIAgent): + """Extension of the CrewAI agent that contains the event handler for Agent Spec Tracing""" + + _agentspec_event_listener: Optional[AgentSpecEventListener] = PrivateAttr(default=None) + + @contextmanager + def agentspec_event_listener(self) -> Generator[None, Any, None]: + """ + Context manager that yields the agent spec event listener. + + Example of usage: + + from pyagentspec.agent import Agent + + system_prompt = '''You are an expert in computer science. Please help the users with their requests.''' + agent = Agent( + name="Adaptive expert agent", + system_prompt=system_prompt, + llm_config=llm_config, + ) + + from pyagentspec.adapters.crewai import AgentSpecLoader + from pyagentspec.tracing.trace import Trace + + crewai_agent = AgentSpecLoader().load_component(agent) + with Trace(name="crewai_tracing_test"): + with crewai_agent.agentspec_event_listener(): + response = crewai_agent.kickoff(messages="Talk about the Dijkstra's algorithm") + + """ + if self._agentspec_event_listener is None: + raise RuntimeError( + "Called Agent Spec event listener context manager, but no instance was provided. " + "Please set the _agentspec_event_listener attribute first." + ) + with self._agentspec_event_listener.record_listener(): + yield diff --git a/pyagentspec/src/pyagentspec/adapters/langgraph/_langgraphconverter.py b/pyagentspec/src/pyagentspec/adapters/langgraph/_langgraphconverter.py index 275fffb0..5e7ebe73 100644 --- a/pyagentspec/src/pyagentspec/adapters/langgraph/_langgraphconverter.py +++ b/pyagentspec/src/pyagentspec/adapters/langgraph/_langgraphconverter.py @@ -5,16 +5,17 @@ # (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. -from typing import Any, Dict, List, Optional, cast +from typing import Any, Dict, List, Literal, Optional, Tuple, Union from uuid import uuid4 import httpx -from pydantic import BaseModel, Field, SecretStr, create_model +from pydantic import BaseModel, ConfigDict, Field, SecretStr, create_model from pyagentspec import Component as AgentSpecComponent from pyagentspec.adapters._utils import render_template from pyagentspec.adapters.langgraph._node_execution import NodeExecutor from pyagentspec.adapters.langgraph._types import ( + BaseCallbackHandler, BaseChatModel, Checkpointer, CompiledStateGraph, @@ -31,6 +32,7 @@ langgraph_graph, langgraph_prebuilt, ) +from pyagentspec.adapters.langgraph.tracing import AgentSpecCallbackHandler from pyagentspec.agent import Agent as AgentSpecAgent from pyagentspec.flows.edges.controlflowedge import ControlFlowEdge from pyagentspec.flows.flow import Flow as AgentSpecFlow @@ -63,50 +65,117 @@ from pyagentspec.tools import Tool as AgentSpecTool +class SchemaRegistry: + def __init__(self) -> None: + self.models: Dict[str, type[BaseModel]] = {} + + +def _build_type_from_schema( + name: str, + schema: Dict[str, Any], + registry: SchemaRegistry, +) -> Any: + # Enum -> Literal[…] + if "enum" in schema and isinstance(schema["enum"], list): + values = schema["enum"] + # Literal supports a tuple of literal values as a single subscription argument + return Literal[tuple(values)] + + # anyOf / oneOf -> Union[…] + for key in ("anyOf", "oneOf"): + if key in schema: + variants = [ + _build_type_from_schema(f"{name}Alt{i}", s, registry) + for i, s in enumerate(schema[key]) + ] + return Union[tuple(variants)] + + t = schema.get("type") + + # list of types -> Union[…] + if isinstance(t, list): + variants = [ + _build_type_from_schema(f"{name}Alt{i}", {"type": subtype}, registry) + for i, subtype in enumerate(t) + ] + return Union[tuple(variants)] + + # arrays + if t == "array": + items_schema = schema.get("items", {"type": "any"}) + item_type = _build_type_from_schema(f"{name}Item", items_schema, registry) + return List[item_type] # type: ignore + # objects + if t == "object" or ("properties" in schema or "required" in schema): + # Create or reuse a Pydantic model for this object schema + model_name = schema.get("title") or name + unique_name = model_name + suffix = 1 + while unique_name in registry.models: + suffix += 1 + unique_name = f"{model_name}_{suffix}" + + props = schema.get("properties", {}) or {} + required = set(schema.get("required", [])) + + fields: Dict[str, Tuple[Any, Any]] = {} + for prop_name, prop_schema in props.items(): + prop_type = _build_type_from_schema(f"{unique_name}_{prop_name}", prop_schema, registry) + desc = prop_schema.get("description") + default_field = ( + Field(..., description=desc) + if prop_name in required + else Field(None, description=desc) + ) + fields[prop_name] = (prop_type, default_field) + + # Enforce additionalProperties: False (extra=forbid) + extra_forbid = schema.get("additionalProperties") is False + model_kwargs: Dict[str, Any] = {} + if extra_forbid: + # Pydantic v2: pass a ConfigDict/dict into __config__ + model_kwargs["__config__"] = ConfigDict(extra="forbid") + + model_cls = create_model(unique_name, **fields, **model_kwargs) # type: ignore + registry.models[unique_name] = model_cls + return model_cls + + # primitives / fallback + mapping = { + "string": str, + "number": float, + "integer": int, + "boolean": bool, + "null": type(None), + "any": Any, + None: Any, + "": Any, + } + return mapping.get(t, Any) + + def _create_pydantic_model_from_properties( model_name: str, properties: List[AgentSpecProperty] ) -> type[BaseModel]: - # Create a pydantic model whose attributes are the given properties - fields: Dict[str, Any] = {} + registry = SchemaRegistry() + fields: Dict[str, Tuple[Any, Any]] = {} + for property_ in properties: - field_parameters: Dict[str, Any] = {} - param_name = property_.title - if property_.default is not _agentspec_empty_default: - field_parameters["default"] = property_.default - if property_.description: - field_parameters["description"] = property_.description - annotation = _json_schema_type_to_python_annotation(property_.json_schema) - fields[param_name] = (annotation, Field(**field_parameters)) - return cast(type[BaseModel], create_model(model_name, **fields)) + # Build the annotation from the json_schema (handles enum/array/object/etc.) + annotation = _build_type_from_schema(property_.title, property_.json_schema, registry) + field_params: Dict[str, Any] = {} + if property_.description: + field_params["description"] = property_.description -def _json_schema_type_to_python_annotation(json_schema: Dict[str, Any]) -> str: - if "anyOf" in json_schema: - possible_types = set( - _json_schema_type_to_python_annotation(inner_json_schema_type) - for inner_json_schema_type in json_schema["anyOf"] - ) - return f"Union[{','.join(possible_types)}]" - json_schema_type = json_schema.get("type", "") - if isinstance(json_schema_type, list): - possible_types = set( - _json_schema_type_to_python_annotation(inner_json_schema_type) - for inner_json_schema_type in json_schema_type - ) - return f"Union[{','.join(possible_types)}]" + if property_.default is not _agentspec_empty_default: + default_field = Field(property_.default, **field_params) + else: + default_field = Field(..., **field_params) - if json_schema_type == "array": - return f"List[{_json_schema_type_to_python_annotation(json_schema['items'])}]" - mapping = { - "string": "str", - "number": "float", - "integer": "int", - "boolean": "bool", - "null": "None", - "object": "Dict[str, Any]", - } + fields[property_.title] = (annotation, default_field) - return mapping.get(json_schema_type, "Any") + return create_model(model_name, **fields) # type: ignore class AgentSpecToLangGraphConverter: @@ -121,11 +190,14 @@ def convert( """Convert the given PyAgentSpec component object into the corresponding LangGraph component""" if converted_components is None: converted_components = {} - if config is None and checkpointer is not None: - config = RunnableConfig({"configurable": {"thread_id": str(uuid4())}}) + if config is None: + if checkpointer is not None: + config = RunnableConfig({"configurable": {"thread_id": str(uuid4())}}) + else: + config = RunnableConfig({}) if agentspec_component.id not in converted_components: converted_components[agentspec_component.id] = self._convert( - agentspec_component, tool_registry, converted_components, checkpointer, config # type: ignore[arg-type] + agentspec_component, tool_registry, converted_components, checkpointer, config ) return converted_components[agentspec_component.id] @@ -138,17 +210,24 @@ def _convert( config: RunnableConfig, ) -> Any: if isinstance(agentspec_component, AgentSpecAgent): + callback = AgentSpecCallbackHandler( + llm_config=agentspec_component.llm_config, + tools=agentspec_component.tools, + ) + config_with_callbacks = _add_callback_to_runnable_config(callback, config) return self._agent_convert_to_langgraph( agentspec_component, tool_registry=tool_registry, converted_components=converted_components, checkpointer=checkpointer, - config=config, + config=config_with_callbacks, ) elif isinstance(agentspec_component, AgentSpecLlmConfig): - return self._llm_convert_to_langgraph(agentspec_component) + return self._llm_convert_to_langgraph(agentspec_component, config=config) elif isinstance(agentspec_component, AgentSpecServerTool): - return self._server_tool_convert_to_langgraph(agentspec_component, tool_registry) + return self._server_tool_convert_to_langgraph( + agentspec_component, tool_registry, config=config + ) elif isinstance(agentspec_component, AgentSpecClientTool): if checkpointer is None: raise ValueError( @@ -156,7 +235,7 @@ def _convert( ) return self._client_tool_convert_to_langgraph(agentspec_component) elif isinstance(agentspec_component, AgentSpecRemoteTool): - return self._remote_tool_convert_to_langgraph(agentspec_component) + return self._remote_tool_convert_to_langgraph(agentspec_component, config=config) elif isinstance(agentspec_component, AgentSpecFlow): return self._flow_convert_to_langgraph( agentspec_component, @@ -483,6 +562,7 @@ def _start_node_convert_to_langgraph(self, start_node: AgentSpecStartNode) -> "N def _remote_tool_convert_to_langgraph( self, remote_tool: AgentSpecRemoteTool, + config: RunnableConfig, ) -> LangGraphTool: def _remote_tool(**kwargs: Any) -> Any: remote_tool_data = {k: render_template(v, kwargs) for k, v in remote_tool.data.items()} @@ -513,6 +593,7 @@ def _remote_tool(**kwargs: Any) -> Any: description=remote_tool.description or "", args_schema=args_model, func=_remote_tool, + callbacks=config.get("callbacks"), ) return structured_tool @@ -520,6 +601,7 @@ def _server_tool_convert_to_langgraph( self, agentspec_server_tool: AgentSpecServerTool, tool_registry: Dict[str, LangGraphTool], + config: RunnableConfig, ) -> LangGraphTool: # Ensure the tool exists in the registry if agentspec_server_tool.name not in tool_registry: @@ -547,6 +629,7 @@ def _server_tool_convert_to_langgraph( description=description, args_schema=args_model, # model class, not a dict func=tool_obj, + callbacks=config.get("callbacks"), ) return wrapped @@ -674,7 +757,9 @@ def _agent_convert_to_langgraph( config=config, ) - def _llm_convert_to_langgraph(self, llm_config: AgentSpecLlmConfig) -> BaseChatModel: + def _llm_convert_to_langgraph( + self, llm_config: AgentSpecLlmConfig, config: RunnableConfig + ) -> BaseChatModel: """Create the LLM model object for the chosen llm configuration.""" generation_config: Dict[str, Any] = {} generation_parameters = llm_config.default_generation_parameters @@ -696,6 +781,7 @@ def _llm_convert_to_langgraph(self, llm_config: AgentSpecLlmConfig) -> BaseChatM api_key=SecretStr("EMPTY"), base_url=_prepare_openai_compatible_url(llm_config.url), use_responses_api=use_responses_api, + callbacks=config.get("callbacks"), **generation_config, ) elif isinstance(llm_config, OllamaConfig): @@ -709,6 +795,7 @@ def _llm_convert_to_langgraph(self, llm_config: AgentSpecLlmConfig) -> BaseChatM return ChatOllama( base_url=llm_config.url, model=llm_config.model_id, + callbacks=config.get("callbacks"), **generation_config, ) elif isinstance(llm_config, OpenAiConfig): @@ -717,6 +804,7 @@ def _llm_convert_to_langgraph(self, llm_config: AgentSpecLlmConfig) -> BaseChatM return ChatOpenAI( model=llm_config.model_id, use_responses_api=use_responses_api, + callbacks=config.get("callbacks"), **generation_config, ) elif isinstance(llm_config, OpenAiCompatibleConfig): @@ -726,6 +814,7 @@ def _llm_convert_to_langgraph(self, llm_config: AgentSpecLlmConfig) -> BaseChatM model=llm_config.model_id, base_url=_prepare_openai_compatible_url(llm_config.url), use_responses_api=use_responses_api, + callbacks=config.get("callbacks"), **generation_config, ) else: @@ -760,3 +849,16 @@ def _prepare_openai_compatible_url(url: str) -> str: final_url = urlunparse(v1_url_parts) return str(final_url) + + +def _add_callback_to_runnable_config( + callback: BaseCallbackHandler, config: RunnableConfig +) -> RunnableConfig: + callbacks = [callback] + existing_callbacks = config.get("callbacks") + if not existing_callbacks: + existing_callbacks = [] + if isinstance(existing_callbacks, list): + existing_callbacks = existing_callbacks + callbacks + config_with_callbacks = RunnableConfig({**config, "callbacks": existing_callbacks}) + return config_with_callbacks diff --git a/pyagentspec/src/pyagentspec/adapters/langgraph/_types.py b/pyagentspec/src/pyagentspec/adapters/langgraph/_types.py index c6b880e7..f6f7c7d6 100644 --- a/pyagentspec/src/pyagentspec/adapters/langgraph/_types.py +++ b/pyagentspec/src/pyagentspec/adapters/langgraph/_types.py @@ -26,6 +26,7 @@ import langgraph.prebuilt as langgraph_prebuilt import langgraph.types as langgraph_types import langgraph_core # type: ignore + from langchain_core.callbacks import BaseCallbackHandler from langchain_core.language_models import BaseChatModel from langchain_core.messages import BaseMessage, SystemMessage from langchain_core.runnables import RunnableBinding, RunnableConfig @@ -54,6 +55,7 @@ StateGraph = langgraph_graph.StateGraph Messages = LazyLoader("langgraph.graph.message").Messages CompiledStateGraph = LazyLoader("langgraph.graph.state").CompiledStateGraph + BaseCallbackHandler = LazyLoader("langchain_core.callbacks").BaseCallbackHandler RunnableBinding = LazyLoader("langchain_core.runnables").RunnableBinding RunnableConfig = LazyLoader("langchain_core.runnables").RunnableConfig StateNodeSpec = LazyLoader("langgraph.graph._node").StateNodeSpec @@ -150,4 +152,5 @@ class FlowOutputSchema(TypedDict): "RunnableConfig", "Messages", "BranchSpec", + "BaseCallbackHandler", ] diff --git a/pyagentspec/src/pyagentspec/adapters/langgraph/tracing.py b/pyagentspec/src/pyagentspec/adapters/langgraph/tracing.py new file mode 100644 index 00000000..77dba866 --- /dev/null +++ b/pyagentspec/src/pyagentspec/adapters/langgraph/tracing.py @@ -0,0 +1,354 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +import ast +import json +import typing +from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union +from uuid import UUID + +from langchain_core.callbacks import BaseCallbackHandler as LangchainBaseCallbackHandler +from langchain_core.messages import BaseMessage, ToolMessage +from langchain_core.outputs import ChatGenerationChunk, GenerationChunk, LLMResult +from typing_extensions import NotRequired + +from pyagentspec.llms.llmconfig import LlmConfig as AgentSpecLlmConfig +from pyagentspec.tools import Tool as AgentSpecTool +from pyagentspec.tracing.events import ( + LlmGenerationChunkReceived as AgentSpecLlmGenerationChunkReceived, +) +from pyagentspec.tracing.events import LlmGenerationRequest as AgentSpecLlmGenerationRequest +from pyagentspec.tracing.events import LlmGenerationResponse as AgentSpecLlmGenerationResponse +from pyagentspec.tracing.events import ToolExecutionRequest as AgentSpecToolExecutionRequest +from pyagentspec.tracing.events import ToolExecutionResponse as AgentSpecToolExecutionResponse +from pyagentspec.tracing.events.llmgeneration import ToolCall as AgentSpecToolCall +from pyagentspec.tracing.messages.message import Message as AgentSpecMessage +from pyagentspec.tracing.spans import LlmGenerationSpan as AgentSpecLlmGenerationSpan +from pyagentspec.tracing.spans import Span as AgentSpecSpan +from pyagentspec.tracing.spans import ToolExecutionSpan as AgentSpecToolExecutionSpan + +MessageInProgress = TypedDict( + "MessageInProgress", + { + "id": str, # chunk.message.id + "tool_call_id": NotRequired[str], + "tool_call_name": NotRequired[str], + }, +) + +MessagesInProgressRecord = Dict[Union[str, UUID], MessageInProgress] # keys are run_id + + +LANGCHAIN_ROLES_TO_OPENAI_ROLES = { + "human": "user", + "ai": "assistant", + "tool": "tool", + "system": "system", +} + + +class AgentSpecCallbackHandler(LangchainBaseCallbackHandler): + + def __init__( + self, + llm_config: AgentSpecLlmConfig, + tools: Optional[List[AgentSpecTool]] = None, + ) -> None: + # This is only added during tool-call streaming to associate run_id with tool_call_id + # (tool_call_id is not available mid-stream) + self.messages_in_process: MessagesInProgressRecord = {} + # Track spans per run_id + self.agentspec_spans_registry: Dict[str, AgentSpecSpan] = {} + # configs for spans + self.llm_config = llm_config + self.tools_map: Dict[str, AgentSpecTool] = {t.name: t for t in (tools or [])} + + def _get_or_start_llm_span(self, run_id_str: str) -> AgentSpecLlmGenerationSpan: + span = self.agentspec_spans_registry.get(run_id_str) + if not isinstance(span, AgentSpecLlmGenerationSpan): + span = AgentSpecLlmGenerationSpan(llm_config=self.llm_config) + self.agentspec_spans_registry[run_id_str] = span + span.start() + return span + + def on_chat_model_start( + self, + serialized: Dict[str, Any], + messages: List[List[BaseMessage]], + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> Any: + run_id_str = str(run_id) + # Start an LLM span and emit request with prompt and tools + span = self._get_or_start_llm_span(run_id_str) + + # not sure why it is a list of lists, assert that the outer list is size 1 + if len(messages) != 1: + raise ValueError( + f"[on_chat_model_start] langchain messages is a nested list of list of BaseMessage, " + "expected the outer list to have size one but got size {len(messages)}" + ) + list_of_messages = messages[0] + + prompt = [ + AgentSpecMessage( + content=_ensure_string(m.content), + sender="", + role=LANGCHAIN_ROLES_TO_OPENAI_ROLES[m.type], + ) + for m in list_of_messages + ] + + tools = list(self.tools_map.values()) if self.tools_map else [] + + span.add_event( + AgentSpecLlmGenerationRequest( + request_id=run_id_str, + llm_config=self.llm_config, + llm_generation_config=self.llm_config.default_generation_parameters, + prompt=prompt, + tools=tools, + ) + ) + + def on_llm_new_token( + self, + token: str, + *, + chunk: Optional[Union[ChatGenerationChunk, GenerationChunk]] = None, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + tags: Optional[List[str]] = None, + **kwargs: Any, + ) -> Any: + # streaming: can stream text chunks and/or tool_call_chunks + + # tool call chunks explanation: + # shape: chunk.message.tool_call_chunks (can be empty) + # if not empty: it is a list of length 1 + # for each on_llm_new_token invocation: + # the first chunk would contain id and name, and empty args + # the next chunks would not contain id and name, only args (deltas) + + # text chunks explanation: + # shape: chunk.message.content contains the deltas + + # expected behavior: + # it should emit LlmGenerationChunkReceived and ToolCallChunkReceived + # NOTE: on_llm_new_token seems to be called a few times at the beginning with empty everything except for the id=run--id894224... + if chunk is None: + raise ValueError("[on_llm_new_token] Expected chunk to not be None") + run_id_str = str(run_id) + span = self._get_or_start_llm_span(run_id_str) + chunk_message = chunk.message # type: ignore + + # Note that chunk_message.response_metadata.id is None during streaming, but it's populated when not streaming + + if not isinstance(chunk_message.id, str): + raise ValueError( + f"[on_llm_new_token] Expected chunk_message.id to be a string but got: {type(chunk_message.id)}" + ) + message_id = chunk_message.id + + agentspec_tool_calls: List[AgentSpecToolCall] = [] + tool_call_chunks = chunk_message.tool_call_chunks or [] # type: ignore + if tool_call_chunks: + if len(tool_call_chunks) != 1: + raise ValueError( + "[on_llm_new_token] Expected exactly one tool call chunk " + f"if streaming tool calls, but got: {tool_call_chunks}" + ) + tool_call_chunk = tool_call_chunks[0] + tool_name, tool_args, call_id = ( + tool_call_chunk["name"], + tool_call_chunk["args"], + tool_call_chunk["id"], + ) + if call_id is None: + current_stream = self.messages_in_process[run_id] + tool_name, call_id = ( + current_stream["tool_call_name"], + current_stream["tool_call_id"], + ) + else: + self.messages_in_process[run_id] = { + "id": message_id, + "tool_call_id": call_id, + "tool_call_name": tool_name, + } + agentspec_tool_calls = [ + AgentSpecToolCall(call_id=call_id, tool_name=tool_name, arguments=tool_args or "") + ] + + span.add_event( + AgentSpecLlmGenerationChunkReceived( + request_id=run_id_str, + completion_id=message_id, + content=_ensure_string(chunk_message.content or ""), + llm_config=self.llm_config, + tool_calls=agentspec_tool_calls, + ) + ) + + @typing.no_type_check + def on_llm_end( + self, + response: LLMResult, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> Any: + run_id = str(run_id) + span = self._get_or_start_llm_span(run_id) + message_id, content, tool_calls = _extract_message_content_and_tool_calls(response) + span.add_event( + AgentSpecLlmGenerationResponse( + llm_config=self.llm_config, + request_id=run_id, + completion_id=message_id, + content=content, + tool_calls=tool_calls, + ) + ) + span.end() + self.agentspec_spans_registry.pop(run_id, None) + self.messages_in_process.pop(run_id, None) + + def on_tool_start( + self, + serialized: Dict[str, Any], + input_str: str, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> Any: + if kwargs.get("tool_call_id"): + # note that this run_id is different from the run_id in LLM events + # so we cannot use it to correlate with tool_call_id above + raise NotImplementedError( + "[on_tool_start] This is implemented starting from langchain 1.1.2, and we should support it" + ) + # get run_id and tool config + run_id_str = str(run_id) + tool_name = serialized.get("name") + if not tool_name: + raise ValueError("[on_tool_start] Expected tool name in serialized metadata") + tool_obj = self.tools_map.get(tool_name) + if tool_obj is None: + raise ValueError(f"[on_tool_start] Unknown tool: {tool_name}") + + # starting a tool span for this tool + tool_span = AgentSpecToolExecutionSpan(tool=tool_obj) + self.agentspec_spans_registry[run_id_str] = tool_span + tool_span.start() + + inputs: Dict[str, Any] = ( + ast.literal_eval(input_str) if isinstance(input_str, str) else input_str + ) + # instead of the real tool_call_id, we use the run_id to correlate between tool request and tool result + tool_span.add_event( + AgentSpecToolExecutionRequest(request_id=run_id_str, tool=tool_span.tool, inputs=inputs) + ) + + def on_tool_end( + self, + output: Any, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> Any: + if not isinstance(output, ToolMessage): + raise ValueError("[on_tool_end] Expected ToolMessage for tool end") + run_id_str = str(run_id) + tool_span = self.agentspec_spans_registry.get(run_id_str) + + try: + parsed = ( + json.loads(output.content) if isinstance(output.content, str) else output.content + ) + except json.JSONDecodeError as e: + parsed = str(output.content) + outputs = parsed if isinstance(parsed, dict) else {"output": parsed} + + if not isinstance(tool_span, AgentSpecToolExecutionSpan): + raise ValueError( + f"Expected tool_span to be a ToolExecutionSpan but got {type(tool_span)}" + ) + + tool_span.add_event( + AgentSpecToolExecutionResponse( + request_id=output.tool_call_id, + tool=tool_span.tool, + outputs=outputs, + ) + ) + tool_span.end() + self.agentspec_spans_registry.pop(run_id_str, None) + + def on_tool_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + **kwargs: Any, + ) -> None: + raise error + + +def _ensure_string(obj: Any) -> str: + if obj is None: + raise ValueError("can only coerce non-string objects to string") + if not isinstance(obj, str): + try: + return str(obj) + except: + raise ValueError(f"obj is not a valid JSON dict: {obj}") + return obj + + +@typing.no_type_check +def _extract_message_content_and_tool_calls( + response: LLMResult, +) -> Tuple[str, str, List[AgentSpecToolCall]]: + """ + Returns content, tool_calls + """ + if len(response.generations) != 1 or len(response.generations[0]) != 1: + raise ValueError("Expected response to contain one generation and one chat_generation") + chat_generation = response.generations[0][0] + finish_reason = chat_generation.generation_info["finish_reason"] + content = chat_generation.message.content + tool_calls = chat_generation.message.additional_kwargs.get("tool_calls", []) + # NOTE: content can be empty (empty string "") + # in that case, chat_generation.generation_info["finish_reason"] is "tool_calls" + # and tool_calls should not be empty + if content == "" and not tool_calls: + raise ValueError("Expected tool_calls to not be empty when content is empty") + content = _ensure_string(content) + agentspec_tool_calls = [_build_agentspec_tool_call(tc) for tc in tool_calls] + # if streaming, response_id is not provided, must rely on run_id + run_id = chat_generation.message.id + completion_id = chat_generation.message.response_metadata.get("id") + message_id = run_id or completion_id + return message_id, content, agentspec_tool_calls + + +def _build_agentspec_tool_call(tool_call: Dict[str, Any]) -> AgentSpecToolCall: + tc_id = tool_call["id"] + if "function" in tool_call: + tool_call: Dict[str, Any] = tool_call["function"] # type: ignore[no-redef] + args_key = "arguments" + else: + args_key = "args" + tc_name = tool_call["name"] + tc_args = _ensure_string(tool_call[args_key]) + return AgentSpecToolCall(call_id=tc_id, tool_name=tc_name, arguments=tc_args) diff --git a/pyagentspec/src/pyagentspec/llms/openaiconfig.py b/pyagentspec/src/pyagentspec/llms/openaiconfig.py index 9ed26df8..331b299d 100644 --- a/pyagentspec/src/pyagentspec/llms/openaiconfig.py +++ b/pyagentspec/src/pyagentspec/llms/openaiconfig.py @@ -6,9 +6,12 @@ """Defines the class for configuring how to connect to a LLM hosted by a vLLM instance.""" +from typing import Optional + from pyagentspec.component import SerializeAsEnum from pyagentspec.llms.llmconfig import LlmConfig from pyagentspec.llms.openaicompatibleconfig import OpenAIAPIType +from pyagentspec.sensitive_field import SensitiveField from pyagentspec.versioning import AgentSpecVersionEnum @@ -24,6 +27,9 @@ class OpenAiConfig(LlmConfig): api_type: SerializeAsEnum[OpenAIAPIType] = OpenAIAPIType.CHAT_COMPLETIONS """OpenAI API protocol to use""" + api_key: SensitiveField[Optional[str]] = None + """An optional API KEY for the remote LLM model. If specified, the value of the api_key will be + excluded and replaced by a reference when exporting the configuration.""" def _versioned_model_fields_to_exclude( self, agentspec_version: AgentSpecVersionEnum @@ -31,11 +37,15 @@ def _versioned_model_fields_to_exclude( fields_to_exclude = set() if agentspec_version < AgentSpecVersionEnum.v25_4_2: fields_to_exclude.add("api_type") + fields_to_exclude.add("api_key") return fields_to_exclude def _infer_min_agentspec_version_from_configuration(self) -> AgentSpecVersionEnum: parent_min_version = super()._infer_min_agentspec_version_from_configuration() current_object_min_version = self.min_agentspec_version + if self.api_key is not None: + # `api_key` is only introduced starting from 25.4.2 + current_object_min_version = AgentSpecVersionEnum.v25_4_2 if self.api_type != OpenAIAPIType.CHAT_COMPLETIONS: # If the api type is not chat completions, then we need to use the new AgentSpec version # If not, the old version will work as it was the de-facto diff --git a/pyagentspec/src/pyagentspec/tracing/__init__.py b/pyagentspec/src/pyagentspec/tracing/__init__.py new file mode 100644 index 00000000..a66f2195 --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/__init__.py @@ -0,0 +1,7 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +"""This module and its submodules define all the Agent Spec Tracing components and utilities.""" diff --git a/pyagentspec/src/pyagentspec/tracing/_basemodel.py b/pyagentspec/src/pyagentspec/tracing/_basemodel.py new file mode 100644 index 00000000..37afa1ee --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/_basemodel.py @@ -0,0 +1,44 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from typing import Any, Dict + +from pydantic import BaseModel + +from pyagentspec.sensitive_field import is_sensitive_field +from pyagentspec.serialization.serializationcontext import _SerializationContextImpl +from pyagentspec.versioning import AgentSpecVersionEnum + +_PII_MASK = "** MASKED **" + + +class _TracingSerializationContextImpl(_SerializationContextImpl): + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.agentspec_version = AgentSpecVersionEnum.current_version + + +class BaseModelWithSensitiveInfo(BaseModel): + + def model_dump(self, *args: Any, **kwargs: Any) -> Dict[str, Any]: + """ + Serialize a Pydantic Component masking sensitive information. + + Is invoked upon a ``model_dump`` call. + """ + mask_sensitive_information = kwargs.pop("mask_sensitive_information", True) + if "context" not in kwargs: + kwargs["context"] = _TracingSerializationContextImpl() + serialized_model_dict = super().model_dump(*args, **kwargs) + for field_name, field_info in self.__class__.model_fields.items(): + if field_name in serialized_model_dict: + if getattr(field_info, "exclude", False): + serialized_model_dict.pop(field_name) + elif mask_sensitive_information and is_sensitive_field(field_info): + serialized_model_dict[field_name] = _PII_MASK + serialized_model_dict["type"] = self.__class__.__name__ + return serialized_model_dict diff --git a/pyagentspec/src/pyagentspec/tracing/events/__init__.py b/pyagentspec/src/pyagentspec/tracing/events/__init__.py new file mode 100644 index 00000000..b4854084 --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/events/__init__.py @@ -0,0 +1,49 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from .agent import AgentExecutionEnd, AgentExecutionStart +from .event import Event +from .exception import ExceptionRaised +from .flow import FlowExecutionEnd, FlowExecutionStart +from .humanintheloop import HumanInTheLoopRequest, HumanInTheLoopResponse +from .llmgeneration import ( + LlmGenerationChunkReceived, + LlmGenerationRequest, + LlmGenerationResponse, +) +from .managerworkers import ManagerWorkersExecutionEnd, ManagerWorkersExecutionStart +from .node import NodeExecutionEnd, NodeExecutionStart +from .swarm import SwarmExecutionEnd, SwarmExecutionStart +from .tool import ( + ToolConfirmationRequest, + ToolConfirmationResponse, + ToolExecutionRequest, + ToolExecutionResponse, +) + +__all__ = [ + "AgentExecutionStart", + "AgentExecutionEnd", + "Event", + "ExceptionRaised", + "LlmGenerationRequest", + "LlmGenerationResponse", + "LlmGenerationChunkReceived", + "ToolConfirmationRequest", + "ToolConfirmationResponse", + "ToolExecutionRequest", + "ToolExecutionResponse", + "NodeExecutionStart", + "NodeExecutionEnd", + "FlowExecutionStart", + "FlowExecutionEnd", + "ManagerWorkersExecutionStart", + "ManagerWorkersExecutionEnd", + "SwarmExecutionStart", + "SwarmExecutionEnd", + "HumanInTheLoopRequest", + "HumanInTheLoopResponse", +] diff --git a/pyagentspec/src/pyagentspec/tracing/events/agent.py b/pyagentspec/src/pyagentspec/tracing/events/agent.py new file mode 100644 index 00000000..00cefaa1 --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/events/agent.py @@ -0,0 +1,31 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from typing import Any, Dict + +from pyagentspec import Agent +from pyagentspec.sensitive_field import SensitiveField +from pyagentspec.tracing.events.event import Event + + +class AgentExecutionStart(Event): + """The execution of an agent is starting. Emitted when an AgentExecutionSpan starts.""" + + agent: Agent + "The Agent being executed" + + inputs: SensitiveField[Dict[str, Any]] + "The inputs used for the agent's execution, one per property defined in agent's inputs" + + +class AgentExecutionEnd(Event): + """The execution of an agent is ending. Emitted when an AgentExecutionSpan ends.""" + + agent: Agent + "The Agent being executed" + + outputs: SensitiveField[Dict[str, Any]] + "The outputs generated by the agent's execution, one per property defined in agent's outputs" diff --git a/pyagentspec/src/pyagentspec/tracing/events/event.py b/pyagentspec/src/pyagentspec/tracing/events/event.py new file mode 100644 index 00000000..c9e41edb --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/events/event.py @@ -0,0 +1,33 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +import time +import uuid +from typing import Any, Optional + +from pydantic import Field + +from pyagentspec.tracing._basemodel import BaseModelWithSensitiveInfo + + +class Event(BaseModelWithSensitiveInfo): + + id: str = Field(default_factory=lambda: str(uuid.uuid4()), frozen=True) + """A unique identifier for the event""" + name: Optional[str] = None + """The name of the event. If None, the event class name is used.""" + description: str = "" + """The description of the event.""" + timestamp: int = Field(default_factory=time.time_ns) + """The timestamp of when the event occurred""" + metadata: dict[str, Any] = Field(default_factory=dict) + """Metadata related to the event""" + + def model_post_init(self, __context: Any) -> None: + """Set the default name if it is not provided.""" + super().model_post_init(__context) + if not self.name: + self.name = self.__class__.__name__ diff --git a/pyagentspec/src/pyagentspec/tracing/events/exception.py b/pyagentspec/src/pyagentspec/tracing/events/exception.py new file mode 100644 index 00000000..ab18824f --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/events/exception.py @@ -0,0 +1,23 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from pyagentspec.sensitive_field import SensitiveField +from pyagentspec.tracing.events.event import Event + + +class ExceptionRaised(Event): + """ + This event is recorded whenever an exception occurs. + """ + + exception_type: str + """Type of the exception""" + + exception_message: SensitiveField[str] + """Message of the exception""" + + exception_stacktrace: SensitiveField[str] = "" + """Stacktrace of the exception""" diff --git a/pyagentspec/src/pyagentspec/tracing/events/flow.py b/pyagentspec/src/pyagentspec/tracing/events/flow.py new file mode 100644 index 00000000..3b140d39 --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/events/flow.py @@ -0,0 +1,34 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from typing import Any, Dict + +from pyagentspec.flows.flow import Flow +from pyagentspec.sensitive_field import SensitiveField +from pyagentspec.tracing.events.event import Event + + +class FlowExecutionStart(Event): + """The execution of a flow is starting. Emitted when a FlowExecutionSpan starts.""" + + flow: Flow + "The Flow being executed" + + inputs: SensitiveField[Dict[str, Any]] + "The inputs used for the flow's execution, one per property defined in flow's inputs" + + +class FlowExecutionEnd(Event): + """The execution of a flow is ending. Emitted at a FlowExecutionSpan ends.""" + + flow: Flow + "The Flow being executed" + + outputs: SensitiveField[Dict[str, Any]] + "The outputs generated by the flow's execution, one per property defined in flow's outputs" + + branch_selected: str + "The exit branch selected at the end of the Flow's execution" diff --git a/pyagentspec/src/pyagentspec/tracing/events/humanintheloop.py b/pyagentspec/src/pyagentspec/tracing/events/humanintheloop.py new file mode 100644 index 00000000..8a5afc54 --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/events/humanintheloop.py @@ -0,0 +1,30 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from typing import Any, Dict + +from pyagentspec.sensitive_field import SensitiveField +from pyagentspec.tracing.events.event import Event + + +class HumanInTheLoopRequest(Event): + """A human-in-the-loop (HITL) intervention is required. Emitted when the execution is interrupted due to HITL request""" + + request_id: str + "Identifier of the human-in-the-loop request" + + content: SensitiveField[Dict[str, Any]] + "The content of the request forwarded to the user" + + +class HumanInTheLoopResponse(Event): + """A human-in-the-loop response is provided. Emitted when the execution restarts after HITL response.""" + + request_id: str + "Identifier of the human-in-the-loop request" + + content: SensitiveField[Dict[str, Any]] + "The content of the response received from the user" diff --git a/pyagentspec/src/pyagentspec/tracing/events/llmgeneration.py b/pyagentspec/src/pyagentspec/tracing/events/llmgeneration.py new file mode 100644 index 00000000..fac7f6c0 --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/events/llmgeneration.py @@ -0,0 +1,93 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from typing import List, Optional + +from pydantic import BaseModel + +from pyagentspec.llms import LlmConfig, LlmGenerationConfig +from pyagentspec.sensitive_field import SensitiveField +from pyagentspec.tools import Tool +from pyagentspec.tracing.events.event import Event +from pyagentspec.tracing.messages.message import Message + + +class ToolCall(BaseModel): + """Model for an LLM tool call.""" + + call_id: str + "Identifier of the tool call" + + tool_name: str + "The name of the tool that should be called" + + arguments: str + "The values of the arguments that should be passed to the tool, in JSON format" + + +class LlmGenerationRequest(Event): + """An LLM generation request was received. Start of the LlmGenerationSpan.""" + + llm_config: LlmConfig + "The LlmConfig that performs the generation" + + prompt: SensitiveField[List[Message]] + "The content of the prompt that will be sent to the LLM." + + tools: List[Tool] + "The list of tools sent as part of the generation request" + + request_id: str + "Identifier of the generation request" + + llm_generation_config: Optional[LlmGenerationConfig] = None + "The LLM configuration used for this LLM call" + + +class LlmGenerationResponse(Event): + """An LLM response was received. End of an LlmGenerationSpan.""" + + llm_config: LlmConfig + "The LlmConfig that performed the generation" + + content: SensitiveField[Optional[str]] + "The content of the response received from the LLM" + + tool_calls: SensitiveField[List[ToolCall]] = [] + "The list of tool calls that should be performed, received as part of the generation response" + + request_id: str + "Identifier of the generation request" + + completion_id: Optional[str] = None + "The identifier of the completion related to this response" + + input_tokens: Optional[int] = None + "Number of input tokens" + + output_tokens: Optional[int] = None + "Number of output tokens" + + +class LlmGenerationChunkReceived(Event): + + llm_config: LlmConfig + "The LlmConfig that performs the generation" + + content: SensitiveField[Optional[str]] + "The content of the chunk received from the LLM" + + request_id: str + "Identifier of the generation request" + + tool_calls: SensitiveField[List[ToolCall]] = [] + "The list of tool calls that should be performed, received as part of the generation response chunk" + + completion_id: Optional[str] = None + "The identifier of the completion related to this response chunk" + + output_tokens: Optional[int] = None + "Number of output tokens for this chunk" diff --git a/pyagentspec/src/pyagentspec/tracing/events/managerworkers.py b/pyagentspec/src/pyagentspec/tracing/events/managerworkers.py new file mode 100644 index 00000000..119f55ec --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/events/managerworkers.py @@ -0,0 +1,31 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from typing import Any, Dict + +from pyagentspec.managerworkers import ManagerWorkers +from pyagentspec.sensitive_field import SensitiveField +from pyagentspec.tracing.events.event import Event + + +class ManagerWorkersExecutionStart(Event): + """The execution of a manager-workers is starting. Emitted when a ManagerWorkersExecutionSpan starts""" + + managerworkers: ManagerWorkers + "The ManagerWorkers being executed" + + inputs: SensitiveField[Dict[str, Any]] + "The inputs used for the manager-workers's execution, one per property defined in manager-workers's inputs" + + +class ManagerWorkersExecutionEnd(Event): + """The execution of a manager-workers is ending. Emitted when a ManagerWorkersExecutionSpan ends.""" + + managerworkers: ManagerWorkers + "The ManagerWorkers being executed" + + outputs: SensitiveField[Dict[str, Any]] + "The outputs generated by the manager-workers's execution, one per property defined in manager-workers's outputs" diff --git a/pyagentspec/src/pyagentspec/tracing/events/node.py b/pyagentspec/src/pyagentspec/tracing/events/node.py new file mode 100644 index 00000000..90ae661a --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/events/node.py @@ -0,0 +1,34 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from typing import Any, Dict + +from pyagentspec.flows.node import Node +from pyagentspec.sensitive_field import SensitiveField +from pyagentspec.tracing.events.event import Event + + +class NodeExecutionStart(Event): + """The execution of a node is starting. Emitted ad a NodeExecutionSpan starts.""" + + node: Node + "The Node being executed" + + inputs: SensitiveField[Dict[str, Any]] + "The inputs used for the node's execution, one per property defined in node's inputs" + + +class NodeExecutionEnd(Event): + """The execution of a node is ending. Emitted at a NodeExecutionSpan ends.""" + + node: Node + "The Node being executed" + + outputs: SensitiveField[Dict[str, Any]] + "The outputs generated by the node's execution, one per property defined in node's outputs" + + branch_selected: str + "The exit branch selected at the end of the Node's execution" diff --git a/pyagentspec/src/pyagentspec/tracing/events/swarm.py b/pyagentspec/src/pyagentspec/tracing/events/swarm.py new file mode 100644 index 00000000..ef3d7d82 --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/events/swarm.py @@ -0,0 +1,31 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from typing import Any, Dict + +from pyagentspec.sensitive_field import SensitiveField +from pyagentspec.swarm import Swarm +from pyagentspec.tracing.events.event import Event + + +class SwarmExecutionStart(Event): + """The execution of a swarm is starting. Emitted when a SwarmExecutionSpan starts""" + + swarm: Swarm + "The Swarm being executed" + + inputs: SensitiveField[Dict[str, Any]] + "The inputs used for the swarm's execution, one per property defined in swarm's inputs" + + +class SwarmExecutionEnd(Event): + """The execution of a swarm is ending. Emitted when a SwarmExecutionSpan ends.""" + + swarm: Swarm + "The Swarm being executed" + + outputs: SensitiveField[Dict[str, Any]] + "The outputs generated by the swarm's execution, one per property defined in swarm's outputs" diff --git a/pyagentspec/src/pyagentspec/tracing/events/tool.py b/pyagentspec/src/pyagentspec/tracing/events/tool.py new file mode 100644 index 00000000..c8697d5b --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/events/tool.py @@ -0,0 +1,66 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from typing import Any, Dict, Optional + +from pyagentspec.sensitive_field import SensitiveField +from pyagentspec.tools import Tool +from pyagentspec.tracing.events.event import Event + + +class ToolExecutionRequest(Event): + """A tool execution request is received. Emitted when a ToolExecutionSpan starts, or a client tool is called.""" + + tool: Tool + "The Tool being executed" + + inputs: SensitiveField[Dict[str, Any]] + "The input values that should be used to execute the tool, one per property defined in tool's inputs" + + request_id: str + "Identifier of the tool execution request" + + +class ToolExecutionResponse(Event): + """A tool execution finishes and a result is received. Raised when a ToolExecutionSpan ends, or a client tool result is received.""" + + tool: Tool + "The Tool being executed" + + outputs: SensitiveField[Dict[str, Any]] + "The return value generated by the tool's execution, one per property defined in tool's outputs" + + request_id: str + "Identifier of the tool execution request" + + +class ToolConfirmationRequest(Event): + """A tool confirmation request is raised.""" + + tool: Tool + "The Tool being executed" + + request_id: str + "Identifier of the confirmation request" + + tool_execution_request_id: Optional[str] = None + "Identifier of the tool execution request this confirmation relates to" + + +class ToolConfirmationResponse(Event): + """A tool confirmation response is received.""" + + tool: Tool + "The Tool being executed" + + execution_confirmed: bool + "Whether the execution of the tool was confirmed" + + request_id: str + "Identifier of the confirmation request" + + tool_execution_request_id: Optional[str] = None + "Identifier of the tool execution request this confirmation relates to" diff --git a/pyagentspec/src/pyagentspec/tracing/messages/__init__.py b/pyagentspec/src/pyagentspec/tracing/messages/__init__.py new file mode 100644 index 00000000..b6a1c6eb --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/messages/__init__.py @@ -0,0 +1,5 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. diff --git a/pyagentspec/src/pyagentspec/tracing/messages/message.py b/pyagentspec/src/pyagentspec/tracing/messages/message.py new file mode 100644 index 00000000..bad2554d --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/messages/message.py @@ -0,0 +1,25 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from typing import Optional + +from pydantic import BaseModel + + +class Message(BaseModel): + """Model used to specify LLM message details in events and spans""" + + id: Optional[str] = None + "Identifier of the message" + + content: str + "Content of the message" + + sender: Optional[str] = None + "Sender of the message" + + role: str + "Role of the sender of the message. Typically 'user', 'assistant', or 'system'" diff --git a/pyagentspec/src/pyagentspec/tracing/spanprocessor.py b/pyagentspec/src/pyagentspec/tracing/spanprocessor.py new file mode 100644 index 00000000..c0a7b463 --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/spanprocessor.py @@ -0,0 +1,107 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from abc import ABC, abstractmethod + +from pyagentspec.tracing.events.event import Event +from pyagentspec.tracing.spans.span import Span + + +class SpanProcessor(ABC): + """ + Interface which allows hooks for `Span` start and end method invocations. + + Aligned with OpenTelemetry APIs. + """ + + def __init__(self, mask_sensitive_information: bool = True) -> None: + self.mask_sensitive_information = mask_sensitive_information + + @abstractmethod + def on_start(self, span: "Span") -> None: + """ + Called when a `Span` is started. + + Parameters + ---------- + span: + The spans that starts + """ + + @abstractmethod + async def on_start_async(self, span: "Span") -> None: + """ + Called when a `Span` is started. Asynchronous method. + + Parameters + ---------- + span: + The spans that starts + """ + + @abstractmethod + def on_end(self, span: "Span") -> None: + """ + Called when a `Span` is ended. + + Parameters + ---------- + span: + The spans that ends + """ + + @abstractmethod + async def on_end_async(self, span: "Span") -> None: + """ + Called when a `Span` is ended. Asynchronous method. + + Parameters + ---------- + span: + The spans that ends + """ + + @abstractmethod + def on_event(self, event: Event, span: Span) -> None: + """ + Called when an `Event` is triggered. + + Parameters + ---------- + event: + The event that is happening + span: + The spans where the event occurs + """ + + @abstractmethod + async def on_event_async(self, event: Event, span: Span) -> None: + """ + Called when an `Event` is triggered. Asynchronous method. + + Parameters + ---------- + event: + The event that is happening + span: + The spans where the event occurs + """ + + @abstractmethod + def startup(self) -> None: + """Called when a `Trace` is started.""" + + @abstractmethod + async def startup_async(self) -> None: + """Called when a `Trace` is started. Asynchronous method.""" + + @abstractmethod + def shutdown(self) -> None: + """Called when a `Trace` is shutdown.""" + + @abstractmethod + async def shutdown_async(self) -> None: + """Called when a `Trace` is shutdown. Asynchronous method.""" diff --git a/pyagentspec/src/pyagentspec/tracing/spans/__init__.py b/pyagentspec/src/pyagentspec/tracing/spans/__init__.py new file mode 100644 index 00000000..d4859f13 --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/spans/__init__.py @@ -0,0 +1,27 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from .agent import AgentExecutionSpan +from .flow import FlowExecutionSpan +from .llm import LlmGenerationSpan +from .managerworkers import ManagerWorkersExecutionSpan +from .node import NodeExecutionSpan +from .root import RootSpan +from .span import Span +from .swarm import SwarmExecutionSpan +from .tool import ToolExecutionSpan + +__all__ = [ + "Span", + "AgentExecutionSpan", + "LlmGenerationSpan", + "ToolExecutionSpan", + "NodeExecutionSpan", + "FlowExecutionSpan", + "ManagerWorkersExecutionSpan", + "SwarmExecutionSpan", + "RootSpan", +] diff --git a/pyagentspec/src/pyagentspec/tracing/spans/agent.py b/pyagentspec/src/pyagentspec/tracing/spans/agent.py new file mode 100644 index 00000000..76b7ed3f --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/spans/agent.py @@ -0,0 +1,20 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from pyagentspec.agent import Agent +from pyagentspec.tracing.spans.span import Span + + +class AgentExecutionSpan(Span): + """ + Span to represent the execution of an agent. Can be nested when executing sub-agents. + + - Starts when: agent execution starts + - Ends when: the agent execution is completed, and the result is ready to be processed + """ + + agent: Agent + "The Agent being executed" diff --git a/pyagentspec/src/pyagentspec/tracing/spans/flow.py b/pyagentspec/src/pyagentspec/tracing/spans/flow.py new file mode 100644 index 00000000..46666435 --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/spans/flow.py @@ -0,0 +1,20 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from pyagentspec.flows.flow import Flow +from pyagentspec.tracing.spans.span import Span + + +class FlowExecutionSpan(Span): + """ + Span that covers the execution of a Flow. + + - Starts when: the StartNode execution of this flow starts + - Ends when: one of the EndNode execution finishes + """ + + flow: Flow + "The Flow being executed" diff --git a/pyagentspec/src/pyagentspec/tracing/spans/llm.py b/pyagentspec/src/pyagentspec/tracing/spans/llm.py new file mode 100644 index 00000000..4221554b --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/spans/llm.py @@ -0,0 +1,20 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from pyagentspec.llms import LlmConfig +from pyagentspec.tracing.spans.span import Span + + +class LlmGenerationSpan(Span): + """ + Span that covers the whole LLM generation process + + - Starts when: the LLM generation request is received and the LLM call is performed + - Ends when: the LLM output was generated, and it's ready to be processed + """ + + llm_config: LlmConfig + "The LlmConfig that performs the generation" diff --git a/pyagentspec/src/pyagentspec/tracing/spans/managerworkers.py b/pyagentspec/src/pyagentspec/tracing/spans/managerworkers.py new file mode 100644 index 00000000..c72ea84c --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/spans/managerworkers.py @@ -0,0 +1,20 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from pyagentspec.managerworkers import ManagerWorkers +from pyagentspec.tracing.spans.span import Span + + +class ManagerWorkersExecutionSpan(Span): + """ + Span to represent the execution of a ManagerWorkers. Can be nested when executing sub-agents. + + - Starts when: manager-workers pattern execution starts + - Ends when: the manager-workers execution is completed and the result is ready to be processed + """ + + managerworkers: ManagerWorkers + "The ManagerWorkers being executed" diff --git a/pyagentspec/src/pyagentspec/tracing/spans/node.py b/pyagentspec/src/pyagentspec/tracing/spans/node.py new file mode 100644 index 00000000..02674395 --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/spans/node.py @@ -0,0 +1,20 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from pyagentspec.flows.node import Node +from pyagentspec.tracing.spans.span import Span + + +class NodeExecutionSpan(Span): + """ + Span that covers the execution of a Node. + + - Starts when: the node execution starts on the given inputs + - Ends when: the node execution ends and outputs are ready to be processed + """ + + node: Node + "The Node being executed" diff --git a/pyagentspec/src/pyagentspec/tracing/spans/root.py b/pyagentspec/src/pyagentspec/tracing/spans/root.py new file mode 100644 index 00000000..7e34ba95 --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/spans/root.py @@ -0,0 +1,16 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from pyagentspec.tracing.spans.span import Span + + +class RootSpan(Span): + """ + Span that covers a whole Trace. + + - Starts when: a Trace is started + - Ends when: a Trace is closed + """ diff --git a/pyagentspec/src/pyagentspec/tracing/spans/span.py b/pyagentspec/src/pyagentspec/tracing/spans/span.py new file mode 100644 index 00000000..18276f58 --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/spans/span.py @@ -0,0 +1,268 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +import sys +import time +import traceback +import uuid +from contextvars import ContextVar +from types import TracebackType +from typing import TYPE_CHECKING, Any, List, Optional, Type + +from pydantic import ConfigDict, Field, PrivateAttr +from typing_extensions import Self + +from pyagentspec.tracing._basemodel import BaseModelWithSensitiveInfo +from pyagentspec.tracing.events.event import Event + +if TYPE_CHECKING: + from pyagentspec.tracing.spanprocessor import SpanProcessor + from pyagentspec.tracing.trace import Trace + + +_ACTIVE_SPAN_STACK: ContextVar[List["Span"]] = ContextVar("_ACTIVE_SPAN_STACK", default=[]) + +# setting it will ensure it's seen by `contextvars.copy_context()` +# because this doesn't use values with default that have not been passed +# this call is used in async <-> sync transitions to ensure propagation of +# context variables updates +_ACTIVE_SPAN_STACK.set([]) + + +def _append_span_to_active_stack(span: "Span") -> None: + span_stack = get_active_span_stack(return_copy=True) + span_stack.append(span) + _ACTIVE_SPAN_STACK.set(span_stack) + + +def _pop_span_from_active_stack() -> None: + span_stack = get_active_span_stack(return_copy=True) + span_stack.pop(-1) + _ACTIVE_SPAN_STACK.set(span_stack) + + +def get_active_span_stack(return_copy: bool = True) -> List["Span"]: + """ + Retrieve the stack of active spans in this context. + + Returns + ------- + The stack of active spans in this context + """ + from copy import copy + + span_stack = _ACTIVE_SPAN_STACK.get() + return copy(span_stack) if return_copy else span_stack + + +def get_current_span() -> Optional["Span"]: + """ + Retrieve the currently active span in this context. + + Returns + ------- + The active span in this context + """ + span_stack = get_active_span_stack(return_copy=False) + if len(span_stack) > 0: + return span_stack[-1] + return None + + +class Span(BaseModelWithSensitiveInfo): + + model_config = ConfigDict(arbitrary_types_allowed=True) + + id: str = Field(default_factory=lambda: str(uuid.uuid4()), frozen=True) + """A unique identifier for the event""" + name: Optional[str] = None + """The name of the span. If None, the span class name is used.""" + description: str = "" + """The description of the span.""" + start_time: Optional[int] = None + """The timestamp of when the span was started""" + end_time: Optional[int] = None + """The timestamp of when the span was closed""" + events: List[Event] = Field(default_factory=list) + """The list of events recorded in the scope of this span""" + metadata: dict[str, Any] = Field(default_factory=dict) + """Metadata related to the span""" + _parent_span: Optional["Span"] = PrivateAttr(default=None) + _end_event_was_triggered: bool = PrivateAttr(default=False) + _span_was_appended_to_active_stack: bool = PrivateAttr(default=False) + _started_span_processors: List["SpanProcessor"] = PrivateAttr(default_factory=list) + + def model_post_init(self, __context: Any) -> None: + """Set the default name if it is not provided.""" + super().model_post_init(__context) + if not self.name: + self.name = self.__class__.__name__ + + @property + def _trace(self) -> Optional["Trace"]: + """The Trace where this Span is being stored""" + from pyagentspec.tracing.trace import get_trace + + return get_trace() + + @property + def _span_processors(self) -> List[Any]: + """The list of SpanProcessors to which this Span should be forwarded""" + return self._trace.span_processors if self._trace else [] + + def __enter__(self) -> Self: + self.start() + return self + + async def __aenter__(self) -> Self: + await self.start_async() + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback_obj: Optional[TracebackType], + ) -> None: + if exc_value is not None: + from pyagentspec.tracing.events import ExceptionRaised + + self.add_event( + ExceptionRaised( + exception_type=exc_type.__name__ if exc_type else "Unknown", + exception_message=str(exc_value), + exception_stacktrace="".join( + traceback.format_exception(exc_type, exc_value, traceback_obj) + ), + ) + ) + self.end() + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: Optional[TracebackType], + ) -> None: + if exc_value is not None: + from pyagentspec.tracing.events import ExceptionRaised + + await self.add_event_async( + ExceptionRaised( + exception_type=exc_type.__name__ if exc_type else "Unknown", + exception_message=str(exc_value), + exception_stacktrace=str(traceback), + ) + ) + await self.end_async() + + def start(self) -> None: + """ + Start the span. + + This includes calling the ``on_start`` method of the active SpanProcessors. + """ + try: + self._parent_span = get_current_span() + self.start_time = time.time_ns() + for span_processor in self._span_processors: + span_processor.on_start(self) + # We remember which span processors were started, so that we call on_end on them only + # when we exit, e.g., because of an exception happening + self._started_span_processors.append(span_processor) + _append_span_to_active_stack(self) + self._span_was_appended_to_active_stack = True + except Exception as e: + # If anything happens during the recording of the start span, + # we still have to do the work needed to exit the context + # including the span_processors.on_end call and removing the span from the active stack + self.__exit__(*sys.exc_info()) + raise e + + async def start_async(self) -> None: + """ + Start the span. Asynchronous method. + + This includes calling the ``on_start_async`` method of the active SpanProcessors. + """ + try: + self._parent_span = get_current_span() + self.start_time = time.time_ns() + for span_processor in self._span_processors: + await span_processor.on_start_async(self) + # We remember which span processors were started, so that we call on_end on them only + # when we exit, e.g., because of an exception happening + self._started_span_processors.append(span_processor) + _append_span_to_active_stack(self) + self._span_was_appended_to_active_stack = True + except Exception as e: + # If anything happens during the recording of the start span, + # we still have to do the work needed to exit the context + # including the span_processors.on_end call and removing the span from the active stack + await self.__aexit__(*sys.exc_info()) + raise e + + def end(self) -> None: + """ + End the span. + + This includes calling the ``on_end`` method of the active SpanProcessors. + """ + try: + exceptions_list: List[Exception] = [] + self.end_time = time.time_ns() + # We call on_end only on the span_processors that were successfully started + for span_processor in self._started_span_processors: + # We catch the exceptions that are raised to ensure we call on_end on all + # the span processors on which on_start was called + try: + span_processor.on_end(self) + except Exception as e: + exceptions_list.append(e) + # If we caught exceptions in span processors, we raise one of them here (the first we caught) + if len(exceptions_list) > 0: + raise exceptions_list[0] + finally: + # Whatever happens, we have to pop the span if it is on the active spans stack + if self._span_was_appended_to_active_stack: + _pop_span_from_active_stack() + + async def end_async(self) -> None: + """ + End the span. Asynchronous method. + + This includes calling the ``on_end_async`` method of the active SpanProcessors. + """ + try: + exceptions_list: List[Exception] = [] + self.end_time = time.time_ns() + # We call on_end only on the span_processors that were successfully started + for span_processor in self._started_span_processors: + # We catch the exceptions that are raised to ensure we call on_end on all + # the span processors on which on_start was called + try: + await span_processor.on_end_async(self) + except Exception as e: + exceptions_list.append(e) + # If we caught exceptions in span processors, we raise one of them here (the first we caught) + if len(exceptions_list) > 0: + raise exceptions_list[0] + finally: + # Whatever happens, we have to pop the span if it is on the active spans stack + if self._span_was_appended_to_active_stack: + _pop_span_from_active_stack() + + def add_event(self, event: Event) -> None: + """Add an event to the span and trigger ``on_event`` on the active ``SpanProcessors``.""" + self.events.append(event) + for span_processor in self._started_span_processors: + span_processor.on_event(event, self) + + async def add_event_async(self, event: Event) -> None: + """Add an event to the span and trigger ``on_event_async`` on the active ``SpanProcessors``.""" + self.events.append(event) + for span_processor in self._started_span_processors: + await span_processor.on_event_async(event, self) diff --git a/pyagentspec/src/pyagentspec/tracing/spans/swarm.py b/pyagentspec/src/pyagentspec/tracing/spans/swarm.py new file mode 100644 index 00000000..c81d0400 --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/spans/swarm.py @@ -0,0 +1,20 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from pyagentspec.swarm import Swarm +from pyagentspec.tracing.spans.span import Span + + +class SwarmExecutionSpan(Span): + """ + Span to represent the execution of a Swarm. Can be nested when executing sub-agents. + + - Starts when: swarm pattern execution starts + - Ends when: the swarm execution is completed and the result is ready to be processed + """ + + swarm: Swarm + "The Swarm being executed" diff --git a/pyagentspec/src/pyagentspec/tracing/spans/tool.py b/pyagentspec/src/pyagentspec/tracing/spans/tool.py new file mode 100644 index 00000000..6149e63d --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/spans/tool.py @@ -0,0 +1,20 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from pyagentspec.tools import Tool +from pyagentspec.tracing.spans.span import Span + + +class ToolExecutionSpan(Span): + """ + Span that covers a tool execution. This does not include client tools. + + - Starts when: tool execution starts + - Ends when: the tool execution is completed and the result is ready to be processed + """ + + tool: Tool + "The Tool being executed" diff --git a/pyagentspec/src/pyagentspec/tracing/trace.py b/pyagentspec/src/pyagentspec/tracing/trace.py new file mode 100644 index 00000000..25e76df3 --- /dev/null +++ b/pyagentspec/src/pyagentspec/tracing/trace.py @@ -0,0 +1,116 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +import uuid +from contextvars import ContextVar +from types import TracebackType +from typing import List, Optional, Type + +from pyagentspec.tracing.spanprocessor import SpanProcessor +from pyagentspec.tracing.spans import RootSpan, Span + +_TRACE: ContextVar[Optional["Trace"]] = ContextVar("_TRACE", default=None) + + +def get_trace() -> Optional["Trace"]: + """ + Get the Trace object active in the current context. + + Returns + ------- + The active Trace object + """ + return _TRACE.get() + + +class Trace: + """ + The root of a collection of Spans. + + It is used to group together all the spans and events emitted during the execution of an assistant. + """ + + def __init__( + self, + name: Optional[str] = None, + id: Optional[str] = None, + span_processors: Optional[List[SpanProcessor]] = None, + shutdown_on_exit: bool = True, + root_span: Optional[Span] = None, + ): + """ + Parameters + ---------- + name: Optional[str] + The name of the trace + id: str + A unique identifier for the trace + span_processors: List[SpanProcessor] + The list of SpanProcessors active on this trace + shutdown_on_exit: bool + Whether to call shutdown on span processors when the trace context is closed + root_span: Optional[Span] + The root span of the trace. If None, a new RootSpan with default values is used. + """ + self.name = name or "Trace" + self.id = id or str(uuid.uuid4()) + self.span_processors = span_processors or [] + self.shutdown_on_exit = shutdown_on_exit + self._root_span = root_span or RootSpan() + + def __enter__(self) -> "Trace": + self._start() + return self + + async def __aenter__(self) -> "Trace": + await self._start_async() + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: TracebackType, + ) -> None: + self._end() + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]], + exc_value: Optional[BaseException], + traceback: TracebackType, + ) -> None: + await self._end_async() + + def _start(self) -> None: + if _TRACE.get() is not None: + raise RuntimeError("A Trace already exists. Cannot create two nested Traces.") + _TRACE.set(self) + for span_processor in self.span_processors: + span_processor.startup() + self._root_span.start() + + async def _start_async(self) -> None: + if _TRACE.get() is not None: + raise RuntimeError("A Trace already exists. Cannot create two nested Traces.") + _TRACE.set(self) + for span_processor in self.span_processors: + await span_processor.startup_async() + await self._root_span.start_async() + + def _end(self) -> None: + self._root_span.end() + _TRACE.set(None) + if self.shutdown_on_exit: + for span_processor in self.span_processors: + span_processor.shutdown() + + async def _end_async(self) -> None: + await self._root_span.end_async() + _TRACE.set(None) + if self.shutdown_on_exit: + for span_processor in self.span_processors: + await span_processor.shutdown_async() diff --git a/pyagentspec/tests/adapters/crewai/test_tracing.py b/pyagentspec/tests/adapters/crewai/test_tracing.py new file mode 100644 index 00000000..55913243 --- /dev/null +++ b/pyagentspec/tests/adapters/crewai/test_tracing.py @@ -0,0 +1,167 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. +from pathlib import Path +from typing import List, Tuple + +from pyagentspec.tracing.events import ( + AgentExecutionEnd, + AgentExecutionStart, + Event, + LlmGenerationRequest, + LlmGenerationResponse, + ToolExecutionRequest, + ToolExecutionResponse, +) +from pyagentspec.tracing.spanprocessor import SpanProcessor +from pyagentspec.tracing.spans import AgentExecutionSpan, LlmGenerationSpan, Span, ToolExecutionSpan +from pyagentspec.tracing.trace import Trace + +from ..conftest import _replace_config_placeholders + +CONFIGS = Path(__file__).parent / "configs" + + +class DummySpanProcessor(SpanProcessor): + """ + Minimal processor mirroring the behavior used in tests/tracing/test_tracing.py + to capture span lifecycle and events for assertions. + """ + + def __init__(self, mask_sensitive_information: bool = True) -> None: + super().__init__(mask_sensitive_information=mask_sensitive_information) + self.started_up = False + self.shut_down = False + self.started_up_async = False + self.shut_down_async = False + self.starts: List[Span] = [] + self.ends: List[Span] = [] + self.events: List[Tuple[Event, Span]] = [] + self.starts_async: List[Span] = [] + self.ends_async: List[Span] = [] + self.events_async: List[Tuple[Event, Span]] = [] + + def on_start(self, span: Span) -> None: + self.starts.append(span) + + async def on_start_async(self, span: Span) -> None: + self.starts_async.append(span) + + def on_end(self, span: Span) -> None: + self.ends.append(span) + + async def on_end_async(self, span: Span) -> None: + self.ends_async.append(span) + + def on_event(self, event: Event, span: Span) -> None: + self.events.append((event, span)) + + async def on_event_async(self, event: Event, span: Span) -> None: + self.events_async.append((event, span)) + + def startup(self) -> None: + self.started_up = True + + def shutdown(self) -> None: + self.shut_down = True + + async def startup_async(self) -> None: + self.started_up_async = True + + async def shutdown_async(self) -> None: + self.shut_down_async = True + + +def check_dummyspanprocessor_events_and_spans(span_processor: DummySpanProcessor) -> None: + # Assertions on spans started/ended + # We expect at least one of each span type during a normal run + started_types = [type(s) for s in span_processor.starts] + ended_types = [type(s) for s in span_processor.ends] + assert any( + issubclass(t, AgentExecutionSpan) for t in started_types + ), "AgentExecutionSpan did not start" + assert any( + issubclass(t, AgentExecutionSpan) for t in ended_types + ), "AgentExecutionSpan did not end" + + assert any( + issubclass(t, LlmGenerationSpan) for t in started_types + ), "LlmGenerationSpan did not start" + assert any( + issubclass(t, LlmGenerationSpan) for t in ended_types + ), "LlmGenerationSpan did not end" + + assert any( + issubclass(t, ToolExecutionSpan) for t in started_types + ), "ToolExecutionSpan did not start" + assert any( + issubclass(t, ToolExecutionSpan) for t in ended_types + ), "ToolExecutionSpan did not end" + + # Assertions on key events observed + event_types = [type(e) for (e, _s) in span_processor.events] + assert any( + issubclass(t, AgentExecutionStart) for t in event_types + ), "AgentExecutionStart not emitted" + assert any( + issubclass(t, AgentExecutionEnd) for t in event_types + ), "AgentExecutionEnd not emitted" + assert any( + issubclass(t, LlmGenerationRequest) for t in event_types + ), "LlmGenerationRequest not emitted" + assert any( + issubclass(t, LlmGenerationResponse) for t in event_types + ), "LlmGenerationResponse not emitted" + assert any( + issubclass(t, ToolExecutionRequest) for t in event_types + ), "ToolExecutionRequest not emitted" + assert any( + issubclass(t, ToolExecutionResponse) for t in event_types + ), "ToolExecutionResponse not emitted" + + +def test_crewai_crew_tracing_emits_agent_llm_and_tool_events(json_server: str) -> None: + + from pyagentspec.adapters.crewai import AgentSpecLoader + from pyagentspec.adapters.crewai._types import crewai + + # Prepare YAML config with placeholders replaced + yaml_content = (CONFIGS / "weather_agent_remote_tool.yaml").read_text() + final_yaml = _replace_config_placeholders(yaml_content, json_server) + weather_agent = AgentSpecLoader().load_yaml(final_yaml) + + # Build a simple task/crew run + task = crewai.Task( + description="Use your tool to answer this simple request from the user: {user_input}", + expected_output="A helpful, concise reply to the user.", + agent=weather_agent, + ) + crew = crewai.Crew(agents=[weather_agent], tasks=[task], verbose=False) + + proc = DummySpanProcessor() + with Trace(name="crewai_tracing_test", span_processors=[proc]): + with weather_agent.agentspec_event_listener(): + response = crew.kickoff(inputs={"user_input": "What's the weather in Agadir?"}) + assert "sunny" in str(response).lower() + + check_dummyspanprocessor_events_and_spans(proc) + + +def test_crewai_agent_tracing_emits_agent_llm_and_tool_events(json_server: str) -> None: + + from pyagentspec.adapters.crewai import AgentSpecLoader + + # Prepare YAML config with placeholders replaced + yaml_content = (CONFIGS / "weather_agent_remote_tool.yaml").read_text() + final_yaml = _replace_config_placeholders(yaml_content, json_server) + weather_agent = AgentSpecLoader().load_yaml(final_yaml) + + proc = DummySpanProcessor() + with Trace(name="crewai_tracing_test", span_processors=[proc]): + with weather_agent.agentspec_event_listener(): + response = weather_agent.kickoff(messages="What's the weather in Agadir?") + assert "sunny" in str(response).lower() + + check_dummyspanprocessor_events_and_spans(proc) diff --git a/pyagentspec/tests/adapters/langgraph/test_tracing.py b/pyagentspec/tests/adapters/langgraph/test_tracing.py new file mode 100644 index 00000000..33ae311d --- /dev/null +++ b/pyagentspec/tests/adapters/langgraph/test_tracing.py @@ -0,0 +1,165 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. +from pathlib import Path +from typing import List, Tuple + +from pyagentspec.tracing.events import ( + Event, + LlmGenerationRequest, + LlmGenerationResponse, + ToolExecutionRequest, + ToolExecutionResponse, +) +from pyagentspec.tracing.spanprocessor import SpanProcessor +from pyagentspec.tracing.spans import LlmGenerationSpan, Span, ToolExecutionSpan +from pyagentspec.tracing.trace import Trace + +from ..conftest import _replace_config_placeholders + +CONFIGS = Path(__file__).parent / "configs" + + +class DummySpanProcessor(SpanProcessor): + """ + Minimal processor mirroring the behavior used in tests/tracing/test_tracing.py + to capture span lifecycle and events for assertions. + """ + + def __init__(self, mask_sensitive_information: bool = True) -> None: + super().__init__(mask_sensitive_information=mask_sensitive_information) + self.started_up = False + self.shut_down = False + self.started_up_async = False + self.shut_down_async = False + self.starts: List[Span] = [] + self.ends: List[Span] = [] + self.events: List[Tuple[Event, Span]] = [] + self.starts_async: List[Span] = [] + self.ends_async: List[Span] = [] + self.events_async: List[Tuple[Event, Span]] = [] + + def on_start(self, span: Span) -> None: + self.starts.append(span) + + async def on_start_async(self, span: Span) -> None: + self.starts_async.append(span) + + def on_end(self, span: Span) -> None: + self.ends.append(span) + + async def on_end_async(self, span: Span) -> None: + self.ends_async.append(span) + + def on_event(self, event: Event, span: Span) -> None: + self.events.append((event, span)) + + async def on_event_async(self, event: Event, span: Span) -> None: + self.events_async.append((event, span)) + + def startup(self) -> None: + self.started_up = True + + def shutdown(self) -> None: + self.shut_down = True + + async def startup_async(self) -> None: + self.started_up_async = True + + async def shutdown_async(self) -> None: + self.shut_down_async = True + + +def check_dummyspanprocessor_events_and_spans(span_processor: DummySpanProcessor) -> None: + # Assertions on spans started/ended + # We expect at least one of each span type during a normal run + started_types = [type(s) for s in span_processor.starts] + ended_types = [type(s) for s in span_processor.ends] + + # Agent execution events are not emitted yet + # assert any(issubclass(t, AgentExecutionSpan) for t in started_types), "AgentExecutionSpan did not start" + # assert any(issubclass(t, AgentExecutionSpan) for t in ended_types), "AgentExecutionSpan did not end" + assert any( + issubclass(t, LlmGenerationSpan) for t in started_types + ), "LlmGenerationSpan did not start" + assert any( + issubclass(t, LlmGenerationSpan) for t in ended_types + ), "LlmGenerationSpan did not end" + + assert any( + issubclass(t, ToolExecutionSpan) for t in started_types + ), "ToolExecutionSpan did not start" + assert any( + issubclass(t, ToolExecutionSpan) for t in ended_types + ), "ToolExecutionSpan did not end" + + # Assertions on key events observed + event_types = [type(e) for (e, _s) in span_processor.events] + # Agent execution events are not emitted yet + # assert any(issubclass(t, AgentExecutionStart) for t in event_types), "AgentExecutionStart not emitted" + # assert any(issubclass(t, AgentExecutionEnd) for t in event_types), "AgentExecutionEnd not emitted" + assert any( + issubclass(t, LlmGenerationRequest) for t in event_types + ), "LlmGenerationRequest not emitted" + assert any( + issubclass(t, LlmGenerationResponse) for t in event_types + ), "LlmGenerationResponse not emitted" + assert any( + issubclass(t, ToolExecutionRequest) for t in event_types + ), "ToolExecutionRequest not emitted" + assert any( + issubclass(t, ToolExecutionResponse) for t in event_types + ), "ToolExecutionResponse not emitted" + + +def test_langgraph_invoke_tracing_emits_agent_llm_and_tool_events(json_server: str) -> None: + + from pyagentspec.adapters.langgraph import AgentSpecLoader + + # Prepare YAML config with placeholders replaced + yaml_content = (CONFIGS / "weather_agent_remote_tool.yaml").read_text() + final_yaml = _replace_config_placeholders(yaml_content, json_server) + + # Convert to LangGraph agent + weather_agent = AgentSpecLoader().load_yaml(final_yaml) + + proc = DummySpanProcessor() + with Trace(name="langgraph_tracing_test", span_processors=[proc]): + agent_input = { + "inputs": {}, + "messages": [{"role": "user", "content": "What's the weather in Agadir?"}], + } + response = weather_agent.invoke(input=agent_input) + assert "sunny" in str(response).lower() + + check_dummyspanprocessor_events_and_spans(proc) + + +def test_langgraph_stream_tracing_emits_agent_llm_and_tool_events(json_server: str) -> None: + + from pyagentspec.adapters.langgraph import AgentSpecLoader + + # Prepare YAML config with placeholders replaced + yaml_content = (CONFIGS / "weather_agent_remote_tool.yaml").read_text() + final_yaml = _replace_config_placeholders(yaml_content, json_server) + + # Convert to LangGraph agent + weather_agent = AgentSpecLoader().load_yaml(final_yaml) + + proc = DummySpanProcessor() + with Trace(name="langgraph_tracing_test", span_processors=[proc]): + agent_input = { + "inputs": {}, + "messages": [{"role": "user", "content": "What's the weather in Agadir?"}], + } + response = "" + for message_chunk, metadata in weather_agent.stream( + input=agent_input, stream_mode="messages" + ): + if message_chunk.content: + response += message_chunk.content + assert "sunny" in str(response).lower() + + check_dummyspanprocessor_events_and_spans(proc) diff --git a/pyagentspec/tests/serialization/test_llm_config.py b/pyagentspec/tests/serialization/test_llm_config.py index 37a79f2b..1834b175 100644 --- a/pyagentspec/tests/serialization/test_llm_config.py +++ b/pyagentspec/tests/serialization/test_llm_config.py @@ -59,6 +59,7 @@ def test_can_serialize_and_deserialize_llm_config_with_arbitrary_generation_conf name="agi2", model_id="agi_model2", url="http://some.where", + api_key="api_key", default_generation_parameters=LlmGenerationConfig(top_p=3), ), VllmConfig( @@ -72,6 +73,7 @@ def test_can_serialize_and_deserialize_llm_config_with_arbitrary_generation_conf id="openai", name="agi4", model_id="agi_model4", + api_key="api_key", default_generation_parameters=LlmGenerationConfig(), ), ], @@ -82,5 +84,8 @@ def test_can_serialize_and_deserialize_llm_config(llm_config: VllmConfig) -> Non assert f"component_type: {type(llm_config).__name__}" assert f"name: agi" assert f"model_id: agi_model" - deserialized_llm = AgentSpecDeserializer().from_yaml(serialized_llm) + deserialized_llm = AgentSpecDeserializer().from_yaml( + serialized_llm, + components_registry={"openai.api_key": "api_key", "openai_compatible.api_key": "api_key"}, + ) assert llm_config == deserialized_llm diff --git a/pyagentspec/tests/tracing/__init__.py b/pyagentspec/tests/tracing/__init__.py new file mode 100644 index 00000000..b6a1c6eb --- /dev/null +++ b/pyagentspec/tests/tracing/__init__.py @@ -0,0 +1,5 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. diff --git a/pyagentspec/tests/tracing/conftest.py b/pyagentspec/tests/tracing/conftest.py new file mode 100644 index 00000000..0f5ce685 --- /dev/null +++ b/pyagentspec/tests/tracing/conftest.py @@ -0,0 +1,105 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +import pytest + +from pyagentspec.agent import Agent +from pyagentspec.flows.edges import ControlFlowEdge, DataFlowEdge +from pyagentspec.flows.flow import Flow +from pyagentspec.flows.node import Node +from pyagentspec.flows.nodes import EndNode, LlmNode, StartNode +from pyagentspec.llms import LlmConfig +from pyagentspec.llms.openaiconfig import OpenAiConfig +from pyagentspec.managerworkers import ManagerWorkers +from pyagentspec.property import IntegerProperty, StringProperty +from pyagentspec.swarm import Swarm +from pyagentspec.tools import ServerTool, Tool + + +@pytest.fixture +def dummy_llm_config() -> LlmConfig: + return OpenAiConfig(name="openai", model_id="gpt-test") + + +@pytest.fixture +def dummy_agent(dummy_llm_config: LlmConfig) -> Agent: + return Agent(name="agent", llm_config=dummy_llm_config, system_prompt="Hello") + + +@pytest.fixture +def dummy_tool() -> Tool: + return ServerTool( + name="servertool", + inputs=[IntegerProperty(title="x")], + outputs=[IntegerProperty(title="y")], + ) + + +@pytest.fixture +def dummy_flow(dummy_llm_config: LlmConfig) -> Flow: + prompt_prop = StringProperty(title="prompt") + llm_out_prop = StringProperty(title="generated_text") + start_node = StartNode(name="start", inputs=[prompt_prop]) + llm_node = LlmNode( + name="llm", + llm_config=dummy_llm_config, + prompt_template="{{prompt}}", + inputs=[prompt_prop], + outputs=[llm_out_prop], + ) + end_node = EndNode(name="end", outputs=[llm_out_prop]) + control_flow_edges = [ + ControlFlowEdge(name="s_to_llm", from_node=start_node, to_node=llm_node), + ControlFlowEdge(name="llm_to_e", from_node=llm_node, to_node=end_node), + ] + data_flow_edges = [ + DataFlowEdge( + name="prompt_edge", + source_node=start_node, + source_output="prompt", + destination_node=llm_node, + destination_input="prompt", + ), + DataFlowEdge( + name="out_edge", + source_node=llm_node, + source_output="generated_text", + destination_node=end_node, + destination_input="generated_text", + ), + ] + return Flow( + name="flow", + start_node=start_node, + nodes=[start_node, llm_node, end_node], + control_flow_connections=control_flow_edges, + data_flow_connections=data_flow_edges, + ) + + +@pytest.fixture +def dummy_node(dummy_llm_config: LlmConfig) -> Node: + return LlmNode( + name="llm_node", + llm_config=dummy_llm_config, + prompt_template="{{prompt}}", + inputs=[StringProperty(title="prompt")], + outputs=[StringProperty(title="generated_text")], + ) + + +@pytest.fixture +def dummy_managerworkers(dummy_llm_config: LlmConfig) -> ManagerWorkers: + mgr = Agent(name="manager", llm_config=dummy_llm_config, system_prompt="You are a manager") + w1 = Agent(name="worker", llm_config=dummy_llm_config, system_prompt="You are a worker") + return ManagerWorkers(name="mw", group_manager=mgr, workers=[w1]) + + +@pytest.fixture +def dummy_swarm(dummy_llm_config: LlmConfig) -> Swarm: + a1 = Agent(name="a1", llm_config=dummy_llm_config, system_prompt="You are a1") + a2 = Agent(name="a2", llm_config=dummy_llm_config, system_prompt="You are a2") + return Swarm(name="sw", first_agent=a1, relationships=[(a1, a2)]) diff --git a/pyagentspec/tests/tracing/events/__init__.py b/pyagentspec/tests/tracing/events/__init__.py new file mode 100644 index 00000000..b6a1c6eb --- /dev/null +++ b/pyagentspec/tests/tracing/events/__init__.py @@ -0,0 +1,5 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. diff --git a/pyagentspec/tests/tracing/events/test_events.py b/pyagentspec/tests/tracing/events/test_events.py new file mode 100644 index 00000000..f6c2aedd --- /dev/null +++ b/pyagentspec/tests/tracing/events/test_events.py @@ -0,0 +1,344 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +from pyagentspec.agent import Agent +from pyagentspec.flows.flow import Flow +from pyagentspec.flows.node import Node +from pyagentspec.llms import LlmConfig +from pyagentspec.managerworkers import ManagerWorkers +from pyagentspec.swarm import Swarm +from pyagentspec.tools import Tool +from pyagentspec.tracing._basemodel import _PII_MASK +from pyagentspec.tracing.events import ( + AgentExecutionEnd, + AgentExecutionStart, + ExceptionRaised, + FlowExecutionEnd, + FlowExecutionStart, + HumanInTheLoopRequest, + HumanInTheLoopResponse, + LlmGenerationChunkReceived, + LlmGenerationRequest, + LlmGenerationResponse, + ManagerWorkersExecutionEnd, + ManagerWorkersExecutionStart, + NodeExecutionEnd, + NodeExecutionStart, + SwarmExecutionEnd, + SwarmExecutionStart, + ToolConfirmationRequest, + ToolConfirmationResponse, + ToolExecutionRequest, + ToolExecutionResponse, +) +from pyagentspec.tracing.messages.message import Message + + +# Exception events +def test_exception_event_creation(): + ev = ExceptionRaised( + exception_type="ValueError", exception_message="bad", exception_stacktrace="trace" + ) + assert ev.exception_type == "ValueError" + assert str(ev.exception_message) == "bad" + assert isinstance(ev.exception_stacktrace, str) + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["exception_message"] == _PII_MASK + assert masked["exception_stacktrace"] == _PII_MASK + assert unmasked["exception_message"] == "bad" + assert unmasked["exception_stacktrace"] == "trace" + assert masked["type"] == "ExceptionRaised" + + +def test_agent_execution_start_creation(dummy_agent: Agent): + ev = AgentExecutionStart(agent=dummy_agent, inputs={"x": 1}, name="custom") + assert ev.name == "custom" + assert ev.agent is dummy_agent + assert ev.inputs == {"x": 1} + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["inputs"] == _PII_MASK + assert unmasked["inputs"] == {"x": 1} + assert masked["type"] == "AgentExecutionStart" + + +def test_agent_execution_end_creation(dummy_agent: Agent): + ev = AgentExecutionEnd(agent=dummy_agent, outputs={"y": 2}) + assert ev.agent is dummy_agent + assert ev.outputs == {"y": 2} + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["outputs"] == _PII_MASK + assert unmasked["outputs"] == {"y": 2} + assert masked["type"] == "AgentExecutionEnd" + + +# Flow events +def test_flow_execution_start_creation(dummy_flow: Flow): + ev = FlowExecutionStart(flow=dummy_flow, inputs={"a": 1}, name="flow_start_custom") + assert ev.name == "flow_start_custom" + assert ev.flow is dummy_flow + assert ev.inputs == {"a": 1} + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["inputs"] == _PII_MASK + assert unmasked["inputs"] == {"a": 1} + assert masked["type"] == "FlowExecutionStart" + + +def test_flow_execution_end_creation(dummy_flow: Flow): + ev = FlowExecutionEnd(flow=dummy_flow, outputs={"b": 2}, branch_selected="next") + assert ev.flow is dummy_flow + assert ev.outputs == {"b": 2} + assert ev.branch_selected == "next" + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["outputs"] == _PII_MASK + assert unmasked["outputs"] == {"b": 2} + assert masked["branch_selected"] == "next" + assert unmasked["branch_selected"] == "next" + assert masked["type"] == "FlowExecutionEnd" + + +# HITL events +def test_humanintheloop_request_creation(): + ev = HumanInTheLoopRequest(request_id="r1", content={"question": "ok?"}) + assert ev.request_id == "r1" + assert ev.content == {"question": "ok?"} + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["content"] == _PII_MASK + assert unmasked["content"] == {"question": "ok?"} + assert masked["type"] == "HumanInTheLoopRequest" + + +def test_humanintheloop_response_creation(): + ev = HumanInTheLoopResponse(request_id="r1", content={"answer": "yes"}) + assert ev.request_id == "r1" + assert ev.content == {"answer": "yes"} + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["content"] == _PII_MASK + assert unmasked["content"] == {"answer": "yes"} + assert masked["type"] == "HumanInTheLoopResponse" + + +# LLM generation events +def test_llm_generation_request_creation(dummy_tool: Tool, dummy_llm_config: LlmConfig): + msgs = [Message(content="hello", role="user")] + ev = LlmGenerationRequest( + llm_config=dummy_llm_config, + prompt=msgs, + tools=[dummy_tool], + request_id="req-1", + ) + assert ev.llm_config is dummy_llm_config + assert ev.prompt == msgs + assert ev.tools == [dummy_tool] + assert ev.request_id == "req-1" + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["prompt"] == _PII_MASK + assert unmasked["prompt"] == [m.model_dump() for m in msgs] + assert unmasked["tools"][0]["name"] == dummy_tool.name + assert masked["type"] == "LlmGenerationRequest" + + +def test_llm_generation_response_creation(dummy_llm_config: LlmConfig): + from pyagentspec.tracing.events.llmgeneration import ToolCall + + ev = LlmGenerationResponse( + llm_config=dummy_llm_config, + content="hi", + request_id="req-1", + completion_id="c-1", + tool_calls=[ToolCall(call_id="a", tool_name="b", arguments="{'c': 1}")], + input_tokens=10, + output_tokens=2, + ) + assert str(ev.content) == "hi" + assert ev.request_id == "req-1" + assert ev.completion_id == "c-1" + assert ev.input_tokens == 10 + assert ev.output_tokens == 2 + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["content"] == _PII_MASK + assert masked["tool_calls"] == _PII_MASK + assert unmasked["content"] == "hi" + assert isinstance(unmasked["tool_calls"], list) and len(unmasked["tool_calls"]) == 1 + assert unmasked["request_id"] == "req-1" + assert unmasked["completion_id"] == "c-1" + assert unmasked["input_tokens"] == 10 + assert unmasked["output_tokens"] == 2 + assert masked["type"] == "LlmGenerationResponse" + + +def test_llm_generation_chunk_received_creation(dummy_llm_config: LlmConfig): + from pyagentspec.tracing.events.llmgeneration import ToolCall + + ev = LlmGenerationChunkReceived( + llm_config=dummy_llm_config, + content="piece", + tool_calls=[ToolCall(call_id="a", tool_name="b", arguments="{'c': 1}")], + request_id="r", + completion_id="c", + output_tokens=1, + ) + assert ev.llm_config is dummy_llm_config + assert str(ev.content) == "piece" + assert ev.request_id == "r" + assert ev.output_tokens == 1 + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["content"] == _PII_MASK + assert masked["tool_calls"] == _PII_MASK + assert unmasked["content"] == "piece" + assert isinstance(unmasked["tool_calls"], list) and len(unmasked["tool_calls"]) == 1 + assert masked["type"] == "LlmGenerationChunkReceived" + + +# Manager-workers events +def test_managerworkers_execution_start_creation(dummy_managerworkers: ManagerWorkers): + ev = ManagerWorkersExecutionStart(managerworkers=dummy_managerworkers, inputs={"foo": "bar"}) + assert ev.managerworkers is dummy_managerworkers + assert ev.inputs == {"foo": "bar"} + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["inputs"] == _PII_MASK + assert unmasked["inputs"] == {"foo": "bar"} + assert masked["type"] == "ManagerWorkersExecutionStart" + + +def test_managerworkers_execution_end_creation(dummy_managerworkers: ManagerWorkers): + ev = ManagerWorkersExecutionEnd(managerworkers=dummy_managerworkers, outputs={"foo": "baz"}) + assert ev.managerworkers is dummy_managerworkers + assert ev.outputs == {"foo": "baz"} + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["outputs"] == _PII_MASK + assert unmasked["outputs"] == {"foo": "baz"} + assert masked["type"] == "ManagerWorkersExecutionEnd" + + +# Node events +def test_node_execution_start_creation(dummy_node: Node): + ev = NodeExecutionStart(node=dummy_node, inputs={"v": 3}) + assert ev.node is dummy_node + assert ev.inputs == {"v": 3} + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["inputs"] == _PII_MASK + assert unmasked["inputs"] == {"v": 3} + assert masked["type"] == "NodeExecutionStart" + + +def test_node_execution_end_creation(dummy_node: Node): + ev = NodeExecutionEnd(node=dummy_node, outputs={"v": 4}, branch_selected="next") + assert ev.node is dummy_node + assert ev.outputs == {"v": 4} + assert ev.branch_selected == "next" + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["outputs"] == _PII_MASK + assert unmasked["outputs"] == {"v": 4} + assert masked["branch_selected"] == "next" + assert unmasked["branch_selected"] == "next" + assert masked["type"] == "NodeExecutionEnd" + + +# Swarm events +def test_swarm_execution_start_creation(dummy_swarm: Swarm): + ev = SwarmExecutionStart(swarm=dummy_swarm, inputs={"q": "x"}) + assert ev.swarm is dummy_swarm + assert ev.inputs == {"q": "x"} + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["inputs"] == _PII_MASK + assert unmasked["inputs"] == {"q": "x"} + assert masked["type"] == "SwarmExecutionStart" + + +def test_swarm_execution_end_creation(dummy_swarm: Swarm): + ev = SwarmExecutionEnd(swarm=dummy_swarm, outputs={"r": "y"}) + assert ev.swarm is dummy_swarm + assert ev.outputs == {"r": "y"} + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["outputs"] == _PII_MASK + assert unmasked["outputs"] == {"r": "y"} + assert masked["type"] == "SwarmExecutionEnd" + + +# Tool events +def test_tool_execution_request_creation(dummy_tool: Tool): + ev = ToolExecutionRequest(tool=dummy_tool, inputs={"x": 1}, request_id="t1") + assert ev.tool is dummy_tool + assert ev.inputs == {"x": 1} + assert ev.request_id == "t1" + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["inputs"] == _PII_MASK + assert unmasked["inputs"] == {"x": 1} + assert unmasked["request_id"] == "t1" + assert masked["type"] == "ToolExecutionRequest" + + +def test_tool_execution_response_creation(dummy_tool: Tool): + ev = ToolExecutionResponse(tool=dummy_tool, outputs={"y": 2}, request_id="t1") + assert ev.tool is dummy_tool + assert ev.outputs == {"y": 2} + assert ev.request_id == "t1" + # Masking behavior + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked["outputs"] == _PII_MASK + assert unmasked["outputs"] == {"y": 2} + assert unmasked["request_id"] == "t1" + assert masked["type"] == "ToolExecutionResponse" + + +def test_tool_confirmation_request_creation(dummy_tool: Tool): + ev = ToolConfirmationRequest(tool=dummy_tool, request_id="c1", tool_execution_request_id="t1") + assert ev.tool is dummy_tool + assert ev.request_id == "c1" + assert ev.tool_execution_request_id == "t1" + # Masking behavior (no sensitive fields) + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked == unmasked + assert masked["type"] == "ToolConfirmationRequest" + + +def test_tool_confirmation_response_creation(dummy_tool: Tool): + ev = ToolConfirmationResponse( + tool=dummy_tool, execution_confirmed=True, request_id="c1", tool_execution_request_id="t1" + ) + assert ev.tool is dummy_tool + assert ev.execution_confirmed is True + assert ev.request_id == "c1" + # Masking behavior (no sensitive fields) + masked = ev.model_dump(mask_sensitive_information=True) + unmasked = ev.model_dump(mask_sensitive_information=False) + assert masked == unmasked diff --git a/pyagentspec/tests/tracing/spans/__init__.py b/pyagentspec/tests/tracing/spans/__init__.py new file mode 100644 index 00000000..b6a1c6eb --- /dev/null +++ b/pyagentspec/tests/tracing/spans/__init__.py @@ -0,0 +1,5 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. diff --git a/pyagentspec/tests/tracing/spans/test_spans.py b/pyagentspec/tests/tracing/spans/test_spans.py new file mode 100644 index 00000000..b3e245df --- /dev/null +++ b/pyagentspec/tests/tracing/spans/test_spans.py @@ -0,0 +1,122 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. +import pytest + +from pyagentspec.agent import Agent +from pyagentspec.flows.flow import Flow +from pyagentspec.flows.node import Node +from pyagentspec.llms import LlmConfig +from pyagentspec.managerworkers import ManagerWorkers +from pyagentspec.swarm import Swarm +from pyagentspec.tools import Tool +from pyagentspec.tracing.events import Event +from pyagentspec.tracing.spans import ( + AgentExecutionSpan, + FlowExecutionSpan, + LlmGenerationSpan, + ManagerWorkersExecutionSpan, + NodeExecutionSpan, + SwarmExecutionSpan, + ToolExecutionSpan, +) + + +@pytest.fixture +def dummy_event() -> Event: + return Event(id="dummy_event_id", name="dummy_event") + + +def test_agent_execution_span_creation(dummy_agent: Agent, dummy_event: Event): + span = AgentExecutionSpan(agent=dummy_agent, name="custom_agent_span") + assert span.name == "custom_agent_span" + assert span.agent is dummy_agent + span.add_event(dummy_event) + assert len(span.events) == 1 + assert span.events[0] is dummy_event + # Masking behavior (no sensitive fields in spans) + masked = span.model_dump(mask_sensitive_information=True) + unmasked = span.model_dump(mask_sensitive_information=False) + assert masked == unmasked + + +def test_flow_execution_span_creation(dummy_flow: Flow, dummy_event: Event): + span = FlowExecutionSpan(flow=dummy_flow, name="custom_flow_span") + assert span.name == "custom_flow_span" + assert span.flow is dummy_flow + span.add_event(dummy_event) + assert len(span.events) == 1 + assert span.events[0] is dummy_event + # Masking behavior (no sensitive fields in spans) + masked = span.model_dump(mask_sensitive_information=True) + unmasked = span.model_dump(mask_sensitive_information=False) + assert masked == unmasked + + +def test_llm_generation_span_creation(dummy_llm_config: LlmConfig, dummy_event: Event): + span = LlmGenerationSpan(llm_config=dummy_llm_config, name="custom_llm_span") + assert span.name == "custom_llm_span" + assert span.llm_config is dummy_llm_config + span.add_event(dummy_event) + assert len(span.events) == 1 + assert span.events[0] is dummy_event + # Masking behavior (no sensitive fields in spans) + masked = span.model_dump(mask_sensitive_information=True) + unmasked = span.model_dump(mask_sensitive_information=False) + assert masked == unmasked + + +def test_managerworkers_execution_span_creation( + dummy_managerworkers: ManagerWorkers, dummy_event: Event +): + span = ManagerWorkersExecutionSpan(managerworkers=dummy_managerworkers, name="custom_mw_span") + assert span.name == "custom_mw_span" + assert span.managerworkers is dummy_managerworkers + span.add_event(dummy_event) + assert len(span.events) == 1 + assert span.events[0] is dummy_event + # Masking behavior (no sensitive fields in spans) + masked = span.model_dump(mask_sensitive_information=True) + unmasked = span.model_dump(mask_sensitive_information=False) + assert masked == unmasked + + +def test_node_execution_span_creation(dummy_node: Node, dummy_event: Event): + span = NodeExecutionSpan(node=dummy_node, name="custom_node_span") + assert span.name == "custom_node_span" + assert span.node is dummy_node + span.add_event(dummy_event) + assert len(span.events) == 1 + assert span.events[0] is dummy_event + # Masking behavior (no sensitive fields in spans) + masked = span.model_dump(mask_sensitive_information=True) + unmasked = span.model_dump(mask_sensitive_information=False) + assert masked == unmasked + + +def test_swarm_execution_span_creation(dummy_swarm: Swarm, dummy_event: Event): + span = SwarmExecutionSpan(swarm=dummy_swarm, name="custom_swarm_span") + assert span.name == "custom_swarm_span" + assert span.swarm is dummy_swarm + span.add_event(dummy_event) + assert len(span.events) == 1 + assert span.events[0] is dummy_event + # Masking behavior (no sensitive fields in spans) + masked = span.model_dump(mask_sensitive_information=True) + unmasked = span.model_dump(mask_sensitive_information=False) + assert masked == unmasked + + +def test_tool_execution_span_creation(dummy_tool: Tool, dummy_event: Event): + span = ToolExecutionSpan(tool=dummy_tool, name="custom_tool_span") + assert span.name == "custom_tool_span" + assert span.tool is dummy_tool + span.add_event(dummy_event) + assert len(span.events) == 1 + assert span.events[0] is dummy_event + # Masking behavior (no sensitive fields in spans) + masked = span.model_dump(mask_sensitive_information=True) + unmasked = span.model_dump(mask_sensitive_information=False) + assert masked == unmasked diff --git a/pyagentspec/tests/tracing/test_tracing.py b/pyagentspec/tests/tracing/test_tracing.py new file mode 100644 index 00000000..86e78e99 --- /dev/null +++ b/pyagentspec/tests/tracing/test_tracing.py @@ -0,0 +1,389 @@ +# Copyright © 2025 Oracle and/or its affiliates. +# +# This software is under the Apache License 2.0 +# (LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0) or Universal Permissive License +# (UPL) 1.0 (LICENSE-UPL or https://oss.oracle.com/licenses/upl), at your option. + +import asyncio +import re +import time +from typing import List, Tuple + +import pytest + +from pyagentspec.tracing.events import Event, ExceptionRaised +from pyagentspec.tracing.spanprocessor import SpanProcessor +from pyagentspec.tracing.spans import RootSpan +from pyagentspec.tracing.spans.span import ( + Span, + get_active_span_stack, + get_current_span, +) +from pyagentspec.tracing.trace import Trace, get_trace + +UUID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$" +) + + +class DummySpanProcessor(SpanProcessor): + + def __init__(self, mask_sensitive_information: bool = True) -> None: + super().__init__(mask_sensitive_information=mask_sensitive_information) + self.started_up = False + self.shut_down = False + self.started_up_async = False + self.shut_down_async = False + self.starts: List[Span] = [] + self.ends: List[Span] = [] + self.events: List[Tuple[Event, Span]] = [] + self.starts_async: List[Span] = [] + self.ends_async: List[Span] = [] + self.events_async: List[Tuple[Event, Span]] = [] + + def on_start(self, span: Span) -> None: + self.starts.append(span) + + async def on_start_async(self, span: Span) -> None: + self.starts_async.append(span) + + def on_end(self, span: Span) -> None: + self.ends.append(span) + + async def on_end_async(self, span: Span) -> None: + self.ends_async.append(span) + + def on_event(self, event: Event, span: Span) -> None: + self.events.append((event, span)) + + async def on_event_async(self, event: Event, span: Span) -> None: + self.events_async.append((event, span)) + + def startup(self) -> None: + self.started_up = True + + def shutdown(self) -> None: + self.shut_down = True + + async def startup_async(self) -> None: + self.started_up_async = True + + async def shutdown_async(self) -> None: + self.shut_down_async = True + + +@pytest.fixture +def dummy_span_processor() -> SpanProcessor: + return DummySpanProcessor() + + +def test_event_default_name_and_fields() -> None: + before_timestamp = time.time_ns() + e = Event() + after_timestamp = time.time_ns() + # Default name is class name + assert e.name == "Event" + # id should look like a uuid + assert isinstance(e.id, str) and UUID_RE.match(e.id) + # timestamp should be an integer in ns + assert before_timestamp <= e.timestamp <= after_timestamp and e.timestamp > 0 + + +def test_span_instantiation_defaults() -> None: + s = Span() + # Default name is class name + assert s.name == "Span" + # id should look like a uuid + assert isinstance(s.id, str) and UUID_RE.match(s.id) + # start/end times are None until the span is started/ended + assert s.start_time is None and s.end_time is None + # No events by default + assert s.events == [] + # Not entered as a context, shouldn't be active + assert get_current_span() is None + + +def test_exception_event_creation_defaults() -> None: + ex = ExceptionRaised(exception_type="ValueError", exception_message="bad input") + # Name defaults to class name + assert ex.name == "ExceptionRaised" + assert ex.exception_type == "ValueError" + # SensitiveField should accept plain str and retain value + assert str(ex.exception_message) == "bad input" + # Stacktrace has default + assert isinstance(ex.exception_stacktrace, str) + + +def test_span_start_end_and_active_stack() -> None: + stack_len_before = len(get_active_span_stack()) + s = Span(name="current_span") + before_span_start = time.time_ns() + s.start() + after_span_start_before_end = time.time_ns() + # Span s is the current one while active + assert get_current_span() is s + assert ( + s.start_time is not None + and before_span_start <= s.start_time <= after_span_start_before_end + ) + assert s.end_time is None + # Active stack grew by 1 + assert len(get_active_span_stack()) == stack_len_before + 1 + s.end() + after_span_end = time.time_ns() + # After exit, span is closed and stack restored + assert ( + s.start_time is not None + and before_span_start <= s.start_time <= after_span_start_before_end + ) + assert s.end_time is not None and after_span_start_before_end <= s.end_time <= after_span_end + assert get_current_span() is None + assert len(get_active_span_stack()) == stack_len_before + + +def test_span_start_end_and_active_stack_with_context_manager() -> None: + stack_len_before = len(get_active_span_stack()) + with Span(name="current_span") as s: + # Span s is the current one while active + assert get_current_span() is s + assert s.start_time is not None and s.end_time is None + # Active stack grew by 1 + assert len(get_active_span_stack()) == stack_len_before + 1 + # After exit, span is closed and stack restored + assert s.end_time is not None and s.end_time >= s.start_time # type: ignore[arg-type] + assert get_current_span() is None + assert len(get_active_span_stack()) == stack_len_before + + +def test_span_parent_span_and_get_current() -> None: + with Span() as parent: + with Span() as child: + # child recorded its parent span + assert child._parent_span is parent + assert get_current_span() is child + # After child exits, current is parent + assert get_current_span() is parent + assert get_current_span() is None + + +def test_spanprocessor_hooks_called_by_span(dummy_span_processor: DummySpanProcessor) -> None: + root_span = RootSpan() + with Trace(name="T1", span_processors=[dummy_span_processor], root_span=root_span) as t: + # Trace set in context + assert root_span in get_active_span_stack() + assert root_span is get_current_span() + assert dummy_span_processor.starts == [root_span] + assert get_trace() is t + with Span() as s: + assert s in get_active_span_stack() + # on_start called for processor + assert dummy_span_processor.starts == [root_span, s] + ev = Event(name="custom_event") + s.add_event(ev) + # Event recorded both in span and processor + assert s.events == [ev] + assert dummy_span_processor.events == [(ev, s)] + # on_end called + assert dummy_span_processor.ends == [s] + # Trace lifecycle hooks were invoked + assert dummy_span_processor.started_up is True + assert root_span in get_active_span_stack() + assert root_span is get_current_span() + assert dummy_span_processor.ends == [s, root_span] + assert dummy_span_processor.shut_down is True + # After exiting Trace, no active trace + assert get_trace() is None + + +def test_trace_startup_shutdown_and_nesting(dummy_span_processor: DummySpanProcessor) -> None: + with Trace(span_processors=[dummy_span_processor]): + assert dummy_span_processor.started_up is True + assert dummy_span_processor.started_up_async is False + assert get_trace() is not None + # Nested Trace not allowed + with pytest.raises(RuntimeError, match="A Trace already exists"): + with Trace(): + pass + assert dummy_span_processor.shut_down is True + assert dummy_span_processor.shut_down_async is False + + +def test_span_emits_exception_event_on_error() -> None: + with Trace(): + with pytest.raises(RuntimeError): + with Span() as s: + raise RuntimeError("boom") + # after exception, span ended and contains ExceptionRaised in events + assert any(isinstance(ev, ExceptionRaised) for ev in s.events) + + +def test_add_event_async_calls_async_handlers(dummy_span_processor: DummySpanProcessor) -> None: + + async def run(): + async_ev = Event(name="async_ev") + with Trace(span_processors=[dummy_span_processor]): + with Span() as s: + await s.add_event_async(async_ev) + return async_ev + + ev = asyncio.run(run()) + assert len(dummy_span_processor.events_async) == 1 + assert dummy_span_processor.events_async[0][0] is ev + + +# ========================= +# Asynchronous variants +# ========================= + + +def test_span_start_end_and_active_stack_async_manual( + dummy_span_processor: DummySpanProcessor, +) -> None: + root_span = RootSpan() + + async def run(): + stack_len_before = len(get_active_span_stack()) + async with Trace(span_processors=[dummy_span_processor], root_span=root_span) as t: + stack_len_in_trace = len(get_active_span_stack()) + assert stack_len_in_trace == stack_len_before + 1 + s = Span(name="current_span") + before_span_start = time.time_ns() + await s.start_async() + after_span_start_before_end = time.time_ns() + # Span s is the current one while active + assert get_current_span() is s + assert ( + s.start_time is not None + and before_span_start <= s.start_time <= after_span_start_before_end + ) + assert s.end_time is None + # Active stack grew by 1 + assert len(get_active_span_stack()) == stack_len_in_trace + 1 + await s.end_async() + after_span_end = time.time_ns() + # After exit, span is closed and stack restored + assert ( + s.start_time is not None + and before_span_start <= s.start_time <= after_span_start_before_end + ) + assert ( + s.end_time is not None + and after_span_start_before_end <= s.end_time <= after_span_end + ) + assert get_current_span() is t._root_span + assert get_current_span() is None + return stack_len_before, s + + stack_len_before, s = asyncio.run(run()) + # After loop, stack back to original size + assert len(get_active_span_stack()) == stack_len_before + # Async processor methods were invoked + assert dummy_span_processor.starts_async == [root_span, s] + assert dummy_span_processor.ends_async == [s, root_span] + + +def test_span_start_end_and_active_stack_with_async_context_manager( + dummy_span_processor: DummySpanProcessor, +) -> None: + root_span = RootSpan() + + async def run(): + stack_len_before = len(get_active_span_stack()) + async with Trace(span_processors=[dummy_span_processor], root_span=root_span): + assert root_span in get_active_span_stack() + assert root_span is get_current_span() + stack_len_in_trace = len(get_active_span_stack()) + assert stack_len_in_trace == stack_len_before + 1 + async with Span(name="current_span") as s: + # Span s is the current one while active + assert get_current_span() is s + assert s.start_time is not None and s.end_time is None + # Active stack grew by 1 + assert len(get_active_span_stack()) == stack_len_in_trace + 1 + # After exit, span is closed and stack restored + assert s.end_time is not None and s.end_time >= s.start_time + assert root_span in get_active_span_stack() + assert root_span is get_current_span() + assert get_current_span() is None + return stack_len_before, s + + stack_len_before, s = asyncio.run(run()) + assert len(get_active_span_stack()) == stack_len_before + assert dummy_span_processor.starts_async == [root_span, s] + assert dummy_span_processor.ends_async == [s, root_span] + + +def test_span_parent_span_and_get_current_async() -> None: + async def run(): + with Trace() as t: + async with Span() as parent: + async with Span() as child: + # child recorded its parent span + assert child._parent_span is parent + assert get_current_span() is child + # After child exits, current is parent + assert get_current_span() is parent + assert get_current_span() is t._root_span + assert get_current_span() is None + return True + + assert asyncio.run(run()) is True + + +def test_spanprocessor_hooks_called_by_span_async( + dummy_span_processor: DummySpanProcessor, +) -> None: + async def run(): + with Trace(name="T1-async", span_processors=[dummy_span_processor]) as t: + # Trace set in context + assert get_trace() is t + async with Span() as s: + assert s in get_active_span_stack() + # on_start_async called for processor + assert dummy_span_processor.starts_async == [s] + ev = Event(name="custom_event_async") + await s.add_event_async(ev) + # Event recorded both in span and processor (async) + assert s.events == [ev] + assert dummy_span_processor.events_async == [(ev, s)] + # on_end_async called + assert dummy_span_processor.ends_async == [s] # type: ignore[name-defined] + # Trace lifecycle hooks were invoked + assert dummy_span_processor.started_up is True + assert dummy_span_processor.shut_down is True + # After exiting Trace, no active trace + assert get_trace() is None + return True + + assert asyncio.run(run()) is True + + +def test_span_emits_exception_event_on_error_async() -> None: + async def run(): + with Trace(): + with pytest.raises(RuntimeError): + async with Span() as s: + raise RuntimeError("boom-async") + # after exception, span ended and contains ExceptionRaised in events + assert any(isinstance(ev, ExceptionRaised) for ev in s.events) + return True + + assert asyncio.run(run()) is True + + +def test_trace_startup_shutdown_and_nesting_async(dummy_span_processor: DummySpanProcessor) -> None: + + async def run(): + async with Trace(span_processors=[dummy_span_processor]): + assert dummy_span_processor.started_up is False + assert dummy_span_processor.started_up_async is True + assert get_trace() is not None + # Nested Trace not allowed + with pytest.raises(RuntimeError, match="A Trace already exists"): + with Trace(): + pass + assert dummy_span_processor.shut_down is False + assert dummy_span_processor.shut_down_async is True + return True + + assert asyncio.run(run()) is True