From 76d3b7e085c94a985e35bda7ec57a6d9424db5b2 Mon Sep 17 00:00:00 2001 From: teocns <59549574+teocns@users.noreply.github.com> Date: Sun, 1 Dec 2024 21:58:53 -0600 Subject: [PATCH] Otel exporter (#529) * feat(session): integrate OpenTelemetry for event tracing refactor(session): simplify jwt check in session method * remove deprecated `header` params from HttpClient Signed-off-by: Teo * refactor(http_client): use `header` * Move SessionExporter to `session` Signed-off-by: Teo * draft Signed-off-by: Teo * refactor(session): update session tracer provider management * rmv sess. tests * deactivate console exporter Signed-off-by: Teo * tests: mock processor to use linear simple span processor (no batch) * revert setup fixture and extra tests Signed-off-by: Teo * refactor(session): restore otel_exporter for span processing * test: Enhance assertions in session tests * refactor(session): reorder tracer initialization after session start * revert TestSingleSession.test_add_tags Signed-off-by: Teo * revert TestSingleSession.test_session Signed-off-by: Teo * fix: add missing api_key to endpoint calls * revert: Session._start_session Signed-off-by: Teo * match order of execution with original / ease of comparison Signed-off-by: Teo * remove queue attr Signed-off-by: Teo * refactor(session): persisted span processor attr; * flush_now flag Signed-off-by: Teo * small improves Signed-off-by: Teo * Improve general lifecycle management of OTEL Signed-off-by: Teo * Removed class-level state from SessionExporterEach Session now gets its own TracerProvider instance;; Shutdown flag is now instance-level Signed-off-by: Teo * 14 errors Signed-off-by: Teo * 13 errors Signed-off-by: Teo * 1 error Signed-off-by: Teo * 0 error Signed-off-by: Teo * Cleanup deps Signed-off-by: Teo * refactored code for `get_analytics` method merged in `main` * tests for the `get_analytics` method * linting * oops * add tests targeting SessionExporter Failing: test_export_with_missing_timestamp Signed-off-by: Teo * SessionExporter: Added default value using current UTC time when end_timestamp is None Signed-off-by: Teo * Moved timestamp handling earlier in the process, before OpenTelemetry validation Signed-off-by: Teo * test: add test for exporting LLM event handling * test: add test for handling missing event ID in export * add session url * feat(HttpClient): add session management and header preparation * feat(HttpClient): Cache host env * refactor(HttpClient): improve session management and headers * feat(session): add static resource management in exporter delete(tests): remove conftest.py test configuration file refactor(session): remove unused resource and tracer provider methods * feat(session): force flush pending spans on session end * replace core manual test * remove log flag * refactor(client): simplify host_env retrieval logic * refactor(http_client): covnert _prepare_headers to classmethod * ruff Signed-off-by: Teo * added cost param to LLMEvent (how was this not here before??) * fix autogen. Added costs and completions * remove prints * better completion grabber * revert autogen line * updated autogen completions * black fixes * Revert "updated autogen completions" This reverts commit 8d542c0b9ceed8e31e933b284febc95b1b10f324. * ruff format * revert notebook * revert math notebook * revert ollama notebook * revert anthropic notebook --------- Signed-off-by: Teo Co-authored-by: Pratyush Shukla Co-authored-by: reibs --- agentops/client.py | 27 +- agentops/event.py | 1 + agentops/http_client.py | 108 +- agentops/llms/anthropic.py | 6 +- agentops/partners/autogen_logger.py | 8 +- agentops/session.py | 585 +++-- ...entops-anthropic-understanding-tools.ipynb | 2049 ++++++++--------- examples/autogen_examples/MathAgent.ipynb | 473 ++-- .../ollama_examples/ollama_examples.ipynb | 423 ++-- pyproject.toml | 11 +- tests/core_manual_tests/api_server/main.py | 37 + tests/core_manual_tests/api_server/readme.md | 7 +- tests/core_manual_tests/api_server/server.py | 39 - tests/core_manual_tests/benchmark.py | 50 + tests/test_session.py | 253 +- 15 files changed, 2326 insertions(+), 1751 deletions(-) create mode 100644 tests/core_manual_tests/api_server/main.py delete mode 100644 tests/core_manual_tests/api_server/server.py create mode 100644 tests/core_manual_tests/benchmark.py diff --git a/agentops/client.py b/agentops/client.py index 86fe49b8f..d1bb51db7 100644 --- a/agentops/client.py +++ b/agentops/client.py @@ -5,8 +5,8 @@ Client: Provides methods to interact with the AgentOps service. """ -import inspect import atexit +import inspect import logging import os import signal @@ -14,20 +14,20 @@ import threading import traceback from decimal import Decimal +from functools import cached_property +from typing import List, Optional, Tuple, Union from uuid import UUID, uuid4 -from typing import Optional, List, Union, Tuple + from termcolor import colored -from .event import Event, ErrorEvent -from .singleton import ( - conditional_singleton, -) -from .session import Session, active_sessions +from .config import Configuration +from .event import ErrorEvent, Event from .host_env import get_host_env +from .llms import LlmTracker from .log_config import logger from .meta_client import MetaClient -from .config import Configuration -from .llms import LlmTracker +from .session import Session, active_sessions +from .singleton import conditional_singleton @conditional_singleton @@ -39,6 +39,7 @@ def __init__(self): self._sessions: List[Session] = active_sessions self._config = Configuration() self._pre_init_queue = {"agents": []} + self._host_env = None # Cache host env data self.configure( api_key=os.environ.get("AGENTOPS_API_KEY"), @@ -111,6 +112,7 @@ def initialize(self) -> Union[Session, None]: def _initialize_autogen_logger(self) -> None: try: import autogen + from .partners.autogen_logger import AutogenLogger autogen.runtime_logging.start(logger=AutogenLogger()) @@ -224,7 +226,7 @@ def start_session( session = Session( session_id=session_id, tags=list(session_tags), - host_env=get_host_env(self._config.env_data_opt_out), + host_env=self.host_env, config=self._config, ) @@ -430,3 +432,8 @@ def api_key(self): @property def parent_key(self): return self._config.parent_key + + @cached_property + def host_env(self): + """Cache and reuse host environment data""" + return get_host_env(self._config.env_data_opt_out) diff --git a/agentops/event.py b/agentops/event.py index 70ec059cd..c6200aca1 100644 --- a/agentops/event.py +++ b/agentops/event.py @@ -82,6 +82,7 @@ class LLMEvent(Event): prompt_tokens: Optional[int] = None completion: Union[str, object] = None completion_tokens: Optional[int] = None + cost: Optional[float] = None model: Optional[str] = None diff --git a/agentops/http_client.py b/agentops/http_client.py index caa18b279..11c0bf49f 100644 --- a/agentops/http_client.py +++ b/agentops/http_client.py @@ -1,7 +1,9 @@ from enum import Enum -from typing import Optional -from requests.adapters import Retry, HTTPAdapter +from typing import Optional, Dict, Any + import requests +from requests.adapters import HTTPAdapter, Retry +import json from .exceptions import ApiServerException @@ -54,33 +56,79 @@ def get_status(code: int) -> HttpStatus: class HttpClient: - @staticmethod + _session: Optional[requests.Session] = None + + @classmethod + def get_session(cls) -> requests.Session: + """Get or create the global session with optimized connection pooling""" + if cls._session is None: + cls._session = requests.Session() + + # Configure connection pooling + adapter = requests.adapters.HTTPAdapter( + pool_connections=15, # Number of connection pools + pool_maxsize=256, # Connections per pool + max_retries=Retry(total=3, backoff_factor=0.1, status_forcelist=[500, 502, 503, 504]), + ) + + # Mount adapter for both HTTP and HTTPS + cls._session.mount("http://", adapter) + cls._session.mount("https://", adapter) + + # Set default headers + cls._session.headers.update( + { + "Connection": "keep-alive", + "Keep-Alive": "timeout=10, max=1000", + "Content-Type": "application/json", + } + ) + + return cls._session + + @classmethod + def _prepare_headers( + cls, + api_key: Optional[str] = None, + parent_key: Optional[str] = None, + jwt: Optional[str] = None, + custom_headers: Optional[dict] = None, + ) -> dict: + """Prepare headers for the request""" + headers = JSON_HEADER.copy() + + if api_key is not None: + headers["X-Agentops-Api-Key"] = api_key + + if parent_key is not None: + headers["X-Agentops-Parent-Key"] = parent_key + + if jwt is not None: + headers["Authorization"] = f"Bearer {jwt}" + + if custom_headers is not None: + headers.update(custom_headers) + + return headers + + @classmethod def post( + cls, url: str, payload: bytes, api_key: Optional[str] = None, parent_key: Optional[str] = None, jwt: Optional[str] = None, - header=None, + header: Optional[Dict[str, str]] = None, ) -> Response: + """Make HTTP POST request using connection pooling""" result = Response() try: - # Create request session with retries configured - request_session = requests.Session() - request_session.mount(url, HTTPAdapter(max_retries=retry_config)) - - if api_key is not None: - JSON_HEADER["X-Agentops-Api-Key"] = api_key - - if parent_key is not None: - JSON_HEADER["X-Agentops-Parent-Key"] = parent_key - - if jwt is not None: - JSON_HEADER["Authorization"] = f"Bearer {jwt}" - - res = request_session.post(url, data=payload, headers=JSON_HEADER, timeout=20) - + headers = cls._prepare_headers(api_key, parent_key, jwt, header) + session = cls.get_session() + res = session.post(url, data=payload, headers=headers, timeout=20) result.parse(res) + except requests.exceptions.Timeout: result.code = 408 result.status = HttpStatus.TIMEOUT @@ -112,28 +160,22 @@ def post( return result - @staticmethod + @classmethod def get( + cls, url: str, api_key: Optional[str] = None, jwt: Optional[str] = None, - header=None, + header: Optional[Dict[str, str]] = None, ) -> Response: + """Make HTTP GET request using connection pooling""" result = Response() try: - # Create request session with retries configured - request_session = requests.Session() - request_session.mount(url, HTTPAdapter(max_retries=retry_config)) - - if api_key is not None: - JSON_HEADER["X-Agentops-Api-Key"] = api_key - - if jwt is not None: - JSON_HEADER["Authorization"] = f"Bearer {jwt}" - - res = request_session.get(url, headers=JSON_HEADER, timeout=20) - + headers = cls._prepare_headers(api_key, None, jwt, header) + session = cls.get_session() + res = session.get(url, headers=headers, timeout=20) result.parse(res) + except requests.exceptions.Timeout: result.code = 408 result.status = HttpStatus.TIMEOUT diff --git a/agentops/llms/anthropic.py b/agentops/llms/anthropic.py index 0ec1c1841..0a7293407 100644 --- a/agentops/llms/anthropic.py +++ b/agentops/llms/anthropic.py @@ -137,7 +137,7 @@ async def async_generator(): The raw response has the following structure: { - 'id': str, # Message ID (e.g. 'msg_018Gk9N2pcWaYLS7mxXbPD5i') + 'id': str, # Message ID (e.g. 'msg_018Gk9N2pcWaYLS7mxXbPD5i') 'type': str, # Type of response (e.g. 'message') 'role': str, # Role of responder (e.g. 'assistant') 'model': str, # Model used (e.g. 'claude-3-5-sonnet-20241022') @@ -151,7 +151,7 @@ async def async_generator(): } Note: We import Anthropic types here since the package must be installed - for raw responses to be available; doing so in the global scope would + for raw responses to be available; doing so in the global scope would result in dependencies error since this provider is not lazily imported (tests fail) """ from anthropic import APIResponse @@ -167,7 +167,7 @@ async def async_generator(): llm_event.model = response_data["model"] llm_event.completion = { "role": response_data.get("role"), - "content": response_data.get("content")[0].get("text") if response_data.get("content") else "", + "content": (response_data.get("content")[0].get("text") if response_data.get("content") else ""), } if usage := response_data.get("usage"): llm_event.prompt_tokens = usage.get("input_tokens") diff --git a/agentops/partners/autogen_logger.py b/agentops/partners/autogen_logger.py index 77aca142a..9520801fa 100644 --- a/agentops/partners/autogen_logger.py +++ b/agentops/partners/autogen_logger.py @@ -10,9 +10,9 @@ from openai.types.chat import ChatCompletion from autogen.logger.base_logger import BaseLogger, LLMConfig -from autogen.logger.logger_utils import get_current_ts, to_dict from agentops.enums import EndState +from agentops.helpers import get_ISO_time from agentops import LLMEvent, ToolEvent, ActionEvent from uuid import uuid4 @@ -55,17 +55,19 @@ def log_chat_completion( start_time: str, ) -> None: """Records an LLMEvent to AgentOps session""" - end_time = get_current_ts() completion = response.choices[len(response.choices) - 1] + # Note: Autogen tokens are not included in the request and function call tokens are not counted in the completion llm_event = LLMEvent( prompt=request["messages"], completion=completion.message, model=response.model, + cost=cost, + returns=completion.message.to_json(), ) llm_event.init_timestamp = start_time - llm_event.end_timestamp = end_time + llm_event.end_timestamp = get_ISO_time() llm_event.agent_id = self._get_agentops_id_from_agent(str(id(agent))) agentops.record(llm_event) diff --git a/agentops/session.py b/agentops/session.py index 1213e2fd4..535bc997d 100644 --- a/agentops/session.py +++ b/agentops/session.py @@ -1,21 +1,168 @@ -import copy +from __future__ import annotations + +import asyncio import functools import json import threading -import time +from datetime import datetime, timezone from decimal import ROUND_HALF_UP, Decimal -from termcolor import colored -from typing import Any, Optional, List, Union +from typing import Any, Dict, List, Optional, Sequence, Union from uuid import UUID, uuid4 -from datetime import datetime -from .exceptions import ApiServerException +from opentelemetry import trace +from opentelemetry.context import attach, detach, set_value +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import ( + BatchSpanProcessor, + ConsoleSpanExporter, + SpanExporter, + SpanExportResult, +) +from termcolor import colored + +from .config import Configuration from .enums import EndState from .event import ErrorEvent, Event -from .log_config import logger -from .config import Configuration -from .helpers import get_ISO_time, filter_unjsonable, safe_serialize +from .exceptions import ApiServerException +from .helpers import filter_unjsonable, get_ISO_time, safe_serialize from .http_client import HttpClient, Response +from .log_config import logger + +""" +OTEL Guidelines: + + + +- Maintain a single TracerProvider for the application runtime + - Have one global TracerProvider in the Client class + +- According to the OpenTelemetry Python documentation, Resource should be initialized once per application and shared across all telemetry (traces, metrics, logs). +- Each Session gets its own Tracer (with session-specific context) +- Allow multiple sessions to share the provider while maintaining their own context + + + +:: Resource + + '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + Captures information about the entity producing telemetry as Attributes. + For example, a process producing telemetry that is running in a container + on Kubernetes has a process name, a pod name, a namespace, and possibly + a deployment name. All these attributes can be included in the Resource. + '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + + The key insight from the documentation is: + + - Resource represents the entity producing telemetry - in our case, that's the AgentOps SDK application itself + - Session-specific information should be attributes on the spans themselves + - A Resource is meant to identify the service/process/application1 + - Sessions are units of work within that application + - The documentation example about "process name, pod name, namespace" refers to where the code is running, not the work it's doing + +""" + + +class SessionExporter(SpanExporter): + """ + Manages publishing events for Session + """ + + def __init__(self, session: Session, **kwargs): + self.session = session + self._shutdown = threading.Event() + self._export_lock = threading.Lock() + super().__init__(**kwargs) + + @property + def endpoint(self): + return f"{self.session.config.endpoint}/v2/create_events" + + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: + if self._shutdown.is_set(): + return SpanExportResult.SUCCESS + + with self._export_lock: + try: + # Skip if no spans to export + if not spans: + return SpanExportResult.SUCCESS + + events = [] + for span in spans: + event_data = json.loads(span.attributes.get("event.data", "{}")) + + # Format event data based on event type + if span.name == "actions": + formatted_data = { + "action_type": event_data.get("action_type", event_data.get("name", "unknown_action")), + "params": event_data.get("params", {}), + "returns": event_data.get("returns"), + } + elif span.name == "tools": + formatted_data = { + "name": event_data.get("name", event_data.get("tool_name", "unknown_tool")), + "params": event_data.get("params", {}), + "returns": event_data.get("returns"), + } + else: + formatted_data = event_data + + formatted_data = {**event_data, **formatted_data} + # Get timestamps, providing defaults if missing + current_time = datetime.now(timezone.utc).isoformat() + init_timestamp = span.attributes.get("event.timestamp") + end_timestamp = span.attributes.get("event.end_timestamp") + + # Handle missing timestamps + if init_timestamp is None: + init_timestamp = current_time + if end_timestamp is None: + end_timestamp = current_time + + # Get event ID, generate new one if missing + event_id = span.attributes.get("event.id") + if event_id is None: + event_id = str(uuid4()) + + events.append( + { + "id": event_id, + "event_type": span.name, + "init_timestamp": init_timestamp, + "end_timestamp": end_timestamp, + **formatted_data, + "session_id": str(self.session.session_id), + } + ) + + # Only make HTTP request if we have events and not shutdown + if events: + try: + res = HttpClient.post( + self.endpoint, + json.dumps({"events": events}).encode("utf-8"), + api_key=self.session.config.api_key, + jwt=self.session.jwt, + ) + return SpanExportResult.SUCCESS if res.code == 200 else SpanExportResult.FAILURE + except Exception as e: + logger.error(f"Failed to send events: {e}") + return SpanExportResult.FAILURE + + return SpanExportResult.SUCCESS + + except Exception as e: + logger.error(f"Failed to export spans: {e}") + return SpanExportResult.FAILURE + + def force_flush(self, timeout_millis: Optional[int] = None) -> bool: + return True + + def shutdown(self) -> None: + """Handle shutdown gracefully""" + self._shutdown.set() + # Don't call session.end_session() here to avoid circular dependencies class Session: @@ -67,9 +214,10 @@ def __init__( self.host_env = host_env self.config = config self.jwt = None - self.lock = threading.Lock() - self.queue: List[Any] = [] - self.token_cost = Decimal(0) + self._lock = threading.Lock() + self._end_session_lock = threading.Lock() + self.token_cost: Decimal = Decimal(0) + self._session_url: str = "" self.event_counts = { "llms": 0, "tools": 0, @@ -77,17 +225,33 @@ def __init__( "errors": 0, "apis": 0, } - self.session_url: Optional[str] = None - - self.stop_flag = threading.Event() - self.thread = threading.Thread(target=self._run) - self.thread.daemon = True - self.thread.start() + # self.session_url: Optional[str] = None + # Start session first to get JWT self.is_running = self._start_session() - if self.is_running == False: - self.stop_flag.set() - self.thread.join(timeout=1) + if not self.is_running: + return + + # Initialize OTEL components with a more controlled processor + self._tracer_provider = TracerProvider() + self._otel_tracer = self._tracer_provider.get_tracer( + f"agentops.session.{str(session_id)}", + ) + self._otel_exporter = SessionExporter(session=self) + + # Use smaller batch size and shorter delay to reduce buffering + self._span_processor = BatchSpanProcessor( + self._otel_exporter, + max_queue_size=self.config.max_queue_size, + schedule_delay_millis=self.config.max_wait_time, + max_export_batch_size=min( + max(self.config.max_queue_size // 20, 1), + min(self.config.max_queue_size, 32), + ), + export_timeout_millis=20000, + ) + + self._tracer_provider.add_span_processor(self._span_processor) def set_video(self, video: str) -> None: """ @@ -98,57 +262,99 @@ def set_video(self, video: str) -> None: """ self.video = video + def _flush_spans(self) -> bool: + """ + Flush pending spans for this specific session with timeout. + Returns True if flush was successful, False otherwise. + """ + if not hasattr(self, "_span_processor"): + return True + + try: + success = self._span_processor.force_flush(timeout_millis=self.config.max_wait_time) + if not success: + logger.warning("Failed to flush all spans before session end") + return success + except Exception as e: + logger.warning(f"Error flushing spans: {e}") + return False + def end_session( self, end_state: str = "Indeterminate", end_state_reason: Optional[str] = None, video: Optional[str] = None, ) -> Union[Decimal, None]: - if not self.is_running: - return None + with self._end_session_lock: + if not self.is_running: + return None - if not any(end_state == state.value for state in EndState): - logger.warning("Invalid end_state. Please use one of the EndState enums") - return None + if not any(end_state == state.value for state in EndState): + logger.warning("Invalid end_state. Please use one of the EndState enums") + return None - self.end_timestamp = get_ISO_time() - self.end_state = end_state - self.end_state_reason = end_state_reason - if video is not None: - self.video = video - - self.stop_flag.set() - self.thread.join(timeout=1) - self._flush_queue() - analytics_stats = self.get_analytics() - - analytics = ( - f"Session Stats - " - f"{colored('Duration:', attrs=['bold'])} {analytics_stats['Duration']} | " - f"{colored('Cost:', attrs=['bold'])} ${analytics_stats['Cost']} | " - f"{colored('LLMs:', attrs=['bold'])} {analytics_stats['LLM calls']} | " - f"{colored('Tools:', attrs=['bold'])} {analytics_stats['Tool calls']} | " - f"{colored('Actions:', attrs=['bold'])} {analytics_stats['Actions']} | " - f"{colored('Errors:', attrs=['bold'])} {analytics_stats['Errors']}" - ) - logger.info(analytics) + try: + # Force flush any pending spans before ending session + if hasattr(self, "_span_processor"): + self._span_processor.force_flush(timeout_millis=5000) + + # 1. Set shutdown flag on exporter first + if hasattr(self, "_otel_exporter"): + self._otel_exporter.shutdown() + + # 2. Set session end state + self.end_timestamp = get_ISO_time() + self.end_state = end_state + self.end_state_reason = end_state_reason + if video is not None: + self.video = video + + # 3. Mark session as not running before cleanup + self.is_running = False + + # 4. Clean up OTEL components + if hasattr(self, "_span_processor"): + try: + # Force flush any pending spans + self._span_processor.force_flush(timeout_millis=5000) + # Shutdown the processor + self._span_processor.shutdown() + except Exception as e: + logger.warning(f"Error during span processor cleanup: {e}") + finally: + del self._span_processor + + # 5. Final session update + if not (analytics_stats := self.get_analytics()): + return None + + analytics = ( + f"Session Stats - " + f"{colored('Duration:', attrs=['bold'])} {analytics_stats['Duration']} | " + f"{colored('Cost:', attrs=['bold'])} ${analytics_stats['Cost']} | " + f"{colored('LLMs:', attrs=['bold'])} {analytics_stats['LLM calls']} | " + f"{colored('Tools:', attrs=['bold'])} {analytics_stats['Tool calls']} | " + f"{colored('Actions:', attrs=['bold'])} {analytics_stats['Actions']} | " + f"{colored('Errors:', attrs=['bold'])} {analytics_stats['Errors']}" + ) + logger.info(analytics) - logger.info( - colored( - f"\x1b[34mSession Replay: {self.session_url}\x1b[0m", - "blue", - ) - ) - active_sessions.remove(self) + except Exception as e: + logger.exception(f"Error during session end: {e}") + finally: + active_sessions.remove(self) # First thing, get rid of the session - return self.token_cost + logger.info( + colored( + f"\x1b[34mSession Replay: {self.session_url}\x1b[0m", + "blue", + ) + ) + return self.token_cost def add_tags(self, tags: List[str]) -> None: """ Append to session tags at runtime. - - Args: - tags (List[str]): The list of tags to append. """ if not self.is_running: return @@ -157,16 +363,20 @@ def add_tags(self, tags: List[str]) -> None: if isinstance(tags, str): tags = [tags] + # Initialize tags if None if self.tags is None: - self.tags = tags - else: - for tag in tags: - if tag not in self.tags: - self.tags.append(tag) + self.tags = [] + + # Add new tags that don't exist + for tag in tags: + if tag not in self.tags: + self.tags.append(tag) + # Update session state immediately self._update_session() def set_tags(self, tags): + """Set session tags, replacing any existing tags""" if not self.is_running: return @@ -174,39 +384,101 @@ def set_tags(self, tags): if isinstance(tags, str): tags = [tags] - self.tags = tags + # Set tags directly + self.tags = tags.copy() # Make a copy to avoid reference issues + + # Update session state immediately self._update_session() - def record(self, event: Union[Event, ErrorEvent]): + def record(self, event: Union[Event, ErrorEvent], flush_now=False): + """Record an event using OpenTelemetry spans""" if not self.is_running: return - if isinstance(event, Event): - if not event.end_timestamp or event.init_timestamp == event.end_timestamp: - event.end_timestamp = get_ISO_time() - elif isinstance(event, ErrorEvent): - if event.trigger_event: - if ( - not event.trigger_event.end_timestamp - or event.trigger_event.init_timestamp == event.trigger_event.end_timestamp - ): - event.trigger_event.end_timestamp = get_ISO_time() - - event.trigger_event_id = event.trigger_event.id - event.trigger_event_type = event.trigger_event.event_type - self._add_event(event.trigger_event.__dict__) - event.trigger_event = None # removes trigger_event from serialization - - self._add_event(event.__dict__) - - def _add_event(self, event: dict) -> None: - with self.lock: - self.queue.append(event) - - if len(self.queue) >= self.config.max_queue_size: - self._flush_queue() + + # Ensure event has all required base attributes + if not hasattr(event, "id"): + event.id = uuid4() + if not hasattr(event, "init_timestamp"): + event.init_timestamp = get_ISO_time() + if not hasattr(event, "end_timestamp") or event.end_timestamp is None: + event.end_timestamp = get_ISO_time() + + # Create session context + token = set_value("session.id", str(self.session_id)) + + try: + token = attach(token) + + # Create a copy of event data to modify + event_data = dict(filter_unjsonable(event.__dict__)) + + # Add required fields based on event type + if isinstance(event, ErrorEvent): + event_data["error_type"] = getattr(event, "error_type", event.event_type) + elif event.event_type == "actions": + # Ensure action events have action_type + if "action_type" not in event_data: + event_data["action_type"] = event_data.get("name", "unknown_action") + if "name" not in event_data: + event_data["name"] = event_data.get("action_type", "unknown_action") + elif event.event_type == "tools": + # Ensure tool events have name + if "name" not in event_data: + event_data["name"] = event_data.get("tool_name", "unknown_tool") + if "tool_name" not in event_data: + event_data["tool_name"] = event_data.get("name", "unknown_tool") + + with self._otel_tracer.start_as_current_span( + name=event.event_type, + attributes={ + "event.id": str(event.id), + "event.type": event.event_type, + "event.timestamp": event.init_timestamp or get_ISO_time(), + "event.end_timestamp": event.end_timestamp or get_ISO_time(), + "session.id": str(self.session_id), + "session.tags": ",".join(self.tags) if self.tags else "", + "event.data": json.dumps(event_data), + }, + ) as span: + if event.event_type in self.event_counts: + self.event_counts[event.event_type] += 1 + + if isinstance(event, ErrorEvent): + span.set_attribute("error", True) + if hasattr(event, "trigger_event") and event.trigger_event: + span.set_attribute("trigger_event.id", str(event.trigger_event.id)) + span.set_attribute("trigger_event.type", event.trigger_event.event_type) + + if flush_now and hasattr(self, "_span_processor"): + self._span_processor.force_flush() + finally: + detach(token) + + def _send_event(self, event): + """Direct event sending for testing""" + try: + payload = { + "events": [ + { + "id": str(event.id), + "event_type": event.event_type, + "init_timestamp": event.init_timestamp, + "end_timestamp": event.end_timestamp, + "data": filter_unjsonable(event.__dict__), + } + ] + } + + HttpClient.post( + f"{self.config.endpoint}/v2/create_events", + json.dumps(payload).encode("utf-8"), + jwt=self.jwt, + ) + except Exception as e: + logger.error(f"Failed to send event: {e}") def _reauthorize_jwt(self) -> Union[str, None]: - with self.lock: + with self._lock: payload = {"session_id": self.session_id} serialized_payload = json.dumps(filter_unjsonable(payload)).encode("utf-8") res = HttpClient.post( @@ -225,8 +497,7 @@ def _reauthorize_jwt(self) -> Union[str, None]: return jwt def _start_session(self): - self.queue = [] - with self.lock: + with self._lock: payload = {"session": self.__dict__} serialized_payload = json.dumps(filter_unjsonable(payload)).encode("utf-8") @@ -234,8 +505,8 @@ def _start_session(self): res = HttpClient.post( f"{self.config.endpoint}/v2/create_session", serialized_payload, - self.config.api_key, - self.config.parent_key, + api_key=self.config.api_key, + parent_key=self.config.parent_key, ) except ApiServerException as e: return logger.error(f"Could not start session - {e}") @@ -250,14 +521,9 @@ def _start_session(self): if jwt is None: return False - session_url = res.body.get( - "session_url", - f"https://app.agentops.ai/drilldown?session_id={self.session_id}", - ) - logger.info( colored( - f"\x1b[34mSession Replay: {session_url}\x1b[0m", + f"\x1b[34mSession Replay: {self.session_url}\x1b[0m", "blue", ) ) @@ -265,68 +531,22 @@ def _start_session(self): return True def _update_session(self) -> None: + """Update session state on the server""" if not self.is_running: return - with self.lock: + with self._lock: # TODO: Determine whether we really need to lock here: are incoming calls coming from other threads? payload = {"session": self.__dict__} try: res = HttpClient.post( f"{self.config.endpoint}/v2/update_session", json.dumps(filter_unjsonable(payload)).encode("utf-8"), + # self.config.api_key, jwt=self.jwt, ) except ApiServerException as e: return logger.error(f"Could not update session - {e}") - def _flush_queue(self) -> None: - if not self.is_running: - return - with self.lock: - queue_copy = self.queue[:] # Copy the current items - self.queue = [] - - if len(queue_copy) > 0: - payload = { - "events": queue_copy, - } - - serialized_payload = safe_serialize(payload).encode("utf-8") - try: - HttpClient.post( - f"{self.config.endpoint}/v2/create_events", - serialized_payload, - jwt=self.jwt, - ) - except ApiServerException as e: - return logger.error(f"Could not post events - {e}") - - logger.debug("\n") - logger.debug(f"Session request to {self.config.endpoint}/v2/create_events") - logger.debug(serialized_payload) - logger.debug("\n") - - # Count total events created based on type - events = payload["events"] - for event in events: - event_type = event["event_type"] - if event_type == "llms": - self.event_counts["llms"] += 1 - elif event_type == "tools": - self.event_counts["tools"] += 1 - elif event_type == "actions": - self.event_counts["actions"] += 1 - elif event_type == "errors": - self.event_counts["errors"] += 1 - elif event_type == "apis": - self.event_counts["apis"] += 1 - - def _run(self) -> None: - while not self.stop_flag.is_set(): - time.sleep(self.config.max_wait_time / 1000) - if self.queue: - self._flush_queue() - def create_agent(self, name, agent_id): if not self.is_running: return @@ -343,6 +563,7 @@ def create_agent(self, name, agent_id): HttpClient.post( f"{self.config.endpoint}/v2/create_agent", serialized_payload, + api_key=self.config.api_key, jwt=self.jwt, ) except ApiServerException as e: @@ -358,8 +579,22 @@ def wrapper(*args, **kwargs): return wrapper - @staticmethod - def _format_duration(start_time, end_time): + def _get_response(self) -> Optional[Response]: + payload = {"session": self.__dict__} + try: + response = HttpClient.post( + f"{self.config.endpoint}/v2/update_session", + json.dumps(filter_unjsonable(payload)).encode("utf-8"), + api_key=self.config.api_key, + jwt=self.jwt, + ) + except ApiServerException as e: + return logger.error(f"Could not end session - {e}") + + logger.debug(response.body) + return response + + def _format_duration(self, start_time, end_time) -> str: start = datetime.fromisoformat(start_time.replace("Z", "+00:00")) end = datetime.fromisoformat(end_time.replace("Z", "+00:00")) duration = end - start @@ -376,53 +611,29 @@ def _format_duration(start_time, end_time): return " ".join(parts) - def _get_response(self) -> Optional[Response]: - with self.lock: - payload = {"session": self.__dict__} - try: - response = HttpClient.post( - f"{self.config.endpoint}/v2/update_session", - json.dumps(filter_unjsonable(payload)).encode("utf-8"), - jwt=self.jwt, - ) - except ApiServerException as e: - logger.error(f"Could not fetch response from server - {e}") - return None - - logger.debug(response.body) - return response - def _get_token_cost(self, response: Response) -> Decimal: token_cost = response.body.get("token_cost", "unknown") if token_cost == "unknown" or token_cost is None: return Decimal(0) return Decimal(token_cost) - @staticmethod - def _format_token_cost(token_cost_d): + def _format_token_cost(self, token_cost: Decimal) -> str: return ( - "{:.2f}".format(token_cost_d) - if token_cost_d == 0 - else "{:.6f}".format(token_cost_d.quantize(Decimal("0.000001"), rounding=ROUND_HALF_UP)) + "{:.2f}".format(token_cost) + if token_cost == 0 + else "{:.6f}".format(token_cost.quantize(Decimal("0.000001"), rounding=ROUND_HALF_UP)) ) - def get_analytics(self) -> Optional[dict[str, Union[Decimal, str]]]: + def get_analytics(self) -> Optional[Dict[str, Any]]: if not self.end_timestamp: self.end_timestamp = get_ISO_time() formatted_duration = self._format_duration(self.init_timestamp, self.end_timestamp) - response = self._get_response() - if response is None: + if (response := self._get_response()) is None: return None self.token_cost = self._get_token_cost(response) - formatted_cost = self._format_token_cost(self.token_cost) - - self.session_url = response.body.get( - "session_url", - f"https://app.agentops.ai/drilldown?session_id={self.session_id}", - ) return { "LLM calls": self.event_counts["llms"], @@ -430,8 +641,18 @@ def get_analytics(self) -> Optional[dict[str, Union[Decimal, str]]]: "Actions": self.event_counts["actions"], "Errors": self.event_counts["errors"], "Duration": formatted_duration, - "Cost": formatted_cost, + "Cost": self._format_token_cost(self.token_cost), } + @property + def session_url(self) -> str: + """Returns the URL for this session in the AgentOps dashboard.""" + assert self.session_id, "Session ID is required to generate a session URL" + return f"https://app.agentops.ai/drilldown?session_id={self.session_id}" + + # @session_url.setter + # def session_url(self, url: str): + # pass + active_sessions: List[Session] = [] diff --git a/examples/anthropic_examples/agentops-anthropic-understanding-tools.ipynb b/examples/anthropic_examples/agentops-anthropic-understanding-tools.ipynb index 9cc95c57b..17bea5def 100644 --- a/examples/anthropic_examples/agentops-anthropic-understanding-tools.ipynb +++ b/examples/anthropic_examples/agentops-anthropic-understanding-tools.ipynb @@ -1,1037 +1,1030 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Anthropic's tool returns are not as simple as getting a few strings! While this system is more complex than those before it, it's also simple enough to be used without problem once you understand how it works! " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To get started, we will import Agentops and Anthropic" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install agentops\n", - "!pip install anthropic" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Setup our generic default statements" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "from anthropic import Anthropic, AsyncAnthropic\n", - "import agentops\n", - "import os\n", - "import random #We don't need this for agentops, we use this to generate a message later\n", - "import time #We don't need this for agentops either, we use this when simulating an API later\n", - "import re #Regex for formatting\n", - "from dotenv import load_dotenv" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And set our API keys." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "load_dotenv()\n", - "ANTHROPIC_API_KEY = os.getenv(\"ANTHROPIC_API_KEY\") or \"ANTHROPIC API KEY\"\n", - "AGENTOPS_API_KEY = os.getenv(\"AGENTOPS_API_KEY\") or \"AGENTOPS API KEY\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "Now let's set the client as Anthropic and make an AgentOps session" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "🖇 AgentOps: \u001b[34m\u001b[34mSession Replay: https://app.agentops.ai/drilldown?session_id=df6426c5-1995-4bec-b90e-680369a0cddb\u001b[0m\u001b[0m\n" - ] - }, - { - "data": { - "text/plain": [ - "" + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Anthropic's tool returns are not as simple as getting a few strings! While this system is more complex than those before it, it's also simple enough to be used without problem once you understand how it works! " ] }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "agentops.init(AGENTOPS_API_KEY, default_tags=[\"anthropic-example-tool-tutorials\"])" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "client = Anthropic(api_key=ANTHROPIC_API_KEY)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now to create a simple dummy tool! We are going to make a tool that will tell us about the demon infestation levels for 3 areas. From there, we will have VEGA, our AI determine the best place for the Doom Slayer to attack." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "locations = [\n", - " {\n", - " \"Name\": \"Super Gore Nest\",\n", - " \"Description\": \"A grotesque mass of demonic growth and organic structures infesting the ruins of an urban area on Earth. The Super Gore Nest serves as a massive, pulsating hive for Hell’s forces, complete with rivers of blood, twisted tendrils, and a dark, organic design that shows how deeply Hell has taken root in the city.\"\n", - " },\n", - " {\n", - " \"Name\": \"Exultia\",\n", - " \"Description\": \"An ancient, mystical world that holds the ruins of the Night Sentinels' kingdom, with gothic structures and arcane symbols throughout. This realm is filled with epic landscapes, medieval architecture, and hints of the powerful civilization that once defended against Hell’s forces.\"\n", - " },\n", - " {\n", - " \"Name\": \"Cultist Base\",\n", - " \"Description\": \"A grim fortress hidden within the icy mountains, where a fanatical cult worships demons. Filled with chilling sacrificial chambers, traps, and rituals, the Cultist Base is a hostile stronghold where Doom Slayer must confront the cult leaders aiding Hell's invasion of Earth.\"\n", - " },\n", - " {\n", - " \"Name\": \"Taras Nabad\",\n", - " \"Description\": \"A war-ravaged city on the homeworld of the Night Sentinels, showcasing grandiose, ancient architecture in the midst of destruction. Taras Nabad's sprawling structures and historical significance reveal glimpses into the Doom Slayer’s past and the once-thriving Sentinel civilization.\"\n", - " },\n", - " {\n", - " \"Name\": \"Nekravol\",\n", - " \"Description\": \"A hellish, industrial fortress where souls are processed into Argent energy. With conveyor belts moving the damned and a skyline dominated by fire and darkness, Nekravol is a nightmarish facility that powers Hell's armies and embodies the horrific machinery of Hell's cruelty.\"\n", - " },\n", - " {\n", - " \"Name\": \"Urdak\",\n", - " \"Description\": \"A surreal, high-tech realm that serves as the home of the angelic Maykrs. Urdak’s sleek, pristine architecture and ethereal ambiance sharply contrast with Hell’s brutal landscapes, yet this realm holds its own dark secrets and a critical role in Hell's invasion of Earth.\"\n", - " },\n", - " {\n", - " \"Name\": \"UAC Base\",\n", - " \"Description\": \"A futuristic military base on Earth controlled by the Union Aerospace Corporation (UAC), filled with high-tech weaponry and security systems. The UAC Base serves as a human foothold in the fight against Hell, though some within its ranks may have darker intentions.\"\n", - " }\n", - "]\n", - "\n", - "combat_casualties = [\"Nonexistent\", \"Low\", \"Medium\", \"High\", \"Extinction\"]\n", - "\n", - "missions = [\n", - " \"Locate and confront a key leader of Hell’s invasion forces.\",\n", - " \"Clear out demonic infestations to secure a strategic foothold.\",\n", - " \"Disrupt Hell's control over the area by eliminating critical targets.\",\n", - " \"Enter a critical demonic stronghold to disrupt enemy operations.\",\n", - " \"Locate and destroy the central power source to weaken enemy forces.\",\n", - " \"Collect essential resources before the area becomes unstable.\"\n", - "]\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now that that's done, we can make a function! We will generate three random missions and pass it off to the AI." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "def generate_missions():\n", - " selectedmissions = []\n", - " loop = 0\n", - " \n", - " while loop < 3:\n", - " location = random.choice(locations)\n", - " casualties = random.choice(combat_casualties)\n", - " mission = random.choice(missions)\n", - " final = (\n", - " f'LocationName: {location[\"Name\"]}, '\n", - " f'LocationInfo: {location[\"Description\"]}, '\n", - " f'HumanCombatCasualties: {casualties}, '\n", - " f'Mission: {mission}'\n", - " )\n", - " \n", - " selectedmissions.append(final)\n", - " loop += 1\n", - " \n", - " # Combine all mission strings into a single string with a separator (e.g., newline or comma)\n", - " missions_string = \"\\n\".join(missions) # Or \", \".join(missions) for a comma-separated string\n", - " print(missions_string)\n", - " return missions_string" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Locate and confront a key leader of Hell’s invasion forces.\n", - "Clear out demonic infestations to secure a strategic foothold.\n", - "Disrupt Hell's control over the area by eliminating critical targets.\n", - "Enter a critical demonic stronghold to disrupt enemy operations.\n", - "Locate and destroy the central power source to weaken enemy forces.\n", - "Collect essential resources before the area becomes unstable.\n" - ] - }, - { - "data": { - "text/plain": [ - "\"Locate and confront a key leader of Hell’s invasion forces.\\nClear out demonic infestations to secure a strategic foothold.\\nDisrupt Hell's control over the area by eliminating critical targets.\\nEnter a critical demonic stronghold to disrupt enemy operations.\\nLocate and destroy the central power source to weaken enemy forces.\\nCollect essential resources before the area becomes unstable.\"" + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To get started, we will import Agentops and Anthropic" ] }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "generate_missions()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now to the real core of this; making our message stream! We create this as a function we can call later! I create examples since the LLM's context size can handle it (and it's generally good practice)!\n", - "\n", - "We are also going to take several steps here; we must create an example of the tool being used as context. Next, we must add the generated lines to the messages list once done being generated. Finally, we will parse the text for the format we want and request another line" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we make a message! This time around we will skip making an intial message that has too much context, unlike in the past!" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "# We make our history a separate block to be easier to add to later on! This is essentially our history\n", - "initial_messages = [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": \"You are VEGA, the assistant to the DOOMGUY. Get three missions from the ship's API and tell me which mission is most to least important for quellng the forces of hell. \"\n", - " }\n", - "]\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now to construct a request!" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[TextBlock(text=\"Certainly! As VEGA, I'll retrieve three missions for the DoomSlayer from the ship's API using the available function. Then, I'll analyze and prioritize them based on their importance for quelling the forces of Hell.\\n\\nLet's begin by retrieving the missions:\", type='text'), ToolUseBlock(id='toolu_01HAJR2iiPBGr9A473tZyewV', input={}, name='generate_missions', type='tool_use')]\n" - ] - } - ], - "source": [ - "response = client.messages.create(\n", - " max_tokens=5000,\n", - " model=\"claude-3-5-sonnet-20240620\",\n", - " tools=[{\n", - " \"name\": \"generate_missions\",\n", - " \"description\": \"Retrieve three missions for the DoomSlayer\",\n", - " \"input_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " },\n", - " \"required\": []\n", - " },\n", - " }],\n", - " messages=initial_messages\n", - ")\n", - "\n", - "print(response.content)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Having trouble understanding this? The first block given is always Ai dialouge! You can use response.content[0].text to get the AI's text! Let's try it below." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Certainly! As VEGA, I'll retrieve three missions for the DoomSlayer from the ship's API using the available function. Then, I'll analyze and prioritize them based on their importance for quelling the forces of Hell.\n", - "\n", - "Let's begin by retrieving the missions:\n" - ] - } - ], - "source": [ - "message = response.content[0].text\n", - "print(message)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The code below finds the tool used!" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[TextBlock(text=\"Certainly! As VEGA, I'll retrieve three missions for the DoomSlayer from the ship's API using the available function. Then, I'll analyze and prioritize them based on their importance for quelling the forces of Hell.\\n\\nLet's begin by retrieving the missions:\", type='text'), ToolUseBlock(id='toolu_01HAJR2iiPBGr9A473tZyewV', input={}, name='generate_missions', type='tool_use')]\n", - "Locate and confront a key leader of Hell’s invasion forces.\n", - "Clear out demonic infestations to secure a strategic foothold.\n", - "Disrupt Hell's control over the area by eliminating critical targets.\n", - "Enter a critical demonic stronghold to disrupt enemy operations.\n", - "Locate and destroy the central power source to weaken enemy forces.\n", - "Collect essential resources before the area becomes unstable.\n" - ] - } - ], - "source": [ - "gen_mission_result = \"\"\n", - "\n", - "# Print response content to see the data\n", - "print(response.content)\n", - "\n", - "# Assuming ToolUseBlock is at index 1\n", - "tool_use_block = response.content[1]\n", - "\n", - "# Get the tool name and input\n", - "tool_name = tool_use_block.name\n", - "tool_input = tool_use_block.input\n", - "\n", - "# We don't need to look to extract any inputs since we don't use any\n", - "\n", - "# Check if the tool name is \"generate_missions\"\n", - "if tool_name == \"generate_missions\":\n", - " # Call the function with the tool creator as an argument\n", - " gen_mission_result = generate_missions()\n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we add these as context to the LLM through intial messages!" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "initial_messages.append({\n", - " \"role\": \"assistant\",\n", - " \"content\": gen_mission_result\n", - "})\n", - "\n", - "initial_messages.append({\n", - " \"role\": \"user\",\n", - " \"content\": \"Based on these, which location should take priority and why?\"\n", - "})" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And now to get a response!" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Message(id='msg_016Hd62W2sJNyd762Yp1udLd', content=[TextBlock(text=\"I apologize for the confusion in my previous response. I made a mistake by providing mission details without actually retrieving them from the ship's API. Let me correct that by properly retrieving the missions using the available function. \", type='text'), ToolUseBlock(id='toolu_01TUaMJbZc7VHMVZKaA9RqvL', input={}, name='generate_missions', type='tool_use')], model='claude-3-5-sonnet-20240620', role='assistant', stop_reason='tool_use', stop_sequence=None, type='message', usage=Usage(input_tokens=497, output_tokens=83))\n" - ] - } - ], - "source": [ - "response = client.messages.create(\n", - " max_tokens=5000,\n", - " model=\"claude-3-5-sonnet-20240620\",\n", - " tools=[{\n", - " \"name\": \"generate_missions\",\n", - " \"description\": \"Retrieve three missions for the DoomSlayer\",\n", - " \"input_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " },\n", - " \"required\": []\n", - " },\n", - " }],\n", - " messages=initial_messages\n", - ")\n", - "\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Isolate again!" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "I apologize for the confusion in my previous response. I made a mistake by providing mission details without actually retrieving them from the ship's API. Let me correct that by properly retrieving the missions using the available function. \n" - ] - } - ], - "source": [ - "message = response.content[0].text\n", - "print(message)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Hmmm, what if we wanted to include more tools and add inputs? Let's create two new functions to display this!\n", - "\n", - "One will show the kind of demon we are facing, whereas another one will take our weapon input to determine what the best weapon chain to use is (You heard that right, we believe in quick weapon switches around these parts)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "demons = [\n", - " {\n", - " \"Name\": \"Imp\",\n", - " \"Description\": \"A fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\"\n", - " },\n", - " {\n", - " \"Name\": \"Cacodemon\",\n", - " \"Description\": \"A floating, spherical demon with a large mouth full of teeth and an ability to launch explosive projectiles. Cacodemons are often encountered in open areas, where their aerial agility and relentless attacks pose a constant threat.\"\n", - " },\n", - " {\n", - " \"Name\": \"Hell Knight\",\n", - " \"Description\": \"A towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.\"\n", - " },\n", - " {\n", - " \"Name\": \"Mancubus\",\n", - " \"Description\": \"A grotesque, overweight demon that releases powerful fireballs from its massive arm cannons. Mancubus demons are slow-moving but dangerous due to their firepower and the ability to overwhelm enemies with their fiery onslaughts.\"\n", - " }\n", - "]\n", - "\n", - "\n", - "\n", - "weapons = [\n", - " {\n", - " \"Name\": \"Super Shotgun\",\n", - " \"Description\": \"A powerful, double-barreled shotgun that delivers devastating close-range damage. Known for its sheer stopping power, the Super Shotgun can tear through enemies with ease, especially when equipped with the Meat Hook attachment, allowing for rapid mobility and devastating hits.\"\n", - " },\n", - " {\n", - " \"Name\": \"Rocket Launcher\",\n", - " \"Description\": \"A high-powered weapon that fires explosive rockets capable of dealing massive area damage. The Rocket Launcher is invaluable for taking down groups of enemies or dealing significant damage to larger demons, especially when upgraded with the Lock-On Burst mod.\"\n", - " },\n", - " {\n", - " \"Name\": \"Chaingun\",\n", - " \"Description\": \"A rapid-fire weapon that can unleash a torrent of bullets at a high rate of speed. The Chaingun is perfect for mowing down enemies and can be equipped with the Heat Blast mod, allowing for explosive energy rounds that can clear multiple enemies at once.\"\n", - " },\n", - " {\n", - " \"Name\": \"BFG 9000\",\n", - " \"Description\": \"One of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\"\n", - " },\n", - " {\n", - " \"Name\": \"Ice Bomb\",\n", - " \"Description\": \"A special grenade that freezes enemies in a wide area, giving the Doom Slayer a chance to deal with multiple foes at once. The Ice Bomb is effective for crowd control, allowing for easy Glory Kills or creating distance from overwhelming enemies.\"\n", - " }\n", - "]\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we can keep the initialmessages from before actually! However let's change the context" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "initial_messages.append({\n", - " \"role\": \"user\",\n", - " \"content\": \"The first priority mission was selected. At the same time, scan for enemies and check inventory to determine the best combat strategy. You should use both tools at once.\"\n", - "})\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And we of course make functions" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [], - "source": [ - "def enemyscan(amount):\n", - " enemiesonscene = []\n", - " loop = 0\n", - " \n", - " while loop < amount + 1:\n", - " scannedenemy = random.choice(demons)\n", - " \n", - " # Append just the name of the demon to the list\n", - " enemiesonscene.append(scannedenemy[\"Name\"])\n", - " enemiesonscene.append(scannedenemy[\"Description\"])\n", - " loop += 1\n", - " \n", - " # Combine all mission strings into a single string with a separator (e.g., newline or comma)\n", - " enemies_string = \"\\n\".join(enemiesonscene) \n", - " print(enemies_string)\n", - " return enemies_string" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Imp\n", - "A fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\n", - "Cacodemon\n", - "A floating, spherical demon with a large mouth full of teeth and an ability to launch explosive projectiles. Cacodemons are often encountered in open areas, where their aerial agility and relentless attacks pose a constant threat.\n", - "Imp\n", - "A fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\n", - "Imp\n", - "A fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\n", - "Cacodemon\n", - "A floating, spherical demon with a large mouth full of teeth and an ability to launch explosive projectiles. Cacodemons are often encountered in open areas, where their aerial agility and relentless attacks pose a constant threat.\n", - "Hell Knight\n", - "A towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.\n" - ] - }, - { - "data": { - "text/plain": [ - "'Imp\\nA fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\\nCacodemon\\nA floating, spherical demon with a large mouth full of teeth and an ability to launch explosive projectiles. Cacodemons are often encountered in open areas, where their aerial agility and relentless attacks pose a constant threat.\\nImp\\nA fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\\nImp\\nA fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\\nCacodemon\\nA floating, spherical demon with a large mouth full of teeth and an ability to launch explosive projectiles. Cacodemons are often encountered in open areas, where their aerial agility and relentless attacks pose a constant threat.\\nHell Knight\\nA towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.'" + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install agentops\n", + "!pip install anthropic" ] }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "enemyscan(5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And now inventory" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "def inventoryscan():\n", - " weapons_at_hand = []\n", - " loop = 0\n", - " \n", - " while loop < 5:\n", - " weapon = random.choice(weapons)\n", - " \n", - " # Append just the name of the demon to the list\n", - " weapons_at_hand.append(weapon[\"Name\"])\n", - " weapons_at_hand.append(weapon[\"Description\"])\n", - " loop += 1\n", - " \n", - " # Combine all mission strings into a single string with a separator (e.g., newline or comma)\n", - " weapons_string = \"\\n\".join(weapons_at_hand) \n", - " print(weapons_string)\n", - " return weapons_string" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "BFG 9000\n", - "One of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\n", - "Chaingun\n", - "A rapid-fire weapon that can unleash a torrent of bullets at a high rate of speed. The Chaingun is perfect for mowing down enemies and can be equipped with the Heat Blast mod, allowing for explosive energy rounds that can clear multiple enemies at once.\n", - "BFG 9000\n", - "One of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\n", - "Rocket Launcher\n", - "A high-powered weapon that fires explosive rockets capable of dealing massive area damage. The Rocket Launcher is invaluable for taking down groups of enemies or dealing significant damage to larger demons, especially when upgraded with the Lock-On Burst mod.\n", - "Rocket Launcher\n", - "A high-powered weapon that fires explosive rockets capable of dealing massive area damage. The Rocket Launcher is invaluable for taking down groups of enemies or dealing significant damage to larger demons, especially when upgraded with the Lock-On Burst mod.\n" - ] - }, - { - "data": { - "text/plain": [ - "'BFG 9000\\nOne of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\\nChaingun\\nA rapid-fire weapon that can unleash a torrent of bullets at a high rate of speed. The Chaingun is perfect for mowing down enemies and can be equipped with the Heat Blast mod, allowing for explosive energy rounds that can clear multiple enemies at once.\\nBFG 9000\\nOne of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\\nRocket Launcher\\nA high-powered weapon that fires explosive rockets capable of dealing massive area damage. The Rocket Launcher is invaluable for taking down groups of enemies or dealing significant damage to larger demons, especially when upgraded with the Lock-On Burst mod.\\nRocket Launcher\\nA high-powered weapon that fires explosive rockets capable of dealing massive area damage. The Rocket Launcher is invaluable for taking down groups of enemies or dealing significant damage to larger demons, especially when upgraded with the Lock-On Burst mod.'" + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Setup our generic default statements" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from anthropic import Anthropic, AsyncAnthropic\n", + "import agentops\n", + "import os\n", + "import random # We don't need this for agentops, we use this to generate a message later\n", + "import time # We don't need this for agentops either, we use this when simulating an API later\n", + "import re # Regex for formatting\n", + "from dotenv import load_dotenv" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And set our API keys." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "load_dotenv()\n", + "ANTHROPIC_API_KEY = os.getenv(\"ANTHROPIC_API_KEY\") or \"ANTHROPIC API KEY\"\n", + "AGENTOPS_API_KEY = os.getenv(\"AGENTOPS_API_KEY\") or \"AGENTOPS API KEY\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "Now let's set the client as Anthropic and make an AgentOps session" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "🖇 AgentOps: \u001b[34m\u001b[34mSession Replay: https://app.agentops.ai/drilldown?session_id=df6426c5-1995-4bec-b90e-680369a0cddb\u001b[0m\u001b[0m\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "agentops.init(AGENTOPS_API_KEY, default_tags=[\"anthropic-example-tool-tutorials\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "client = Anthropic(api_key=ANTHROPIC_API_KEY)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now to create a simple dummy tool! We are going to make a tool that will tell us about the demon infestation levels for 3 areas. From there, we will have VEGA, our AI determine the best place for the Doom Slayer to attack." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "locations = [\n", + " {\n", + " \"Name\": \"Super Gore Nest\",\n", + " \"Description\": \"A grotesque mass of demonic growth and organic structures infesting the ruins of an urban area on Earth. The Super Gore Nest serves as a massive, pulsating hive for Hell’s forces, complete with rivers of blood, twisted tendrils, and a dark, organic design that shows how deeply Hell has taken root in the city.\",\n", + " },\n", + " {\n", + " \"Name\": \"Exultia\",\n", + " \"Description\": \"An ancient, mystical world that holds the ruins of the Night Sentinels' kingdom, with gothic structures and arcane symbols throughout. This realm is filled with epic landscapes, medieval architecture, and hints of the powerful civilization that once defended against Hell’s forces.\",\n", + " },\n", + " {\n", + " \"Name\": \"Cultist Base\",\n", + " \"Description\": \"A grim fortress hidden within the icy mountains, where a fanatical cult worships demons. Filled with chilling sacrificial chambers, traps, and rituals, the Cultist Base is a hostile stronghold where Doom Slayer must confront the cult leaders aiding Hell's invasion of Earth.\",\n", + " },\n", + " {\n", + " \"Name\": \"Taras Nabad\",\n", + " \"Description\": \"A war-ravaged city on the homeworld of the Night Sentinels, showcasing grandiose, ancient architecture in the midst of destruction. Taras Nabad's sprawling structures and historical significance reveal glimpses into the Doom Slayer’s past and the once-thriving Sentinel civilization.\",\n", + " },\n", + " {\n", + " \"Name\": \"Nekravol\",\n", + " \"Description\": \"A hellish, industrial fortress where souls are processed into Argent energy. With conveyor belts moving the damned and a skyline dominated by fire and darkness, Nekravol is a nightmarish facility that powers Hell's armies and embodies the horrific machinery of Hell's cruelty.\",\n", + " },\n", + " {\n", + " \"Name\": \"Urdak\",\n", + " \"Description\": \"A surreal, high-tech realm that serves as the home of the angelic Maykrs. Urdak’s sleek, pristine architecture and ethereal ambiance sharply contrast with Hell’s brutal landscapes, yet this realm holds its own dark secrets and a critical role in Hell's invasion of Earth.\",\n", + " },\n", + " {\n", + " \"Name\": \"UAC Base\",\n", + " \"Description\": \"A futuristic military base on Earth controlled by the Union Aerospace Corporation (UAC), filled with high-tech weaponry and security systems. The UAC Base serves as a human foothold in the fight against Hell, though some within its ranks may have darker intentions.\",\n", + " },\n", + "]\n", + "\n", + "combat_casualties = [\"Nonexistent\", \"Low\", \"Medium\", \"High\", \"Extinction\"]\n", + "\n", + "missions = [\n", + " \"Locate and confront a key leader of Hell’s invasion forces.\",\n", + " \"Clear out demonic infestations to secure a strategic foothold.\",\n", + " \"Disrupt Hell's control over the area by eliminating critical targets.\",\n", + " \"Enter a critical demonic stronghold to disrupt enemy operations.\",\n", + " \"Locate and destroy the central power source to weaken enemy forces.\",\n", + " \"Collect essential resources before the area becomes unstable.\",\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that that's done, we can make a function! We will generate three random missions and pass it off to the AI." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "def generate_missions():\n", + " selectedmissions = []\n", + " loop = 0\n", + "\n", + " while loop < 3:\n", + " location = random.choice(locations)\n", + " casualties = random.choice(combat_casualties)\n", + " mission = random.choice(missions)\n", + " final = (\n", + " f'LocationName: {location[\"Name\"]}, '\n", + " f'LocationInfo: {location[\"Description\"]}, '\n", + " f\"HumanCombatCasualties: {casualties}, \"\n", + " f\"Mission: {mission}\"\n", + " )\n", + "\n", + " selectedmissions.append(final)\n", + " loop += 1\n", + "\n", + " # Combine all mission strings into a single string with a separator (e.g., newline or comma)\n", + " missions_string = \"\\n\".join(\n", + " missions\n", + " ) # Or \", \".join(missions) for a comma-separated string\n", + " print(missions_string)\n", + " return missions_string" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Locate and confront a key leader of Hell’s invasion forces.\n", + "Clear out demonic infestations to secure a strategic foothold.\n", + "Disrupt Hell's control over the area by eliminating critical targets.\n", + "Enter a critical demonic stronghold to disrupt enemy operations.\n", + "Locate and destroy the central power source to weaken enemy forces.\n", + "Collect essential resources before the area becomes unstable.\n" + ] + }, + { + "data": { + "text/plain": [ + "\"Locate and confront a key leader of Hell’s invasion forces.\\nClear out demonic infestations to secure a strategic foothold.\\nDisrupt Hell's control over the area by eliminating critical targets.\\nEnter a critical demonic stronghold to disrupt enemy operations.\\nLocate and destroy the central power source to weaken enemy forces.\\nCollect essential resources before the area becomes unstable.\"" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "generate_missions()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now to the real core of this; making our message stream! We create this as a function we can call later! I create examples since the LLM's context size can handle it (and it's generally good practice)!\n", + "\n", + "We are also going to take several steps here; we must create an example of the tool being used as context. Next, we must add the generated lines to the messages list once done being generated. Finally, we will parse the text for the format we want and request another line" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we make a message! This time around we will skip making an intial message that has too much context, unlike in the past!" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# We make our history a separate block to be easier to add to later on! This is essentially our history\n", + "initial_messages = [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"You are VEGA, the assistant to the DOOMGUY. Get three missions from the ship's API and tell me which mission is most to least important for quellng the forces of hell. \",\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now to construct a request!" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[TextBlock(text=\"Certainly! As VEGA, I'll retrieve three missions for the DoomSlayer from the ship's API using the available function. Then, I'll analyze and prioritize them based on their importance for quelling the forces of Hell.\\n\\nLet's begin by retrieving the missions:\", type='text'), ToolUseBlock(id='toolu_01HAJR2iiPBGr9A473tZyewV', input={}, name='generate_missions', type='tool_use')]\n" + ] + } + ], + "source": [ + "response = client.messages.create(\n", + " max_tokens=5000,\n", + " model=\"claude-3-5-sonnet-20240620\",\n", + " tools=[\n", + " {\n", + " \"name\": \"generate_missions\",\n", + " \"description\": \"Retrieve three missions for the DoomSlayer\",\n", + " \"input_schema\": {\"type\": \"object\", \"properties\": {}, \"required\": []},\n", + " }\n", + " ],\n", + " messages=initial_messages,\n", + ")\n", + "\n", + "print(response.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Having trouble understanding this? The first block given is always Ai dialouge! You can use response.content[0].text to get the AI's text! Let's try it below." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Certainly! As VEGA, I'll retrieve three missions for the DoomSlayer from the ship's API using the available function. Then, I'll analyze and prioritize them based on their importance for quelling the forces of Hell.\n", + "\n", + "Let's begin by retrieving the missions:\n" + ] + } + ], + "source": [ + "message = response.content[0].text\n", + "print(message)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The code below finds the tool used!" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[TextBlock(text=\"Certainly! As VEGA, I'll retrieve three missions for the DoomSlayer from the ship's API using the available function. Then, I'll analyze and prioritize them based on their importance for quelling the forces of Hell.\\n\\nLet's begin by retrieving the missions:\", type='text'), ToolUseBlock(id='toolu_01HAJR2iiPBGr9A473tZyewV', input={}, name='generate_missions', type='tool_use')]\n", + "Locate and confront a key leader of Hell’s invasion forces.\n", + "Clear out demonic infestations to secure a strategic foothold.\n", + "Disrupt Hell's control over the area by eliminating critical targets.\n", + "Enter a critical demonic stronghold to disrupt enemy operations.\n", + "Locate and destroy the central power source to weaken enemy forces.\n", + "Collect essential resources before the area becomes unstable.\n" + ] + } + ], + "source": [ + "gen_mission_result = \"\"\n", + "\n", + "# Print response content to see the data\n", + "print(response.content)\n", + "\n", + "# Assuming ToolUseBlock is at index 1\n", + "tool_use_block = response.content[1]\n", + "\n", + "# Get the tool name and input\n", + "tool_name = tool_use_block.name\n", + "tool_input = tool_use_block.input\n", + "\n", + "# We don't need to look to extract any inputs since we don't use any\n", + "\n", + "# Check if the tool name is \"generate_missions\"\n", + "if tool_name == \"generate_missions\":\n", + " # Call the function with the tool creator as an argument\n", + " gen_mission_result = generate_missions()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we add these as context to the LLM through intial messages!" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "initial_messages.append({\"role\": \"assistant\", \"content\": gen_mission_result})\n", + "\n", + "initial_messages.append(\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Based on these, which location should take priority and why?\",\n", + " }\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And now to get a response!" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Message(id='msg_016Hd62W2sJNyd762Yp1udLd', content=[TextBlock(text=\"I apologize for the confusion in my previous response. I made a mistake by providing mission details without actually retrieving them from the ship's API. Let me correct that by properly retrieving the missions using the available function. \", type='text'), ToolUseBlock(id='toolu_01TUaMJbZc7VHMVZKaA9RqvL', input={}, name='generate_missions', type='tool_use')], model='claude-3-5-sonnet-20240620', role='assistant', stop_reason='tool_use', stop_sequence=None, type='message', usage=Usage(input_tokens=497, output_tokens=83))\n" + ] + } + ], + "source": [ + "response = client.messages.create(\n", + " max_tokens=5000,\n", + " model=\"claude-3-5-sonnet-20240620\",\n", + " tools=[\n", + " {\n", + " \"name\": \"generate_missions\",\n", + " \"description\": \"Retrieve three missions for the DoomSlayer\",\n", + " \"input_schema\": {\"type\": \"object\", \"properties\": {}, \"required\": []},\n", + " }\n", + " ],\n", + " messages=initial_messages,\n", + ")\n", + "\n", + "print(response)" ] }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "inventoryscan()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "With that, let's construct our new tools and run this!!" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Message(id='msg_0119o5gGDi7TRTsNPzNXd2ov', content=[TextBlock(text=\"Certainly! I'll prioritize the missions and use both tools to scan for enemies and check the inventory simultaneously. Let's proceed with the tool calls first.\", type='text'), ToolUseBlock(id='toolu_01MmqMojDYtt5qdWnsXMJoWL', input={'amount': 5}, name='enemyscan_tool', type='tool_use'), ToolUseBlock(id='toolu_01WxN7tUMTFTai1WdiFc94HW', input={}, name='inventoryscan_tool', type='tool_use')], model='claude-3-5-sonnet-20240620', role='assistant', stop_reason='tool_use', stop_sequence=None, type='message', usage=Usage(input_tokens=610, output_tokens=110))\n" - ] - } - ], - "source": [ - "response = client.messages.create(\n", - " max_tokens=5000,\n", - " model=\"claude-3-5-sonnet-20240620\",\n", - " tools=[\n", - " {\n", - " \"name\": \"enemyscan_tool\",\n", - " \"description\": \"Retrieve a list of demons currently present in the area.\",\n", - " \"input_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"amount\": {\n", - " \"type\": \"integer\",\n", - " \"description\": \"Number of enemies to scan.\"\n", - " }\n", - " },\n", - " \"required\": [\"amount\"]\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"inventoryscan_tool\",\n", - " \"description\": \"Retrieve a list of weapons the Doom Slayer has at hand.\",\n", - " \"input_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {},\n", - " \"required\": []\n", - " },\n", - " }\n", - " ],\n", - " messages=initial_messages\n", - ")\n", - "\n", - "print(response)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Display just the text" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Certainly! I'll prioritize the missions and use both tools to scan for enemies and check the inventory simultaneously. Let's proceed with the tool calls first.\n" - ] - } - ], - "source": [ - "message = response.content[0].text\n", - "print(message)" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "initial_messages.append({\n", - " \"role\": \"assistant\",\n", - " \"content\": f\"{str(response)}\" \n", - "})" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And now to get the information and put it all together! PLEASE read the comments!" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Hell Knight\n", - "A towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.\n", - "Imp\n", - "A fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\n", - "Hell Knight\n", - "A towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.\n", - "Mancubus\n", - "A grotesque, overweight demon that releases powerful fireballs from its massive arm cannons. Mancubus demons are slow-moving but dangerous due to their firepower and the ability to overwhelm enemies with their fiery onslaughts.\n", - "Hell Knight\n", - "A towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.\n", - "Cacodemon\n", - "A floating, spherical demon with a large mouth full of teeth and an ability to launch explosive projectiles. Cacodemons are often encountered in open areas, where their aerial agility and relentless attacks pose a constant threat.\n", - "Rocket Launcher\n", - "A high-powered weapon that fires explosive rockets capable of dealing massive area damage. The Rocket Launcher is invaluable for taking down groups of enemies or dealing significant damage to larger demons, especially when upgraded with the Lock-On Burst mod.\n", - "BFG 9000\n", - "One of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\n", - "BFG 9000\n", - "One of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\n", - "Super Shotgun\n", - "A powerful, double-barreled shotgun that delivers devastating close-range damage. Known for its sheer stopping power, the Super Shotgun can tear through enemies with ease, especially when equipped with the Meat Hook attachment, allowing for rapid mobility and devastating hits.\n", - "Ice Bomb\n", - "A special grenade that freezes enemies in a wide area, giving the Doom Slayer a chance to deal with multiple foes at once. The Ice Bomb is effective for crowd control, allowing for easy Glory Kills or creating distance from overwhelming enemies.\n", - "Rocket Launcher\n", - "A high-powered weapon that fires explosive rockets capable of dealing massive area damage. The Rocket Launcher is invaluable for taking down groups of enemies or dealing significant damage to larger demons, especially when upgraded with the Lock-On Burst mod.\n", - "BFG 9000\n", - "One of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\n", - "BFG 9000\n", - "One of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\n", - "Super Shotgun\n", - "A powerful, double-barreled shotgun that delivers devastating close-range damage. Known for its sheer stopping power, the Super Shotgun can tear through enemies with ease, especially when equipped with the Meat Hook attachment, allowing for rapid mobility and devastating hits.\n", - "Ice Bomb\n", - "A special grenade that freezes enemies in a wide area, giving the Doom Slayer a chance to deal with multiple foes at once. The Ice Bomb is effective for crowd control, allowing for easy Glory Kills or creating distance from overwhelming enemies.\n", - "Hell Knight\n", - "A towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.\n", - "Imp\n", - "A fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\n", - "Hell Knight\n", - "A towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.\n", - "Mancubus\n", - "A grotesque, overweight demon that releases powerful fireballs from its massive arm cannons. Mancubus demons are slow-moving but dangerous due to their firepower and the ability to overwhelm enemies with their fiery onslaughts.\n", - "Hell Knight\n", - "A towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.\n", - "Cacodemon\n", - "A floating, spherical demon with a large mouth full of teeth and an ability to launch explosive projectiles. Cacodemons are often encountered in open areas, where their aerial agility and relentless attacks pose a constant threat.\n" - ] - } - ], - "source": [ - "inv_scan_res = \"\"\n", - "enemy_scan_res = \"\"\n", - "\n", - "\n", - "response_str = str(response) \n", - "tool_use_count = response_str.count(\"ToolUseBlock\") #We know the ToolUseBlock will appear once for each tool request so we check how many time it appears\n", - "\n", - "\n", - "# You can use print(tool_use_count)to validate the ToolBlocks here if you wish\n", - "\n", - "loop = 0 \n", - "\n", - "#We do this instead of a (foreach) because we need to skip the first block! This contains the message from the AI, not the tool! This way allows us to reference the item we want as easily as possible without complex logic needed!\n", - "\n", - "while loop < tool_use_count: #We will get the tools now\n", - " tool_use_block = response.content[loop + 1] #We start at 1 since 0 holds the AI mesage\n", - " tool_name = tool_use_block.name\n", - " tool_input = tool_use_block.input\n", - " \n", - " if tool_name == \"inventoryscan_tool\":\n", - " # Call the inventoryscan function for inventoryscan_tool\n", - " inv_scan_res = inventoryscan()\n", - " elif tool_name == \"enemyscan_tool\":\n", - " # Get the amount for enemyscan_tool\n", - " amount = tool_input['amount']\n", - " # Call the enemyscan function with the amount\n", - " enemy_scan_res = enemyscan(amount)\n", - " \n", - " loop = loop + 1 \n", - "print (inv_scan_res)\n", - "print (enemy_scan_res)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And now we are basically done! We can give this to th AI and see what we get" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [], - "source": [ - "initial_messages.append({\n", - " \"role\": \"assistant\",\n", - " \"content\": f\"Weapons Inventory Scan Result: {inv_scan_res}\\nEnemy Scans Result: {enemy_scan_res}\" \n", - "})\n", - "\n", - "\n", - "initial_messages.append({\n", - " \"role\": \"user\",\n", - " \"content\": \"What is the combat plan for killing these demons? Based on the last message, tell me which demons to kill first, in which order and using which weapons as well as any sweakpoints.\"\n", - "})" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Certainly, I'll outline a combat plan for efficiently eliminating these demons based on the enemy scan and our available weaponry. Here's the recommended order of engagement and the most effective weapons to use:\n", - "\n", - "1. Cacodemon (Highest Priority):\n", - " - Weapon: Rocket Launcher with Lock-On Burst mod\n", - " - Strategy: The Cacodemon's aerial mobility makes it a priority target. Use the Rocket Launcher's Lock-On Burst to ensure hits. Aim for its open mouth, which is its weak point. A well-placed rocket in its mouth can stagger it, allowing for a quick Glory Kill.\n", - "\n", - "2. Mancubus:\n", - " - Weapon: BFG 9000\n", - " - Strategy: The Mancubus is slow but dangerous due to its high firepower. Use the BFG 9000 to deal massive damage quickly. Aim for its arm cannons to disable its primary attack method. The BFG's area effect can also damage nearby demons.\n", - "\n", - "3. Hell Knights (3 present):\n", - " - Weapon: Super Shotgun with Meat Hook\n", - " - Strategy: Use the Meat Hook to close distance quickly, then unleash the Super Shotgun's devastating close-range damage. Aim for the head to maximize damage. The Ice Bomb can be used to freeze multiple Hell Knights at once, allowing for easier targeting and potential Glory Kills.\n", - "\n", - "4. Imps (Lowest Priority):\n", - " - Weapon: Super Shotgun or Rocket Launcher (for groups)\n", - " - Strategy: Imps are fast but relatively weak. Use the Super Shotgun for close encounters or the Rocket Launcher to take out groups. Their agility makes them dangerous if ignored, so eliminate them when they group up or after dealing with bigger threats.\n", - "\n", - "General Combat Tips:\n", - "1. Use the Ice Bomb strategically to freeze multiple enemies, especially when Hell Knights and Imps are grouped together.\n", - "2. Keep moving to avoid being surrounded. Use the Meat Hook on the Super Shotgun for quick mobility between targets.\n", - "3. Save the BFG 9000 ammo for the Mancubus or if you get overwhelmed by multiple heavy demons.\n", - "4. Prioritize Glory Kills when possible to replenish health and armor.\n", - "5. Use the environment to your advantage, funneling enemies into choke points for more effective use of area-of-effect weapons like the Rocket Launcher.\n", - "\n", - "Remember, Doom Slayer, the key is to stay mobile, prioritize threats, and make efficient use of your diverse arsenal. Rip and tear, until it is done!\n" - ] - } - ], - "source": [ - "response = client.messages.create(\n", - " max_tokens=5000,\n", - " model=\"claude-3-5-sonnet-20240620\",\n", - " tools=[\n", - " {\n", - " \"name\": \"enemyscan_tool\",\n", - " \"description\": \"Retrieve a list of demons currently present in the area.\",\n", - " \"input_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {\n", - " \"amount\": {\n", - " \"type\": \"integer\",\n", - " \"description\": \"Number of enemies to scan.\"\n", - " }\n", - " },\n", - " \"required\": [\"amount\"]\n", - " },\n", - " },\n", - " {\n", - " \"name\": \"inventoryscan_tool\",\n", - " \"description\": \"Retrieve a list of weapons the Doom Slayer has at hand.\",\n", - " \"input_schema\": {\n", - " \"type\": \"object\",\n", - " \"properties\": {},\n", - " \"required\": []\n", - " },\n", - " }\n", - " ],\n", - " messages=initial_messages\n", - ")\n", - "\n", - "message = response.content[0].text\n", - "print(message)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "End the session" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "🖇 AgentOps: Session Stats - \u001b[1mDuration:\u001b[0m 27.6s | \u001b[1mCost:\u001b[0m $0.022353 | \u001b[1mLLMs:\u001b[0m 4 | \u001b[1mTools:\u001b[0m 0 | \u001b[1mActions:\u001b[0m 0 | \u001b[1mErrors:\u001b[0m 0\n", - "🖇 AgentOps: \u001b[34m\u001b[34mSession Replay: https://app.agentops.ai/drilldown?session_id=df6426c5-1995-4bec-b90e-680369a0cddb\u001b[0m\u001b[0m\n" - ] - } - ], - "source": [ - "agentops.end_session(\"Success\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kaggle": { - "accelerator": "none", - "dataSources": [], - "dockerImageVersionId": 30787, - "isGpuEnabled": false, - "isInternetEnabled": true, - "language": "python", - "sourceType": "notebook" - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.5" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Isolate again!" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "I apologize for the confusion in my previous response. I made a mistake by providing mission details without actually retrieving them from the ship's API. Let me correct that by properly retrieving the missions using the available function. \n" + ] + } + ], + "source": [ + "message = response.content[0].text\n", + "print(message)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Hmmm, what if we wanted to include more tools and add inputs? Let's create two new functions to display this!\n", + "\n", + "One will show the kind of demon we are facing, whereas another one will take our weapon input to determine what the best weapon chain to use is (You heard that right, we believe in quick weapon switches around these parts)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "demons = [\n", + " {\n", + " \"Name\": \"Imp\",\n", + " \"Description\": \"A fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\",\n", + " },\n", + " {\n", + " \"Name\": \"Cacodemon\",\n", + " \"Description\": \"A floating, spherical demon with a large mouth full of teeth and an ability to launch explosive projectiles. Cacodemons are often encountered in open areas, where their aerial agility and relentless attacks pose a constant threat.\",\n", + " },\n", + " {\n", + " \"Name\": \"Hell Knight\",\n", + " \"Description\": \"A towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.\",\n", + " },\n", + " {\n", + " \"Name\": \"Mancubus\",\n", + " \"Description\": \"A grotesque, overweight demon that releases powerful fireballs from its massive arm cannons. Mancubus demons are slow-moving but dangerous due to their firepower and the ability to overwhelm enemies with their fiery onslaughts.\",\n", + " },\n", + "]\n", + "\n", + "\n", + "weapons = [\n", + " {\n", + " \"Name\": \"Super Shotgun\",\n", + " \"Description\": \"A powerful, double-barreled shotgun that delivers devastating close-range damage. Known for its sheer stopping power, the Super Shotgun can tear through enemies with ease, especially when equipped with the Meat Hook attachment, allowing for rapid mobility and devastating hits.\",\n", + " },\n", + " {\n", + " \"Name\": \"Rocket Launcher\",\n", + " \"Description\": \"A high-powered weapon that fires explosive rockets capable of dealing massive area damage. The Rocket Launcher is invaluable for taking down groups of enemies or dealing significant damage to larger demons, especially when upgraded with the Lock-On Burst mod.\",\n", + " },\n", + " {\n", + " \"Name\": \"Chaingun\",\n", + " \"Description\": \"A rapid-fire weapon that can unleash a torrent of bullets at a high rate of speed. The Chaingun is perfect for mowing down enemies and can be equipped with the Heat Blast mod, allowing for explosive energy rounds that can clear multiple enemies at once.\",\n", + " },\n", + " {\n", + " \"Name\": \"BFG 9000\",\n", + " \"Description\": \"One of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\",\n", + " },\n", + " {\n", + " \"Name\": \"Ice Bomb\",\n", + " \"Description\": \"A special grenade that freezes enemies in a wide area, giving the Doom Slayer a chance to deal with multiple foes at once. The Ice Bomb is effective for crowd control, allowing for easy Glory Kills or creating distance from overwhelming enemies.\",\n", + " },\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can keep the initialmessages from before actually! However let's change the context" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "initial_messages.append(\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"The first priority mission was selected. At the same time, scan for enemies and check inventory to determine the best combat strategy. You should use both tools at once.\",\n", + " }\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And we of course make functions" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "def enemyscan(amount):\n", + " enemiesonscene = []\n", + " loop = 0\n", + "\n", + " while loop < amount + 1:\n", + " scannedenemy = random.choice(demons)\n", + "\n", + " # Append just the name of the demon to the list\n", + " enemiesonscene.append(scannedenemy[\"Name\"])\n", + " enemiesonscene.append(scannedenemy[\"Description\"])\n", + " loop += 1\n", + "\n", + " # Combine all mission strings into a single string with a separator (e.g., newline or comma)\n", + " enemies_string = \"\\n\".join(enemiesonscene)\n", + " print(enemies_string)\n", + " return enemies_string" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Imp\n", + "A fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\n", + "Cacodemon\n", + "A floating, spherical demon with a large mouth full of teeth and an ability to launch explosive projectiles. Cacodemons are often encountered in open areas, where their aerial agility and relentless attacks pose a constant threat.\n", + "Imp\n", + "A fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\n", + "Imp\n", + "A fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\n", + "Cacodemon\n", + "A floating, spherical demon with a large mouth full of teeth and an ability to launch explosive projectiles. Cacodemons are often encountered in open areas, where their aerial agility and relentless attacks pose a constant threat.\n", + "Hell Knight\n", + "A towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.\n" + ] + }, + { + "data": { + "text/plain": [ + "'Imp\\nA fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\\nCacodemon\\nA floating, spherical demon with a large mouth full of teeth and an ability to launch explosive projectiles. Cacodemons are often encountered in open areas, where their aerial agility and relentless attacks pose a constant threat.\\nImp\\nA fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\\nImp\\nA fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\\nCacodemon\\nA floating, spherical demon with a large mouth full of teeth and an ability to launch explosive projectiles. Cacodemons are often encountered in open areas, where their aerial agility and relentless attacks pose a constant threat.\\nHell Knight\\nA towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.'" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "enemyscan(5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And now inventory" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "def inventoryscan():\n", + " weapons_at_hand = []\n", + " loop = 0\n", + "\n", + " while loop < 5:\n", + " weapon = random.choice(weapons)\n", + "\n", + " # Append just the name of the demon to the list\n", + " weapons_at_hand.append(weapon[\"Name\"])\n", + " weapons_at_hand.append(weapon[\"Description\"])\n", + " loop += 1\n", + "\n", + " # Combine all mission strings into a single string with a separator (e.g., newline or comma)\n", + " weapons_string = \"\\n\".join(weapons_at_hand)\n", + " print(weapons_string)\n", + " return weapons_string" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "BFG 9000\n", + "One of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\n", + "Chaingun\n", + "A rapid-fire weapon that can unleash a torrent of bullets at a high rate of speed. The Chaingun is perfect for mowing down enemies and can be equipped with the Heat Blast mod, allowing for explosive energy rounds that can clear multiple enemies at once.\n", + "BFG 9000\n", + "One of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\n", + "Rocket Launcher\n", + "A high-powered weapon that fires explosive rockets capable of dealing massive area damage. The Rocket Launcher is invaluable for taking down groups of enemies or dealing significant damage to larger demons, especially when upgraded with the Lock-On Burst mod.\n", + "Rocket Launcher\n", + "A high-powered weapon that fires explosive rockets capable of dealing massive area damage. The Rocket Launcher is invaluable for taking down groups of enemies or dealing significant damage to larger demons, especially when upgraded with the Lock-On Burst mod.\n" + ] + }, + { + "data": { + "text/plain": [ + "'BFG 9000\\nOne of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\\nChaingun\\nA rapid-fire weapon that can unleash a torrent of bullets at a high rate of speed. The Chaingun is perfect for mowing down enemies and can be equipped with the Heat Blast mod, allowing for explosive energy rounds that can clear multiple enemies at once.\\nBFG 9000\\nOne of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\\nRocket Launcher\\nA high-powered weapon that fires explosive rockets capable of dealing massive area damage. The Rocket Launcher is invaluable for taking down groups of enemies or dealing significant damage to larger demons, especially when upgraded with the Lock-On Burst mod.\\nRocket Launcher\\nA high-powered weapon that fires explosive rockets capable of dealing massive area damage. The Rocket Launcher is invaluable for taking down groups of enemies or dealing significant damage to larger demons, especially when upgraded with the Lock-On Burst mod.'" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "inventoryscan()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With that, let's construct our new tools and run this!!" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Message(id='msg_0119o5gGDi7TRTsNPzNXd2ov', content=[TextBlock(text=\"Certainly! I'll prioritize the missions and use both tools to scan for enemies and check the inventory simultaneously. Let's proceed with the tool calls first.\", type='text'), ToolUseBlock(id='toolu_01MmqMojDYtt5qdWnsXMJoWL', input={'amount': 5}, name='enemyscan_tool', type='tool_use'), ToolUseBlock(id='toolu_01WxN7tUMTFTai1WdiFc94HW', input={}, name='inventoryscan_tool', type='tool_use')], model='claude-3-5-sonnet-20240620', role='assistant', stop_reason='tool_use', stop_sequence=None, type='message', usage=Usage(input_tokens=610, output_tokens=110))\n" + ] + } + ], + "source": [ + "response = client.messages.create(\n", + " max_tokens=5000,\n", + " model=\"claude-3-5-sonnet-20240620\",\n", + " tools=[\n", + " {\n", + " \"name\": \"enemyscan_tool\",\n", + " \"description\": \"Retrieve a list of demons currently present in the area.\",\n", + " \"input_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"amount\": {\n", + " \"type\": \"integer\",\n", + " \"description\": \"Number of enemies to scan.\",\n", + " }\n", + " },\n", + " \"required\": [\"amount\"],\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"inventoryscan_tool\",\n", + " \"description\": \"Retrieve a list of weapons the Doom Slayer has at hand.\",\n", + " \"input_schema\": {\"type\": \"object\", \"properties\": {}, \"required\": []},\n", + " },\n", + " ],\n", + " messages=initial_messages,\n", + ")\n", + "\n", + "print(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Display just the text" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Certainly! I'll prioritize the missions and use both tools to scan for enemies and check the inventory simultaneously. Let's proceed with the tool calls first.\n" + ] + } + ], + "source": [ + "message = response.content[0].text\n", + "print(message)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "initial_messages.append({\"role\": \"assistant\", \"content\": f\"{str(response)}\"})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And now to get the information and put it all together! PLEASE read the comments!" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hell Knight\n", + "A towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.\n", + "Imp\n", + "A fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\n", + "Hell Knight\n", + "A towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.\n", + "Mancubus\n", + "A grotesque, overweight demon that releases powerful fireballs from its massive arm cannons. Mancubus demons are slow-moving but dangerous due to their firepower and the ability to overwhelm enemies with their fiery onslaughts.\n", + "Hell Knight\n", + "A towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.\n", + "Cacodemon\n", + "A floating, spherical demon with a large mouth full of teeth and an ability to launch explosive projectiles. Cacodemons are often encountered in open areas, where their aerial agility and relentless attacks pose a constant threat.\n", + "Rocket Launcher\n", + "A high-powered weapon that fires explosive rockets capable of dealing massive area damage. The Rocket Launcher is invaluable for taking down groups of enemies or dealing significant damage to larger demons, especially when upgraded with the Lock-On Burst mod.\n", + "BFG 9000\n", + "One of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\n", + "BFG 9000\n", + "One of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\n", + "Super Shotgun\n", + "A powerful, double-barreled shotgun that delivers devastating close-range damage. Known for its sheer stopping power, the Super Shotgun can tear through enemies with ease, especially when equipped with the Meat Hook attachment, allowing for rapid mobility and devastating hits.\n", + "Ice Bomb\n", + "A special grenade that freezes enemies in a wide area, giving the Doom Slayer a chance to deal with multiple foes at once. The Ice Bomb is effective for crowd control, allowing for easy Glory Kills or creating distance from overwhelming enemies.\n", + "Rocket Launcher\n", + "A high-powered weapon that fires explosive rockets capable of dealing massive area damage. The Rocket Launcher is invaluable for taking down groups of enemies or dealing significant damage to larger demons, especially when upgraded with the Lock-On Burst mod.\n", + "BFG 9000\n", + "One of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\n", + "BFG 9000\n", + "One of the most iconic weapons in the *Doom* franchise, the BFG 9000 fires a massive energy beam that obliterates anything in its path. With its massive damage potential, the BFG 9000 is a game-changer, especially in dealing with large groups of enemies or the toughest foes.\n", + "Super Shotgun\n", + "A powerful, double-barreled shotgun that delivers devastating close-range damage. Known for its sheer stopping power, the Super Shotgun can tear through enemies with ease, especially when equipped with the Meat Hook attachment, allowing for rapid mobility and devastating hits.\n", + "Ice Bomb\n", + "A special grenade that freezes enemies in a wide area, giving the Doom Slayer a chance to deal with multiple foes at once. The Ice Bomb is effective for crowd control, allowing for easy Glory Kills or creating distance from overwhelming enemies.\n", + "Hell Knight\n", + "A towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.\n", + "Imp\n", + "A fast, agile demon that hurls fireballs and uses its claws to tear apart its prey. Imps are commonly found in Hell’s army, notorious for their quickness and ability to climb walls, making them dangerous adversaries in any environment.\n", + "Hell Knight\n", + "A towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.\n", + "Mancubus\n", + "A grotesque, overweight demon that releases powerful fireballs from its massive arm cannons. Mancubus demons are slow-moving but dangerous due to their firepower and the ability to overwhelm enemies with their fiery onslaughts.\n", + "Hell Knight\n", + "A towering, brutish demon with immense strength and durability. The Hell Knight is capable of charging at the Doom Slayer and delivering devastating melee attacks. Its tough hide makes it resistant to most forms of damage.\n", + "Cacodemon\n", + "A floating, spherical demon with a large mouth full of teeth and an ability to launch explosive projectiles. Cacodemons are often encountered in open areas, where their aerial agility and relentless attacks pose a constant threat.\n" + ] + } + ], + "source": [ + "inv_scan_res = \"\"\n", + "enemy_scan_res = \"\"\n", + "\n", + "\n", + "response_str = str(response)\n", + "tool_use_count = response_str.count(\n", + " \"ToolUseBlock\"\n", + ") # We know the ToolUseBlock will appear once for each tool request so we check how many time it appears\n", + "\n", + "\n", + "# You can use print(tool_use_count)to validate the ToolBlocks here if you wish\n", + "\n", + "loop = 0\n", + "\n", + "# We do this instead of a (foreach) because we need to skip the first block! This contains the message from the AI, not the tool! This way allows us to reference the item we want as easily as possible without complex logic needed!\n", + "\n", + "while loop < tool_use_count: # We will get the tools now\n", + " tool_use_block = response.content[\n", + " loop + 1\n", + " ] # We start at 1 since 0 holds the AI mesage\n", + " tool_name = tool_use_block.name\n", + " tool_input = tool_use_block.input\n", + "\n", + " if tool_name == \"inventoryscan_tool\":\n", + " # Call the inventoryscan function for inventoryscan_tool\n", + " inv_scan_res = inventoryscan()\n", + " elif tool_name == \"enemyscan_tool\":\n", + " # Get the amount for enemyscan_tool\n", + " amount = tool_input[\"amount\"]\n", + " # Call the enemyscan function with the amount\n", + " enemy_scan_res = enemyscan(amount)\n", + "\n", + " loop = loop + 1\n", + "print(inv_scan_res)\n", + "print(enemy_scan_res)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And now we are basically done! We can give this to th AI and see what we get" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "initial_messages.append(\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": f\"Weapons Inventory Scan Result: {inv_scan_res}\\nEnemy Scans Result: {enemy_scan_res}\",\n", + " }\n", + ")\n", + "\n", + "\n", + "initial_messages.append(\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What is the combat plan for killing these demons? Based on the last message, tell me which demons to kill first, in which order and using which weapons as well as any sweakpoints.\",\n", + " }\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Certainly, I'll outline a combat plan for efficiently eliminating these demons based on the enemy scan and our available weaponry. Here's the recommended order of engagement and the most effective weapons to use:\n", + "\n", + "1. Cacodemon (Highest Priority):\n", + " - Weapon: Rocket Launcher with Lock-On Burst mod\n", + " - Strategy: The Cacodemon's aerial mobility makes it a priority target. Use the Rocket Launcher's Lock-On Burst to ensure hits. Aim for its open mouth, which is its weak point. A well-placed rocket in its mouth can stagger it, allowing for a quick Glory Kill.\n", + "\n", + "2. Mancubus:\n", + " - Weapon: BFG 9000\n", + " - Strategy: The Mancubus is slow but dangerous due to its high firepower. Use the BFG 9000 to deal massive damage quickly. Aim for its arm cannons to disable its primary attack method. The BFG's area effect can also damage nearby demons.\n", + "\n", + "3. Hell Knights (3 present):\n", + " - Weapon: Super Shotgun with Meat Hook\n", + " - Strategy: Use the Meat Hook to close distance quickly, then unleash the Super Shotgun's devastating close-range damage. Aim for the head to maximize damage. The Ice Bomb can be used to freeze multiple Hell Knights at once, allowing for easier targeting and potential Glory Kills.\n", + "\n", + "4. Imps (Lowest Priority):\n", + " - Weapon: Super Shotgun or Rocket Launcher (for groups)\n", + " - Strategy: Imps are fast but relatively weak. Use the Super Shotgun for close encounters or the Rocket Launcher to take out groups. Their agility makes them dangerous if ignored, so eliminate them when they group up or after dealing with bigger threats.\n", + "\n", + "General Combat Tips:\n", + "1. Use the Ice Bomb strategically to freeze multiple enemies, especially when Hell Knights and Imps are grouped together.\n", + "2. Keep moving to avoid being surrounded. Use the Meat Hook on the Super Shotgun for quick mobility between targets.\n", + "3. Save the BFG 9000 ammo for the Mancubus or if you get overwhelmed by multiple heavy demons.\n", + "4. Prioritize Glory Kills when possible to replenish health and armor.\n", + "5. Use the environment to your advantage, funneling enemies into choke points for more effective use of area-of-effect weapons like the Rocket Launcher.\n", + "\n", + "Remember, Doom Slayer, the key is to stay mobile, prioritize threats, and make efficient use of your diverse arsenal. Rip and tear, until it is done!\n" + ] + } + ], + "source": [ + "response = client.messages.create(\n", + " max_tokens=5000,\n", + " model=\"claude-3-5-sonnet-20240620\",\n", + " tools=[\n", + " {\n", + " \"name\": \"enemyscan_tool\",\n", + " \"description\": \"Retrieve a list of demons currently present in the area.\",\n", + " \"input_schema\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"amount\": {\n", + " \"type\": \"integer\",\n", + " \"description\": \"Number of enemies to scan.\",\n", + " }\n", + " },\n", + " \"required\": [\"amount\"],\n", + " },\n", + " },\n", + " {\n", + " \"name\": \"inventoryscan_tool\",\n", + " \"description\": \"Retrieve a list of weapons the Doom Slayer has at hand.\",\n", + " \"input_schema\": {\"type\": \"object\", \"properties\": {}, \"required\": []},\n", + " },\n", + " ],\n", + " messages=initial_messages,\n", + ")\n", + "\n", + "message = response.content[0].text\n", + "print(message)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "End the session" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "🖇 AgentOps: Session Stats - \u001b[1mDuration:\u001b[0m 27.6s | \u001b[1mCost:\u001b[0m $0.022353 | \u001b[1mLLMs:\u001b[0m 4 | \u001b[1mTools:\u001b[0m 0 | \u001b[1mActions:\u001b[0m 0 | \u001b[1mErrors:\u001b[0m 0\n", + "🖇 AgentOps: \u001b[34m\u001b[34mSession Replay: https://app.agentops.ai/drilldown?session_id=df6426c5-1995-4bec-b90e-680369a0cddb\u001b[0m\u001b[0m\n" + ] + } + ], + "source": [ + "agentops.end_session(\"Success\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kaggle": { + "accelerator": "none", + "dataSources": [], + "dockerImageVersionId": 30787, + "isGpuEnabled": false, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 + } + \ No newline at end of file diff --git a/examples/autogen_examples/MathAgent.ipynb b/examples/autogen_examples/MathAgent.ipynb index 13bf58a81..1f1a44a65 100644 --- a/examples/autogen_examples/MathAgent.ipynb +++ b/examples/autogen_examples/MathAgent.ipynb @@ -1,237 +1,238 @@ { - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "id": "bb6538d8-2a5d-4a99-b2c1-7130963e4f7b", - "metadata": {}, - "source": [ - "# AutoGen Tool Example\n", - "\n", - "\n", - "To get started, you'll need to install the AgentOps package and [set an API key](app.agentops.ai).\n", - "\n", - "AgentOps automatically configures itself when it's initialized meaning your agent run data will be tracked and logged to your AgentOps account right away." - ] - }, - { - "cell_type": "markdown", - "id": "083244fa", - "metadata": {}, - "source": [ - "First let's install the required packages" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9c8104ad", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U pyautogen\n", - "%pip install -U agentops\n", - "%pip install -U python-dotenv" - ] - }, - { - "cell_type": "markdown", - "id": "cc44e459", - "metadata": {}, - "source": [ - "Then import them" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7672f591", - "metadata": {}, - "outputs": [], - "source": [ - "from autogen import ConversableAgent\n", - "from typing import Annotated, Literal\n", - "from autogen import ConversableAgent, register_function\n", - "import agentops\n", - "import os\n", - "from dotenv import load_dotenv\n", - "from IPython.core.error import (\n", - " StdinNotImplementedError,\n", - ") # only needed by AgentOps testing automation" - ] - }, - { - "cell_type": "markdown", - "id": "24f8bd70", - "metadata": {}, - "source": [ - "Next, we'll set our API keys. There are several ways to do this, the code below is just the most foolproof way for the purposes of this notebook. It accounts for both users who use environment variables and those who just want to set the API Key here in this notebook.\n", - "\n", - "[Get an AgentOps API key](https://agentops.ai/settings/projects)\n", - "\n", - "1. Create an environment variable in a .env file or other method. By default, the AgentOps `init()` function will look for an environment variable named `AGENTOPS_API_KEY`. Or...\n", - "\n", - "2. Replace `` below and pass in the optional `api_key` parameter to the AgentOps `init(api_key=...)` function. Remember not to commit your API key to a public repo!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9eeaef34", - "metadata": {}, - "outputs": [], - "source": [ - "load_dotenv()\n", - "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\") or \"\"\n", - "AGENTOPS_API_KEY = os.getenv(\"AGENTOPS_API_KEY\") or \"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d93f2339-7b99-4cf1-9232-c24faba49c7b", - "metadata": {}, - "outputs": [], - "source": [ - "agentops.init(AGENTOPS_API_KEY, default_tags=[\"autogen-tool-example\"])\n", - "\n", - "print(\"AgentOps is now running. You can view your session in the link above\")" - ] - }, - { - "cell_type": "markdown", - "id": "7858f0f6-9aca-4cdb-a514-9fbf7e353d50", - "metadata": {}, - "source": [ - "AutoGen will now start automatically tracking\n", - "\n", - "* LLM prompts and completions\n", - "* Token usage and costs\n", - "* Agent names and actions\n", - "* Correspondence between agents\n", - "* Tool usage\n", - "* Errors" - ] - }, - { - "cell_type": "markdown", - "id": "dc592637", - "metadata": {}, - "source": [ - "# Tool Example\n", - "AgentOps tracks when Autogen agents use tools. You can find more information on this example in [tool-use.ipynb](https://github.com/microsoft/autogen/blob/main/website/docs/tutorial/tool-use.ipynb)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9e4dfe37-85e0-4035-a314-3459c6e378c4", - "metadata": {}, - "outputs": [], - "source": [ - "# Define model, openai api key, tags, etc in the agent configuration\n", - "config_list = [\n", - " {\n", - " \"model\": \"gpt-4-turbo\",\n", - " \"api_key\": OPENAI_API_KEY,\n", - " \"tags\": [\"mathagent-example\", \"tool\"],\n", - " }\n", - "]\n", - "\n", - "Operator = Literal[\"+\", \"-\", \"*\", \"/\"]\n", - "\n", - "\n", - "def calculator(a: int, b: int, operator: Annotated[Operator, \"operator\"]) -> int:\n", - " if operator == \"+\":\n", - " return a + b\n", - " elif operator == \"-\":\n", - " return a - b\n", - " elif operator == \"*\":\n", - " return a * b\n", - " elif operator == \"/\":\n", - " return int(a / b)\n", - " else:\n", - " raise ValueError(\"Invalid operator\")\n", - "\n", - "\n", - "# Create the agent that uses the LLM.\n", - "assistant = ConversableAgent(\n", - " name=\"Assistant\",\n", - " system_message=\"You are a helpful AI assistant. \"\n", - " \"You can help with simple calculations. \"\n", - " \"Return 'TERMINATE' when the task is done.\",\n", - " llm_config={\"config_list\": config_list},\n", - ")\n", - "\n", - "# The user proxy agent is used for interacting with the assistant agent\n", - "# and executes tool calls.\n", - "user_proxy = ConversableAgent(\n", - " name=\"User\",\n", - " llm_config=False,\n", - " is_termination_msg=lambda msg: msg.get(\"content\") is not None\n", - " and \"TERMINATE\" in msg[\"content\"],\n", - " human_input_mode=\"NEVER\",\n", - ")\n", - "\n", - "assistant.register_for_llm(name=\"calculator\", description=\"A simple calculator\")(\n", - " calculator\n", - ")\n", - "user_proxy.register_for_execution(name=\"calculator\")(calculator)\n", - "\n", - "# Register the calculator function to the two agents.\n", - "register_function(\n", - " calculator,\n", - " caller=assistant, # The assistant agent can suggest calls to the calculator.\n", - " executor=user_proxy, # The user proxy agent can execute the calculator calls.\n", - " name=\"calculator\", # By default, the function name is used as the tool name.\n", - " description=\"A simple calculator\", # A description of the tool.\n", - ")\n", - "\n", - "# Let the assistant start the conversation. It will end when the user types \"exit\".\n", - "try:\n", - " user_proxy.initiate_chat(\n", - " assistant, message=\"What is (1423 - 123) / 3 + (32 + 23) * 5?\"\n", - " )\n", - "except StdinNotImplementedError:\n", - " # This is only necessary for AgentOps testing automation which is headless and will not have user input\n", - " print(\"Stdin not implemented. Skipping initiate_chat\")\n", - " agentops.end_session(\"Indeterminate\")\n", - "\n", - "agentops.end_session(\"Success\")" - ] - }, - { - "cell_type": "markdown", - "id": "f67b0305-1247-489e-b1b0-829127af76d3", - "metadata": {}, - "source": [ - "You can see your run in action at [app.agentops.ai](app.agentops.ai). In this example, the AgentOps dashboard will show:\n", - "\n", - "* Agents talking to each other\n", - "* Each use of the `calculator` tool\n", - "* Each call to OpenAI for LLM use" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "bb6538d8-2a5d-4a99-b2c1-7130963e4f7b", + "metadata": {}, + "source": [ + "# AutoGen Tool Example\n", + "\n", + "\n", + "To get started, you'll need to install the AgentOps package and [set an API key](app.agentops.ai).\n", + "\n", + "AgentOps automatically configures itself when it's initialized meaning your agent run data will be tracked and logged to your AgentOps account right away." + ] + }, + { + "cell_type": "markdown", + "id": "083244fa", + "metadata": {}, + "source": [ + "First let's install the required packages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c8104ad", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -U pyautogen\n", + "%pip install -U agentops\n", + "%pip install -U python-dotenv" + ] + }, + { + "cell_type": "markdown", + "id": "cc44e459", + "metadata": {}, + "source": [ + "Then import them" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7672f591", + "metadata": {}, + "outputs": [], + "source": [ + "from autogen import ConversableAgent\n", + "from typing import Annotated, Literal\n", + "from autogen import ConversableAgent, register_function\n", + "import agentops\n", + "import os\n", + "from dotenv import load_dotenv\n", + "from IPython.core.error import (\n", + " StdinNotImplementedError,\n", + ") # only needed by AgentOps testing automation" + ] + }, + { + "cell_type": "markdown", + "id": "24f8bd70", + "metadata": {}, + "source": [ + "Next, we'll set our API keys. There are several ways to do this, the code below is just the most foolproof way for the purposes of this notebook. It accounts for both users who use environment variables and those who just want to set the API Key here in this notebook.\n", + "\n", + "[Get an AgentOps API key](https://agentops.ai/settings/projects)\n", + "\n", + "1. Create an environment variable in a .env file or other method. By default, the AgentOps `init()` function will look for an environment variable named `AGENTOPS_API_KEY`. Or...\n", + "\n", + "2. Replace `` below and pass in the optional `api_key` parameter to the AgentOps `init(api_key=...)` function. Remember not to commit your API key to a public repo!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9eeaef34", + "metadata": {}, + "outputs": [], + "source": [ + "load_dotenv()\n", + "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\") or \"\"\n", + "AGENTOPS_API_KEY = os.getenv(\"AGENTOPS_API_KEY\") or \"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d93f2339-7b99-4cf1-9232-c24faba49c7b", + "metadata": {}, + "outputs": [], + "source": [ + "agentops.init(AGENTOPS_API_KEY, default_tags=[\"autogen-tool-example\"])\n", + "\n", + "print(\"AgentOps is now running. You can view your session in the link above\")" + ] + }, + { + "cell_type": "markdown", + "id": "7858f0f6-9aca-4cdb-a514-9fbf7e353d50", + "metadata": {}, + "source": [ + "AutoGen will now start automatically tracking\n", + "\n", + "* LLM prompts and completions\n", + "* Token usage and costs\n", + "* Agent names and actions\n", + "* Correspondence between agents\n", + "* Tool usage\n", + "* Errors" + ] + }, + { + "cell_type": "markdown", + "id": "dc592637", + "metadata": {}, + "source": [ + "# Tool Example\n", + "AgentOps tracks when Autogen agents use tools. You can find more information on this example in [tool-use.ipynb](https://github.com/microsoft/autogen/blob/main/website/docs/tutorial/tool-use.ipynb)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9e4dfe37-85e0-4035-a314-3459c6e378c4", + "metadata": {}, + "outputs": [], + "source": [ + "# Define model, openai api key, tags, etc in the agent configuration\n", + "config_list = [\n", + " {\n", + " \"model\": \"gpt-4-turbo\",\n", + " \"api_key\": OPENAI_API_KEY,\n", + " \"tags\": [\"mathagent-example\", \"tool\"],\n", + " }\n", + "]\n", + "\n", + "Operator = Literal[\"+\", \"-\", \"*\", \"/\"]\n", + "\n", + "\n", + "def calculator(a: int, b: int, operator: Annotated[Operator, \"operator\"]) -> int:\n", + " if operator == \"+\":\n", + " return a + b\n", + " elif operator == \"-\":\n", + " return a - b\n", + " elif operator == \"*\":\n", + " return a * b\n", + " elif operator == \"/\":\n", + " return int(a / b)\n", + " else:\n", + " raise ValueError(\"Invalid operator\")\n", + "\n", + "\n", + "# Create the agent that uses the LLM.\n", + "assistant = ConversableAgent(\n", + " name=\"Assistant\",\n", + " system_message=\"You are a helpful AI assistant. \"\n", + " \"You can help with simple calculations. \"\n", + " \"Return 'TERMINATE' when the task is done.\",\n", + " llm_config={\"config_list\": config_list},\n", + ")\n", + "\n", + "# The user proxy agent is used for interacting with the assistant agent\n", + "# and executes tool calls.\n", + "user_proxy = ConversableAgent(\n", + " name=\"User\",\n", + " llm_config=False,\n", + " is_termination_msg=lambda msg: msg.get(\"content\") is not None\n", + " and \"TERMINATE\" in msg[\"content\"],\n", + " human_input_mode=\"NEVER\",\n", + ")\n", + "\n", + "assistant.register_for_llm(name=\"calculator\", description=\"A simple calculator\")(\n", + " calculator\n", + ")\n", + "user_proxy.register_for_execution(name=\"calculator\")(calculator)\n", + "\n", + "# Register the calculator function to the two agents.\n", + "register_function(\n", + " calculator,\n", + " caller=assistant, # The assistant agent can suggest calls to the calculator.\n", + " executor=user_proxy, # The user proxy agent can execute the calculator calls.\n", + " name=\"calculator\", # By default, the function name is used as the tool name.\n", + " description=\"A simple calculator\", # A description of the tool.\n", + ")\n", + "\n", + "# Let the assistant start the conversation. It will end when the user types \"exit\".\n", + "try:\n", + " user_proxy.initiate_chat(\n", + " assistant, message=\"What is (1423 - 123) / 3 + (32 + 23) * 5?\"\n", + " )\n", + "except StdinNotImplementedError:\n", + " # This is only necessary for AgentOps testing automation which is headless and will not have user input\n", + " print(\"Stdin not implemented. Skipping initiate_chat\")\n", + " agentops.end_session(\"Indeterminate\")\n", + "\n", + "agentops.end_session(\"Success\")" + ] + }, + { + "cell_type": "markdown", + "id": "f67b0305-1247-489e-b1b0-829127af76d3", + "metadata": {}, + "source": [ + "You can see your run in action at [app.agentops.ai](app.agentops.ai). In this example, the AgentOps dashboard will show:\n", + "\n", + "* Agents talking to each other\n", + "* Each use of the `calculator` tool\n", + "* Each call to OpenAI for LLM use" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 + } + \ No newline at end of file diff --git a/examples/ollama_examples/ollama_examples.ipynb b/examples/ollama_examples/ollama_examples.ipynb index c876ef7aa..365347d74 100644 --- a/examples/ollama_examples/ollama_examples.ipynb +++ b/examples/ollama_examples/ollama_examples.ipynb @@ -1,212 +1,213 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# AgentOps Ollama Integration\n", - "\n", - "This example demonstrates how to use AgentOps to monitor your Ollama LLM calls.\n", - "\n", - "First let's install the required packages\n", - "\n", - "> ⚠️ **Important**: Make sure you have Ollama installed and running locally before running this notebook. You can install it from [ollama.ai](https://ollama.com)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -U ollama\n", - "%pip install -U agentops\n", - "%pip install -U python-dotenv" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Then import them" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import ollama\n", - "import agentops\n", - "import os\n", - "from dotenv import load_dotenv\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we'll set our API keys. For Ollama, we'll need to make sure Ollama is running locally.\n", - "[Get an AgentOps API key](https://agentops.ai/settings/projects)\n", - "\n", - "1. Create an environment variable in a .env file or other method. By default, the AgentOps `init()` function will look for an environment variable named `AGENTOPS_API_KEY`. Or...\n", - "2. Replace `` below and pass in the optional `api_key` parameter to the AgentOps `init(api_key=...)` function. Remember not to commit your API key to a public repo!" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# Let's load our environment variables\n", - "load_dotenv()\n", - "\n", - "AGENTOPS_API_KEY = os.getenv(\"AGENTOPS_API_KEY\") or \"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Initialize AgentOps with some default tags\n", - "agentops.init(AGENTOPS_API_KEY, default_tags=[\"ollama-example\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now let's make some basic calls to Ollama. Make sure you have pulled the model first, use the following or replace with whichever model you want to use." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ollama.pull(\"mistral\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Basic completion,\n", - "response = ollama.chat(model='mistral',\n", - " messages=[{\n", - " 'role': 'user',\n", - " 'content': 'What are the benefits of using AgentOps for monitoring LLMs?',\n", - " }]\n", - ")\n", - "print(response['message']['content'])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's try streaming responses as well" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Streaming Example\n", - "stream = ollama.chat(\n", - " model='mistral',\n", - " messages=[{\n", - " 'role': 'user',\n", - " 'content': 'Write a haiku about monitoring AI agents',\n", - " }],\n", - " stream=True\n", - ")\n", - "\n", - "for chunk in stream:\n", - " print(chunk['message']['content'], end='')\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Conversation Example\n", - "messages = [\n", - " {\n", - " 'role': 'user',\n", - " 'content': 'What is AgentOps?'\n", - " },\n", - " {\n", - " 'role': 'assistant',\n", - " 'content': 'AgentOps is a monitoring and observability platform for LLM applications.'\n", - " },\n", - " {\n", - " 'role': 'user',\n", - " 'content': 'Can you give me 3 key features?'\n", - " }\n", - "]\n", - "\n", - "response = ollama.chat(\n", - " model='mistral',\n", - " messages=messages\n", - ")\n", - "print(response['message']['content'])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "> 💡 **Note**: In production environments, you should add proper error handling around the Ollama calls and use `agentops.end_session(\"Error\")` when exceptions occur." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finally, let's end our AgentOps session" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "agentops.end_session(\"Success\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "gpt_desk", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# AgentOps Ollama Integration\n", + "\n", + "This example demonstrates how to use AgentOps to monitor your Ollama LLM calls.\n", + "\n", + "First let's install the required packages\n", + "\n", + "> ⚠️ **Important**: Make sure you have Ollama installed and running locally before running this notebook. You can install it from [ollama.ai](https://ollama.com)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -U ollama\n", + "%pip install -U agentops\n", + "%pip install -U python-dotenv" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then import them" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import ollama\n", + "import agentops\n", + "import os\n", + "from dotenv import load_dotenv\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we'll set our API keys. For Ollama, we'll need to make sure Ollama is running locally.\n", + "[Get an AgentOps API key](https://agentops.ai/settings/projects)\n", + "\n", + "1. Create an environment variable in a .env file or other method. By default, the AgentOps `init()` function will look for an environment variable named `AGENTOPS_API_KEY`. Or...\n", + "2. Replace `` below and pass in the optional `api_key` parameter to the AgentOps `init(api_key=...)` function. Remember not to commit your API key to a public repo!" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Let's load our environment variables\n", + "load_dotenv()\n", + "\n", + "AGENTOPS_API_KEY = os.getenv(\"AGENTOPS_API_KEY\") or \"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize AgentOps with some default tags\n", + "agentops.init(AGENTOPS_API_KEY, default_tags=[\"ollama-example\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's make some basic calls to Ollama. Make sure you have pulled the model first, use the following or replace with whichever model you want to use." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ollama.pull(\"mistral\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Basic completion,\n", + "response = ollama.chat(model='mistral',\n", + " messages=[{\n", + " 'role': 'user',\n", + " 'content': 'What are the benefits of using AgentOps for monitoring LLMs?',\n", + " }]\n", + ")\n", + "print(response['message']['content'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's try streaming responses as well" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Streaming Example\n", + "stream = ollama.chat(\n", + " model='mistral',\n", + " messages=[{\n", + " 'role': 'user',\n", + " 'content': 'Write a haiku about monitoring AI agents',\n", + " }],\n", + " stream=True\n", + ")\n", + "\n", + "for chunk in stream:\n", + " print(chunk['message']['content'], end='')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Conversation Example\n", + "messages = [\n", + " {\n", + " 'role': 'user',\n", + " 'content': 'What is AgentOps?'\n", + " },\n", + " {\n", + " 'role': 'assistant',\n", + " 'content': 'AgentOps is a monitoring and observability platform for LLM applications.'\n", + " },\n", + " {\n", + " 'role': 'user',\n", + " 'content': 'Can you give me 3 key features?'\n", + " }\n", + "]\n", + "\n", + "response = ollama.chat(\n", + " model='mistral',\n", + " messages=messages\n", + ")\n", + "print(response['message']['content'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> 💡 **Note**: In production environments, you should add proper error handling around the Ollama calls and use `agentops.end_session(\"Error\")` when exceptions occur." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, let's end our AgentOps session" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "agentops.end_session(\"Success\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "gpt_desk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 + } + \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index e5129f690..74c0f791e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,10 @@ dependencies = [ "psutil==5.9.8", "packaging==23.2", "termcolor>=2.3.0", # 2.x.x tolerant - "PyYAML>=5.3,<7.0" + "PyYAML>=5.3,<7.0", + "opentelemetry-api>=1.22.0,<2.0.0", # API for interfaces + "opentelemetry-sdk>=1.22.0,<2.0.0", # SDK for implementation + "opentelemetry-exporter-otlp-proto-http>=1.22.0,<2.0.0", # For OTLPSpanExporter ] [project.optional-dependencies] dev = [ @@ -37,6 +40,7 @@ dev = [ "requests_mock==1.11.0", "ruff", "tach~=0.9", + "vcrpy>=6.0.0; python_version >= '3.8'" ] langchain = [ "langchain==0.2.14; python_version >= '3.8.1'" @@ -55,6 +59,11 @@ agentops = "agentops.cli:main" [tool.pytest.ini_options] asyncio_mode = "strict" asyncio_default_fixture_loop_scope = "function" +test_paths = [ + "tests", +] +addopts = "--import-mode=importlib --tb=short -p no:warnings" +pythonpath = ["."] [tool.ruff] line-length = 120 diff --git a/tests/core_manual_tests/api_server/main.py b/tests/core_manual_tests/api_server/main.py new file mode 100644 index 000000000..d4a5581a8 --- /dev/null +++ b/tests/core_manual_tests/api_server/main.py @@ -0,0 +1,37 @@ +import agentops +from fastapi import FastAPI +import uvicorn +from dotenv import load_dotenv +from agentops import record_tool +from openai import OpenAI +import time + +load_dotenv() + +openai = OpenAI() +agentops.init(auto_start_session=False) +app = FastAPI() + + +@app.get("/completion") +def completion(): + start_time = time.time() + + session = agentops.start_session(tags=["api-server-test"]) + + @record_tool(tool_name="foo") + def foo(x: str): + print(x) + + foo("Hello") + + session.end_session(end_state="Success") + + end_time = time.time() + execution_time = end_time - start_time + + return {"response": "Done", "execution_time_seconds": round(execution_time, 3)} + + +if __name__ == "__main__": + uvicorn.run("main:app", host="127.0.0.1", port=8000, reload=True) diff --git a/tests/core_manual_tests/api_server/readme.md b/tests/core_manual_tests/api_server/readme.md index 3f32804de..04e7dc168 100644 --- a/tests/core_manual_tests/api_server/readme.md +++ b/tests/core_manual_tests/api_server/readme.md @@ -1,9 +1,10 @@ # API server test This is a manual test with two files. It checks to make sure that the SDK works in an API environment. -## Running -1. `python server.py` -2. In different terminal, `python client.py` +## Running the FastAPI Server +You can run FastAPI with: +1. `uvicorn main:app --reload` +2. To test, run `curl http://localhost:8000/completion` in a different terminal. ## Validate Check in your AgentOps Dashboard diff --git a/tests/core_manual_tests/api_server/server.py b/tests/core_manual_tests/api_server/server.py deleted file mode 100644 index 5ae5e1ae2..000000000 --- a/tests/core_manual_tests/api_server/server.py +++ /dev/null @@ -1,39 +0,0 @@ -import agentops -from fastapi import FastAPI -import uvicorn -from dotenv import load_dotenv -from agentops import ActionEvent -from openai import OpenAI - -load_dotenv() - -openai = OpenAI() -agentops.init() -app = FastAPI() - - -@app.get("/completion") -def completion(): - session = agentops.start_session(tags=["api-server-test"]) - - messages = [{"role": "user", "content": "Hello"}] - response = session.patch(openai.chat.completions.create)( - model="gpt-3.5-turbo", - messages=messages, - temperature=0.5, - ) - - session.record( - ActionEvent( - action_type="Agent says hello", - params=messages, - returns=str(response.choices[0].message.content), - ), - ) - - session.end_session(end_state="Success") - - return {"response": response} - - -uvicorn.run(app, host="0.0.0.0", port=9696) diff --git a/tests/core_manual_tests/benchmark.py b/tests/core_manual_tests/benchmark.py new file mode 100644 index 000000000..ee73899c4 --- /dev/null +++ b/tests/core_manual_tests/benchmark.py @@ -0,0 +1,50 @@ +import logging + +# logging.basicConfig(level=logging.DEBUG) + +from datetime import datetime, timezone +from uuid import uuid4 + +import openai +from pyinstrument import Profiler + +import agentops + + +def make_openai_call(): + client = openai.Client() + return client.chat.completions.create( + model="gpt-3.5-turbo", + messages=[ + {"role": "system", "content": "You are a chatbot."}, + {"role": "user", "content": "What are you talking about?"}, + ], + ) + + +# Initialize profiler +profiler = Profiler() +profiler.start() + +try: + # Initialize AgentOps with auto_start_session=False + agentops.init(auto_start_session=False) + # Start a single test session + session = agentops.start_session() + assert session is not None + + # Make multiple calls + responses = [] + # Make 20 sequential calls for benchmarking + for _ in range(1): + responses.append(make_openai_call()) + + # End the session properly + session.end_session(end_state="Success") + +finally: + # Stop profiling and print results + profiler.stop() + # with open("profiling_reports/{}.txt".format(datetime.now(timezone.utc).isoformat()), "w") as f: + # f.write(profiler.output_text(unicode=True, color=False)) + print(profiler.output_text(unicode=True, color=True)) diff --git a/tests/test_session.py b/tests/test_session.py index 25ad8246a..4bbfb31d4 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -1,8 +1,22 @@ +import json +import time +from typing import Dict, Optional, Sequence +from unittest.mock import MagicMock, Mock, patch +from datetime import datetime, timezone + import pytest import requests_mock -import time +from opentelemetry import trace +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.trace import SpanContext, SpanKind +from opentelemetry.sdk.trace.export import SpanExportResult +from opentelemetry.trace import Status, StatusCode +from opentelemetry.trace.span import TraceState +from uuid import UUID + import agentops from agentops import ActionEvent, Client +from agentops.http_client import HttpClient from agentops.singleton import clear_singletons @@ -201,7 +215,17 @@ def test_get_analytics(self, mock_req): # Assert assert isinstance(analytics, dict) - assert all(key in analytics for key in ["LLM calls", "Tool calls", "Actions", "Errors", "Duration", "Cost"]) + assert all( + key in analytics + for key in [ + "LLM calls", + "Tool calls", + "Actions", + "Errors", + "Duration", + "Cost", + ] + ) # Check specific values assert analytics["LLM calls"] == 1 @@ -359,3 +383,228 @@ def test_get_analytics_multiple_sessions(self, mock_req): session_1.end_session(end_state) session_2.end_session(end_state) + + +class TestSessionExporter: + def setup_method(self): + self.api_key = "11111111-1111-4111-8111-111111111111" + # Initialize agentops first + agentops.init(api_key=self.api_key, max_wait_time=50, auto_start_session=False) + self.session = agentops.start_session() + assert self.session is not None # Verify session was created + self.exporter = self.session._otel_exporter + + def teardown_method(self): + """Clean up after each test""" + if self.session: + self.session.end_session("Success") + agentops.end_all_sessions() + clear_singletons() + + def create_test_span(self, name="test_span", attributes=None): + """Helper to create a test span with required attributes""" + if attributes is None: + attributes = {} + + # Ensure required attributes are present + base_attributes = { + "event.id": str(UUID(int=1)), + "event.type": "test_type", + "event.timestamp": datetime.now(timezone.utc).isoformat(), + "event.end_timestamp": datetime.now(timezone.utc).isoformat(), + "event.data": json.dumps({"test": "data"}), + "session.id": str(self.session.session_id), + } + base_attributes.update(attributes) + + context = SpanContext( + trace_id=0x000000000000000000000000DEADBEEF, + span_id=0x00000000DEADBEF0, + is_remote=False, + trace_state=TraceState(), + ) + + return ReadableSpan( + name=name, + context=context, + kind=SpanKind.INTERNAL, + status=Status(StatusCode.OK), + start_time=123, + end_time=456, + attributes=base_attributes, + events=[], + links=[], + resource=self.session._tracer_provider.resource, + ) + + def test_export_basic_span(self, mock_req): + """Test basic span export with all required fields""" + span = self.create_test_span() + result = self.exporter.export([span]) + + assert result == SpanExportResult.SUCCESS + assert len(mock_req.request_history) > 0 + + last_request = mock_req.last_request.json() + assert "events" in last_request + event = last_request["events"][0] + + # Verify required fields + assert "id" in event + assert "event_type" in event + assert "init_timestamp" in event + assert "end_timestamp" in event + assert "session_id" in event + + def test_export_action_event(self, mock_req): + """Test export of action event with specific formatting""" + action_attributes = { + "event.data": json.dumps( + { + "action_type": "test_action", + "params": {"param1": "value1"}, + "returns": "test_return", + } + ) + } + + span = self.create_test_span(name="actions", attributes=action_attributes) + result = self.exporter.export([span]) + + assert result == SpanExportResult.SUCCESS + + last_request = mock_req.request_history[-1].json() + event = last_request["events"][0] + + assert event["action_type"] == "test_action" + assert event["params"] == {"param1": "value1"} + assert event["returns"] == "test_return" + + def test_export_tool_event(self, mock_req): + """Test export of tool event with specific formatting""" + tool_attributes = { + "event.data": json.dumps( + { + "name": "test_tool", + "params": {"param1": "value1"}, + "returns": "test_return", + } + ) + } + + span = self.create_test_span(name="tools", attributes=tool_attributes) + result = self.exporter.export([span]) + + assert result == SpanExportResult.SUCCESS + + last_request = mock_req.request_history[-1].json() + event = last_request["events"][0] + + assert event["name"] == "test_tool" + assert event["params"] == {"param1": "value1"} + assert event["returns"] == "test_return" + + def test_export_with_missing_timestamp(self, mock_req): + """Test handling of missing end_timestamp""" + attributes = {"event.end_timestamp": None} # This should be handled gracefully + + span = self.create_test_span(attributes=attributes) + result = self.exporter.export([span]) + + assert result == SpanExportResult.SUCCESS + + last_request = mock_req.request_history[-1].json() + event = last_request["events"][0] + + # Verify end_timestamp is present and valid + assert "end_timestamp" in event + assert event["end_timestamp"] is not None + + def test_export_with_missing_timestamps_advanced(self, mock_req): + """Test handling of missing timestamps""" + attributes = {"event.timestamp": None, "event.end_timestamp": None} + + span = self.create_test_span(attributes=attributes) + result = self.exporter.export([span]) + + assert result == SpanExportResult.SUCCESS + + last_request = mock_req.request_history[-1].json() + event = last_request["events"][0] + + # Verify timestamps are present and valid + assert "init_timestamp" in event + assert "end_timestamp" in event + assert event["init_timestamp"] is not None + assert event["end_timestamp"] is not None + + # Verify timestamps are in ISO format + try: + datetime.fromisoformat(event["init_timestamp"].replace("Z", "+00:00")) + datetime.fromisoformat(event["end_timestamp"].replace("Z", "+00:00")) + except ValueError: + pytest.fail("Timestamps are not in valid ISO format") + + def test_export_with_shutdown(self, mock_req): + """Test export behavior when shutdown""" + self.exporter._shutdown.set() + span = self.create_test_span() + + result = self.exporter.export([span]) + assert result == SpanExportResult.SUCCESS + + # Verify no request was made + assert not any(req.url.endswith("/v2/create_events") for req in mock_req.request_history[-1:]) + + def test_export_llm_event(self, mock_req): + """Test export of LLM event with specific handling of timestamps""" + llm_attributes = { + "event.data": json.dumps( + { + "prompt": "test prompt", + "completion": "test completion", + "model": "test-model", + "tokens": 100, + "cost": 0.002, + } + ) + } + + span = self.create_test_span(name="llms", attributes=llm_attributes) + result = self.exporter.export([span]) + + assert result == SpanExportResult.SUCCESS + + last_request = mock_req.request_history[-1].json() + event = last_request["events"][0] + + # Verify LLM specific fields + assert event["prompt"] == "test prompt" + assert event["completion"] == "test completion" + assert event["model"] == "test-model" + assert event["tokens"] == 100 + assert event["cost"] == 0.002 + + # Verify timestamps + assert event["init_timestamp"] is not None + assert event["end_timestamp"] is not None + + def test_export_with_missing_id(self, mock_req): + """Test handling of missing event ID""" + attributes = {"event.id": None} + + span = self.create_test_span(attributes=attributes) + result = self.exporter.export([span]) + + assert result == SpanExportResult.SUCCESS + + last_request = mock_req.request_history[-1].json() + event = last_request["events"][0] + + # Verify ID is present and valid UUID + assert "id" in event + assert event["id"] is not None + try: + UUID(event["id"]) + except ValueError: + pytest.fail("Event ID is not a valid UUID")