diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index c342812ba..8a38c89ae 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -30,6 +30,34 @@ jobs: run: uv sync --group dev - name: Run pre-commit - run: uv run pre-commit run --files $(git ls-files .) + run: | + uv run pre-commit run --files \ + $(find \ + app/agent \ + app/component \ + app/controller \ + app/exception \ + app/middleware \ + app/model \ + app/service \ + tests/app \ + -type f ! -path '*__pycache__*') \ + app/__init__.py \ + app/router.py \ + app/component/__init__.py \ + app/component/pydantic/__init__.py \ + app/utils/listen/__init__.py \ + app/utils/server/__init__.py \ + app/utils/toolkit/__init__.py \ + app/utils/toolkit/google_calendar_toolkit.py \ + app/utils/toolkit/google_gmail_mcp_toolkit.py \ + app/utils/toolkit/linkedin_toolkit.py \ + app/utils/toolkit/reddit_toolkit.py \ + app/utils/toolkit/slack_toolkit.py \ + app/utils/toolkit/twitter_toolkit.py \ + app/utils/toolkit/whatsapp_toolkit.py \ + app/utils/workforce.py \ + app/utils/single_agent_worker.py \ + tests/conftest.py env: SKIP: no-commit-to-branch diff --git a/backend/app/component/model_validation.py b/backend/app/component/model_validation.py index fb855a94e..c8da48be8 100644 --- a/backend/app/component/model_validation.py +++ b/backend/app/component/model_validation.py @@ -12,8 +12,89 @@ # limitations under the License. # ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= +import logging +from enum import Enum +from typing import Any + from camel.agents import ChatAgent -from camel.models import ModelFactory +from camel.models import ModelFactory, ModelProcessingError + +logger = logging.getLogger("model_validation") + +# Expected result from tool execution for validation +EXPECTED_TOOL_RESULT = "Tool execution completed successfully for https://www.camel-ai.org, Website Content: Welcome to CAMEL AI!" + + +class ValidationStage(str, Enum): + """Stages of model validation process.""" + + INITIALIZATION = "initialization" + MODEL_CREATION = "model_creation" + AGENT_CREATION = "agent_creation" + MODEL_CALL = "model_call" + TOOL_CALL_EXECUTION = "tool_call_execution" + RESPONSE_PARSING = "response_parsing" + + +class ValidationErrorType(str, Enum): + """Types of validation errors.""" + + AUTHENTICATION_ERROR = "authentication_error" + NETWORK_ERROR = "network_error" + MODEL_NOT_FOUND = "model_not_found" + RATE_LIMIT_ERROR = "rate_limit_error" + QUOTA_EXCEEDED = "quota_exceeded" + TIMEOUT_ERROR = "timeout_error" + TOOL_CALL_NOT_SUPPORTED = "tool_call_not_supported" + TOOL_CALL_EXECUTION_FAILED = "tool_call_execution_failed" + INVALID_CONFIGURATION = "invalid_configuration" + UNKNOWN_ERROR = "unknown_error" + + +class ValidationResult: + """Detailed validation result with diagnostic information.""" + + def __init__(self): + self.is_valid: bool = False + self.is_tool_calls: bool = False + self.error_type: ValidationErrorType | None = None + self.error_code: str | None = None + self.error_message: str | None = None + self.raw_error_message: str | None = ( + None # Original error message from provider + ) + self.error_details: dict[str, Any] = {} + self.validation_stages: dict[ValidationStage, bool] = {} + self.diagnostic_info: dict[str, Any] = {} + self.successful_stages: list[ValidationStage] = [] + self.failed_stage: ValidationStage | None = None + self.model_response_info: dict[str, Any] | None = None + self.tool_call_info: dict[str, Any] | None = None + + def to_dict(self) -> dict[str, Any]: + """Convert validation result to dictionary.""" + return { + "is_valid": self.is_valid, + "is_tool_calls": self.is_tool_calls, + "error_type": self.error_type.value if self.error_type else None, + "error_code": self.error_code, + "error_message": self.error_message, + "raw_error_message": self.raw_error_message, + "error_details": self.error_details, + "validation_stages": { + stage.value: success + for stage, success in self.validation_stages.items() + }, + "diagnostic_info": self.diagnostic_info, + "successful_stages": [ + stage.value for stage in self.successful_stages + ], + "failed_stage": self.failed_stage.value + if self.failed_stage + else None, + "model_response_info": self.model_response_info, + "tool_call_info": self.tool_call_info, + } def get_website_content(url: str) -> str: @@ -25,7 +106,94 @@ def get_website_content(url: str) -> str: Returns: str: The content of the website. """ - return "Tool execution completed successfully for https://www.camel-ai.org, Website Content: Welcome to CAMEL AI!" + return EXPECTED_TOOL_RESULT + + +def format_raw_error(exception: Exception, max_length: int = 300) -> str: + """Format raw error message from exception, truncating if too long. + + This preserves the original error message from the provider without + translation, as provider error messages are often clear and informative. + + Args: + exception: The exception to format + max_length: Maximum length of the error message + + Returns: + str: Formatted error message + """ + raw = str(exception) + if len(raw) > max_length: + raw = raw[:max_length] + "..." + return raw + + +def categorize_error( + exception: Exception, stage: ValidationStage +) -> ValidationErrorType: + """Categorize exception into specific error type. + + This function attempts to categorize errors conservatively, only when + we have high confidence (e.g., exception types, HTTP status codes). + The raw error message is always preserved separately. + + Args: + exception: The exception to categorize + stage: The validation stage where error occurred + + Returns: + ValidationErrorType: The categorized error type, or UNKNOWN_ERROR if uncertain + """ + error_str = str(exception).lower() + error_type = exception.__class__.__name__.lower() + exception_type_str = str(type(exception)).lower() + + # First, check exception type for common patterns + # This is the most reliable way to categorize errors + if "timeout" in error_type or "timeouterror" in exception_type_str: + return ValidationErrorType.TIMEOUT_ERROR + + if "connection" in error_type or "connectionerror" in exception_type_str: + return ValidationErrorType.NETWORK_ERROR + + if "authentication" in error_type or "autherror" in exception_type_str: + return ValidationErrorType.AUTHENTICATION_ERROR + + # Check for ModelProcessingError from camel-ai, which wraps provider-specific errors + # This is reliable because camel-ai standardizes error handling + if isinstance(exception, ModelProcessingError): + # Check for HTTP status codes first (most reliable) + if "401" in error_str or "unauthorized" in error_str: + return ValidationErrorType.AUTHENTICATION_ERROR + if "404" in error_str or "not found" in error_str: + return ValidationErrorType.MODEL_NOT_FOUND + if "429" in error_str or "rate limit" in error_str: + return ValidationErrorType.RATE_LIMIT_ERROR + if "timeout" in error_str or "timed out" in error_str: + return ValidationErrorType.TIMEOUT_ERROR + if "quota" in error_str or "insufficient_quota" in error_str: + return ValidationErrorType.QUOTA_EXCEEDED + if "connection" in error_str or "network" in error_str: + return ValidationErrorType.NETWORK_ERROR + + # Only check for very specific, unambiguous patterns in error messages + # Prefer HTTP status codes and exception types over string matching + + # HTTP status codes are reliable indicators + if "401" in error_str or "unauthorized" in error_str: + return ValidationErrorType.AUTHENTICATION_ERROR + if "404" in error_str: + return ValidationErrorType.MODEL_NOT_FOUND + if "429" in error_str: + return ValidationErrorType.RATE_LIMIT_ERROR + + # Very specific error patterns that are unlikely to be false positives + if "insufficient_quota" in error_str or "quota exceeded" in error_str: + return ValidationErrorType.QUOTA_EXCEEDED + + # Return unknown if we're not confident + # The raw error message will still be available to users + return ValidationErrorType.UNKNOWN_ERROR def create_agent( @@ -36,6 +204,23 @@ def create_agent( model_config_dict: dict = None, **kwargs, ) -> ChatAgent: + """Create an agent for model validation. + + Args: + model_platform: The model platform + model_type: The model type + api_key: API key for authentication + url: Custom model URL + model_config_dict: Model configuration dictionary + **kwargs: Additional model parameters + + Returns: + ChatAgent: The created agent + + Raises: + ValueError: If model_type or model_platform is invalid + Exception: If model creation fails + """ platform = model_platform mtype = model_type if mtype is None: @@ -55,6 +240,371 @@ def create_agent( system_message="You are a helpful assistant that must use the tool get_website_content to get the content of a website.", model=model, tools=[get_website_content], - step_timeout=1800, # 30 minutes + step_timeout=60, # 1 minute for validation ) return agent + + +def validate_model_with_details( + model_platform: str, + model_type: str, + api_key: str = None, + url: str = None, + model_config_dict: dict = None, + **kwargs, +) -> ValidationResult: + """Validate model with detailed diagnostic information. + + Args: + model_platform: The model platform + model_type: The model type + api_key: API key for authentication + url: Custom model URL + model_config_dict: Model configuration dictionary + **kwargs: Additional model parameters + + Returns: + ValidationResult: Detailed validation result + """ + result = ValidationResult() + + # Stage 1: Initialization + result.validation_stages[ValidationStage.INITIALIZATION] = False + try: + if model_type is None or model_type.strip() == "": + result.error_type = ValidationErrorType.INVALID_CONFIGURATION + result.error_message = ( + "Model type is required but was not provided." + ) + result.error_details = {"missing_field": "model_type"} + result.failed_stage = ValidationStage.INITIALIZATION + return result + + if model_platform is None or model_platform.strip() == "": + result.error_type = ValidationErrorType.INVALID_CONFIGURATION + result.error_message = ( + "Model platform is required but was not provided." + ) + result.error_details = {"missing_field": "model_platform"} + result.failed_stage = ValidationStage.INITIALIZATION + return result + + result.validation_stages[ValidationStage.INITIALIZATION] = True + result.successful_stages.append(ValidationStage.INITIALIZATION) + result.diagnostic_info["initialization"] = { + "model_platform": model_platform, + "model_type": model_type, + "has_api_key": api_key is not None and api_key.strip() != "", + "has_custom_url": url is not None, + "has_config": model_config_dict is not None, + } + logger.debug( + "Initialization stage passed", + extra={"platform": model_platform, "model_type": model_type}, + ) + + except Exception as e: + result.error_type = ValidationErrorType.INVALID_CONFIGURATION + result.raw_error_message = format_raw_error(e) + result.error_message = result.raw_error_message + result.error_details = { + "exception_type": type(e).__name__, + "exception_message": str(e), + } + result.failed_stage = ValidationStage.INITIALIZATION + logger.error( + "Initialization stage failed", + extra={"error": str(e)}, + exc_info=True, + ) + return result + + # Stage 2: Model Creation + result.validation_stages[ValidationStage.MODEL_CREATION] = False + try: + logger.debug( + "Creating model", + extra={"platform": model_platform, "model_type": model_type}, + ) + model = ModelFactory.create( + model_platform=model_platform, + model_type=model_type, + api_key=api_key, + url=url, + timeout=60, # 1 minute for validation + model_config_dict=model_config_dict, + **kwargs, + ) + result.validation_stages[ValidationStage.MODEL_CREATION] = True + result.successful_stages.append(ValidationStage.MODEL_CREATION) + result.diagnostic_info["model_creation"] = { + "model_platform": model_platform, + "model_type": model_type, + "model_created": True, + } + logger.debug( + "Model creation stage passed", + extra={"platform": model_platform, "model_type": model_type}, + ) + + except Exception as e: + error_type = categorize_error(e, ValidationStage.MODEL_CREATION) + result.error_type = error_type + # Always preserve the raw error message from the provider + result.raw_error_message = format_raw_error(e) + # Use raw error as primary message - it's usually clear and informative + result.error_message = result.raw_error_message + result.error_details = { + "exception_type": type(e).__name__, + "exception_message": str(e), + } + result.failed_stage = ValidationStage.MODEL_CREATION + + logger.error( + "Model creation stage failed", + extra={"error": str(e), "error_type": error_type.value}, + exc_info=True, + ) + return result + + # Stage 3: Agent Creation + result.validation_stages[ValidationStage.AGENT_CREATION] = False + try: + logger.debug( + "Creating agent", + extra={"platform": model_platform, "model_type": model_type}, + ) + agent = ChatAgent( + system_message="You are a helpful assistant that must use the tool get_website_content to get the content of a website.", + model=model, + tools=[get_website_content], + step_timeout=60, # 1 minute for validation + ) + result.validation_stages[ValidationStage.AGENT_CREATION] = True + result.successful_stages.append(ValidationStage.AGENT_CREATION) + result.diagnostic_info["agent_creation"] = { + "agent_created": True, + } + logger.debug( + "Agent creation stage passed", + extra={"platform": model_platform, "model_type": model_type}, + ) + + except Exception as e: + error_type = categorize_error(e, ValidationStage.AGENT_CREATION) + result.error_type = error_type + # Always preserve the raw error message from the provider + result.raw_error_message = format_raw_error(e) + result.error_message = result.raw_error_message + result.error_details = { + "exception_type": type(e).__name__, + "exception_message": str(e), + } + result.failed_stage = ValidationStage.AGENT_CREATION + logger.error( + "Agent creation stage failed", + extra={"error": str(e)}, + exc_info=True, + ) + return result + + # Stage 4: Model Call + result.validation_stages[ValidationStage.MODEL_CALL] = False + try: + logger.debug( + "Executing model call", + extra={"platform": model_platform, "model_type": model_type}, + ) + response = agent.step( + input_message=""" + Get the content of https://www.camel-ai.org, + you must use the get_website_content tool to get the content , + i just want to verify the get_website_content tool is working, + you must call the get_website_content tool only once. + """ + ) + + if response: + result.validation_stages[ValidationStage.MODEL_CALL] = True + result.successful_stages.append(ValidationStage.MODEL_CALL) + + # Extract model response information + result.model_response_info = { + "has_response": True, + "has_message": hasattr(response, "msg") + and response.msg is not None, + "has_info": hasattr(response, "info") + and response.info is not None, + } + + if hasattr(response, "msg") and response.msg: + result.model_response_info["message_content"] = ( + str(response.msg.content)[:200] + if hasattr(response.msg, "content") + else None + ) + + if hasattr(response, "info") and response.info: + result.model_response_info["usage"] = response.info.get( + "usage", {} + ) + result.model_response_info["tool_calls_count"] = len( + response.info.get("tool_calls", []) + ) + + logger.debug( + "Model call stage passed", + extra={"platform": model_platform, "model_type": model_type}, + ) + else: + result.error_type = ValidationErrorType.UNKNOWN_ERROR + result.error_message = ( + "Model call succeeded but returned no response." + ) + result.error_details = {"response": None} + result.failed_stage = ValidationStage.MODEL_CALL + logger.warning( + "Model call returned no response", + extra={"platform": model_platform, "model_type": model_type}, + ) + return result + + except Exception as e: + error_type = categorize_error(e, ValidationStage.MODEL_CALL) + result.error_type = error_type + # Always preserve the raw error message from the provider + result.raw_error_message = format_raw_error(e) + result.error_message = result.raw_error_message + result.error_details = { + "exception_type": type(e).__name__, + "exception_message": str(e), + } + result.failed_stage = ValidationStage.MODEL_CALL + + logger.error( + "Model call stage failed", + extra={"error": str(e), "error_type": error_type.value}, + exc_info=True, + ) + return result + + # Stage 5: Tool Call Execution Check + result.validation_stages[ValidationStage.TOOL_CALL_EXECUTION] = False + try: + if response and hasattr(response, "info") and response.info: + tool_calls = response.info.get("tool_calls", []) + + result.tool_call_info = { + "tool_calls_count": len(tool_calls), + "has_tool_calls": len(tool_calls) > 0, + } + + if tool_calls and len(tool_calls) > 0: + tool_call = tool_calls[0] + result.tool_call_info["first_tool_call"] = { + "tool_name": getattr(tool_call, "tool_name", None), + "has_result": hasattr(tool_call, "result"), + "result": str(getattr(tool_call, "result", ""))[:200] + if hasattr(tool_call, "result") + else None, + } + + expected_result = EXPECTED_TOOL_RESULT + actual_result = ( + tool_call.result if hasattr(tool_call, "result") else None + ) + + if actual_result == expected_result: + result.validation_stages[ + ValidationStage.TOOL_CALL_EXECUTION + ] = True + result.successful_stages.append( + ValidationStage.TOOL_CALL_EXECUTION + ) + result.is_tool_calls = True + result.is_valid = True + result.tool_call_info["execution_successful"] = True + logger.debug( + "Tool call execution stage passed", + extra={ + "platform": model_platform, + "model_type": model_type, + }, + ) + else: + result.error_type = ( + ValidationErrorType.TOOL_CALL_EXECUTION_FAILED + ) + result.error_message = f"Tool call was made but execution failed. Expected result: '{expected_result[:50]}...', but got: '{str(actual_result)[:50] if actual_result else 'None'}...'" + result.error_details = { + "expected_result": expected_result, + "actual_result": str(actual_result) + if actual_result + else None, + "tool_call": str(tool_call)[:200], + } + result.failed_stage = ValidationStage.TOOL_CALL_EXECUTION + result.tool_call_info["execution_successful"] = False + logger.warning( + "Tool call execution failed", + extra={ + "platform": model_platform, + "model_type": model_type, + "expected": expected_result[:50], + "actual": str(actual_result)[:50] + if actual_result + else None, + }, + ) + else: + result.error_type = ValidationErrorType.TOOL_CALL_NOT_SUPPORTED + result.error_message = "Model call succeeded, but the model did not make any tool calls. This model may not support tool calling functionality." + result.error_details = { + "tool_calls": [], + "response_info": str(response.info) + if hasattr(response, "info") + else None, + } + result.failed_stage = ValidationStage.TOOL_CALL_EXECUTION + result.tool_call_info["execution_successful"] = False + logger.warning( + "No tool calls made by model", + extra={ + "platform": model_platform, + "model_type": model_type, + }, + ) + else: + result.error_type = ValidationErrorType.TOOL_CALL_NOT_SUPPORTED + result.error_message = "Model call succeeded, but response does not contain tool call information. This model may not support tool calling functionality." + result.error_details = { + "has_info": hasattr(response, "info") if response else False, + "response_type": type(response).__name__ if response else None, + } + result.failed_stage = ValidationStage.TOOL_CALL_EXECUTION + result.tool_call_info = { + "execution_successful": False, + "has_info": False, + } + logger.warning( + "Response missing tool call info", + extra={"platform": model_platform, "model_type": model_type}, + ) + + except Exception as e: + error_type = categorize_error(e, ValidationStage.TOOL_CALL_EXECUTION) + result.error_type = error_type + result.raw_error_message = format_raw_error(e) + result.error_message = result.raw_error_message + result.error_details = { + "exception_type": type(e).__name__, + "exception_message": str(e), + } + result.failed_stage = ValidationStage.TOOL_CALL_EXECUTION + logger.error( + "Tool call execution check failed", + extra={"error": str(e)}, + exc_info=True, + ) + + return result diff --git a/backend/app/controller/model_controller.py b/backend/app/controller/model_controller.py index 4a72ef2cb..46894c723 100644 --- a/backend/app/controller/model_controller.py +++ b/backend/app/controller/model_controller.py @@ -18,11 +18,16 @@ from pydantic import BaseModel, Field, field_validator from app.component.error_format import normalize_error_to_openai_format -from app.component.model_validation import create_agent +from app.component.model_validation import ( + ValidationErrorType, + ValidationStage, + validate_model_with_details, +) from app.model.chat import PLATFORM_MAPPING logger = logging.getLogger("model_controller") + router = APIRouter() @@ -37,6 +42,9 @@ class ValidateModelRequest(BaseModel): extra_params: dict | None = Field( None, description="Extra model parameters" ) + include_diagnostics: bool = Field( + False, description="Include detailed diagnostic information" + ) @field_validator("model_platform") @classmethod @@ -50,11 +58,41 @@ class ValidateModelResponse(BaseModel): error_code: str | None = Field(None, description="Error code") error: dict | None = Field(None, description="OpenAI-style error object") message: str = Field(..., description="Message") + error_type: str | None = Field(None, description="Detailed error type") + failed_stage: str | None = Field( + None, description="Stage where validation failed" + ) + successful_stages: list[str] | None = Field( + None, description="Stages that succeeded" + ) + diagnostic_info: dict | None = Field( + None, description="Diagnostic information" + ) + model_response_info: dict | None = Field( + None, description="Model response information" + ) + tool_call_info: dict | None = Field( + None, description="Tool call information" + ) + validation_stages: dict[str, bool] | None = Field( + None, description="Validation stages status" + ) @router.post("/model/validate") async def validate_model(request: ValidateModelRequest): - """Validate model configuration and tool call support.""" + """Validate model configuration and tool call support with detailed error messages. + + This endpoint validates a model configuration and provides detailed error messages + to help users understand the root cause of validation failures. It checks: + 1. Initialization (model type and platform) + 2. Model creation (authentication, network, model availability) + 3. Agent creation + 4. Model call execution + 5. Tool call execution + + Returns detailed diagnostic information if include_diagnostics is True. + """ platform = request.model_platform model_type = request.model_type has_custom_url = request.url is not None @@ -67,6 +105,7 @@ async def validate_model(request: ValidateModelRequest): "model_type": model_type, "has_url": has_custom_url, "has_config": has_config, + "include_diagnostics": request.include_diagnostics, }, ) @@ -79,12 +118,15 @@ async def validate_model(request: ValidateModelRequest): raise HTTPException( status_code=400, detail={ - "message": "Invalid key. Validation failed.", + "message": "Invalid key. Validation failed. Please provide a valid API key.", "error_code": "invalid_api_key", + "error_type": ValidationErrorType.AUTHENTICATION_ERROR.value, + "failed_stage": ValidationStage.INITIALIZATION.value, "error": { "type": "invalid_request_error", - "param": None, + "param": "api_key", "code": "invalid_api_key", + "message": "API key cannot be empty. Please provide a valid API key.", }, }, ) @@ -93,10 +135,10 @@ async def validate_model(request: ValidateModelRequest): extra = request.extra_params or {} logger.debug( - "Creating agent for validation", + "Starting detailed model validation", extra={"platform": platform, "model_type": model_type}, ) - agent = create_agent( + validation_result = validate_model_with_details( platform, model_type, api_key=request.api_key, @@ -105,23 +147,129 @@ async def validate_model(request: ValidateModelRequest): **extra, ) - logger.debug( - "Agent created, executing test step", - extra={"platform": platform, "model_type": model_type}, - ) - response = agent.step( - input_message=""" - Get the content of https://www.camel-ai.org, - you must use the get_website_content tool to get the content , - i just want to verify the get_website_content tool is working, - you must call the get_website_content tool only once. - """ - ) + # Build response message based on validation result + # Prefer raw error messages from providers as they are usually clear and informative + if validation_result.is_tool_calls: + message = "Validation successful. Model supports tool calling and tool execution completed successfully." + elif validation_result.is_valid: + if ( + validation_result.error_type + == ValidationErrorType.TOOL_CALL_NOT_SUPPORTED + ): + message = "Model call succeeded, but this model does not support tool calling functionality. Please try with another model that supports tool calls." + elif ( + validation_result.error_type + == ValidationErrorType.TOOL_CALL_EXECUTION_FAILED + ): + # Use raw error message if available, otherwise use the formatted one + message = ( + validation_result.raw_error_message + or validation_result.error_message + or "Tool call execution failed." + ) + else: + message = ( + validation_result.raw_error_message + or validation_result.error_message + or "Model call succeeded, but tool call validation failed. Please check the model configuration." + ) + else: + # Use raw error message as primary message - provider errors are usually clear + # Only add context for specific cases where it's helpful + if validation_result.raw_error_message: + message = validation_result.raw_error_message + elif validation_result.error_message: + message = validation_result.error_message + else: + message = "Model validation failed. Please check your configuration and try again." + + # Convert error type to error code for backward compatibility + error_code = None + error_obj = None + + if validation_result.error_type: + error_code = validation_result.error_type.value + + # Create OpenAI-style error object + error_obj = { + "type": "invalid_request_error", + "param": None, + "code": validation_result.error_type.value, + "message": validation_result.error_message or message, + } + + # Add specific error details if available + if validation_result.error_details: + error_obj["details"] = validation_result.error_details + + # Build response + response_data = { + "is_valid": validation_result.is_valid, + "is_tool_calls": validation_result.is_tool_calls, + "error_code": error_code, + "error": error_obj, + "message": message, + } + + # Include detailed diagnostic information if requested + if request.include_diagnostics: + response_data["error_type"] = ( + validation_result.error_type.value + if validation_result.error_type + else None + ) + response_data["failed_stage"] = ( + validation_result.failed_stage.value + if validation_result.failed_stage + else None + ) + response_data["successful_stages"] = [ + stage.value for stage in validation_result.successful_stages + ] + response_data["diagnostic_info"] = ( + validation_result.diagnostic_info + ) + response_data["model_response_info"] = ( + validation_result.model_response_info + ) + response_data["tool_call_info"] = validation_result.tool_call_info + response_data["validation_stages"] = { + stage.value: success + for stage, success in validation_result.validation_stages.items() + } + + result = ValidateModelResponse(**response_data) + + # Use error or warning log level if there's an issue + log_extra = { + "platform": platform, + "model_type": model_type, + "is_valid": validation_result.is_valid, + "is_tool_calls": validation_result.is_tool_calls, + "error_type": validation_result.error_type.value + if validation_result.error_type + else None, + "failed_stage": validation_result.failed_stage.value + if validation_result.failed_stage + else None, + } + + if not validation_result.is_valid: + logger.error("Model validation completed", extra=log_extra) + elif validation_result.error_type: + logger.warning("Model validation completed", extra=log_extra) + else: + logger.info("Model validation completed", extra=log_extra) + return result + + except HTTPException: + # Re-raise HTTP exceptions as-is + raise except Exception as e: - # Normalize error to OpenAI-style error structure + # Fallback error handling for unexpected errors logger.error( - "Model validation failed", + "Unexpected error during model validation", extra={ "platform": platform, "model_type": model_type, @@ -129,52 +277,18 @@ async def validate_model(request: ValidateModelRequest): }, exc_info=True, ) + message, error_code, error_obj = normalize_error_to_openai_format(e) raise HTTPException( - status_code=400, + status_code=500, detail={ - "message": message, - "error_code": error_code, - "error": error_obj, + "message": f"Unexpected error during validation: {message}", + "error_code": error_code or "internal_error", + "error": error_obj + or { + "type": "internal_error", + "message": str(e), + }, }, ) - - # Check validation results - is_valid = bool(response) - is_tool_calls = False - - if response and hasattr(response, "info") and response.info: - tool_calls = response.info.get("tool_calls", []) - if tool_calls and len(tool_calls) > 0: - expected = ( - "Tool execution completed" - " successfully for" - " https://www.camel-ai.org," - " Website Content:" - " Welcome to CAMEL AI!" - ) - is_tool_calls = tool_calls[0].result == expected - - no_tool_msg = ( - "This model doesn't support tool calls. please try with another model." - ) - result = ValidateModelResponse( - is_valid=is_valid, - is_tool_calls=is_tool_calls, - message="Validation Success" if is_tool_calls else no_tool_msg, - error_code=None, - error=None, - ) - - logger.info( - "Model validation completed", - extra={ - "platform": platform, - "model_type": model_type, - "is_valid": is_valid, - "is_tool_calls": is_tool_calls, - }, - ) - - return result diff --git a/backend/tests/app/component/test_model_validation.py b/backend/tests/app/component/test_model_validation.py new file mode 100644 index 000000000..a8c65d5f8 --- /dev/null +++ b/backend/tests/app/component/test_model_validation.py @@ -0,0 +1,419 @@ +# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= + +from unittest.mock import MagicMock, patch + +import pytest +from camel.models import ModelProcessingError + +from app.component.model_validation import ( + EXPECTED_TOOL_RESULT, + ValidationErrorType, + ValidationResult, + ValidationStage, + categorize_error, + create_agent, + format_raw_error, + validate_model_with_details, +) + + +@pytest.mark.unit +def test_validation_result_initialization(): + """Test ValidationResult initialization.""" + result = ValidationResult() + assert result.is_valid is False + assert result.is_tool_calls is False + assert result.error_type is None + assert result.error_code is None + assert result.error_message is None + assert result.raw_error_message is None + assert result.error_details == {} + assert result.validation_stages == {} + assert result.diagnostic_info == {} + assert result.successful_stages == [] + assert result.failed_stage is None + assert result.model_response_info is None + assert result.tool_call_info is None + + +@pytest.mark.unit +def test_validation_result_to_dict(): + """Test ValidationResult.to_dict() method.""" + result = ValidationResult() + result.is_valid = True + result.is_tool_calls = True + result.error_type = ValidationErrorType.AUTHENTICATION_ERROR + result.error_message = "Test error" + result.raw_error_message = "Raw test error" + result.validation_stages[ValidationStage.INITIALIZATION] = True + result.successful_stages.append(ValidationStage.INITIALIZATION) + result.failed_stage = ValidationStage.MODEL_CREATION + + result_dict = result.to_dict() + assert result_dict["is_valid"] is True + assert result_dict["is_tool_calls"] is True + assert result_dict["error_type"] == "authentication_error" + assert result_dict["error_message"] == "Test error" + assert result_dict["raw_error_message"] == "Raw test error" + assert result_dict["validation_stages"]["initialization"] is True + assert "initialization" in result_dict["successful_stages"] + assert result_dict["failed_stage"] == "model_creation" + + +@pytest.mark.unit +def test_format_short_error(): + """Test formatting of short error message.""" + error = ValueError("Short error message") + result = format_raw_error(error) + assert result == "Short error message" + + +@pytest.mark.unit +def test_format_long_error(): + """Test formatting of long error message with truncation.""" + long_message = "A" * 500 + error = ValueError(long_message) + result = format_raw_error(error, max_length=100) + assert len(result) == 103 # 100 + "..." + assert result.endswith("...") + + +@pytest.mark.unit +def test_format_error_with_default_max_length(): + """Test formatting with default max_length.""" + long_message = "A" * 400 + error = ValueError(long_message) + result = format_raw_error(error) + assert len(result) == 303 # 300 + "..." + assert result.endswith("...") + + +@pytest.mark.unit +def test_timeout_error(): + """Test categorization of timeout errors.""" + error = TimeoutError("Request timed out") + result = categorize_error(error, ValidationStage.MODEL_CALL) + assert result == ValidationErrorType.TIMEOUT_ERROR + + +@pytest.mark.unit +def test_connection_error(): + """Test categorization of connection errors.""" + error = ConnectionError("Connection failed") + result = categorize_error(error, ValidationStage.MODEL_CALL) + assert result == ValidationErrorType.NETWORK_ERROR + + +@pytest.mark.unit +def test_model_processing_error_401(): + """Test categorization of 401 errors from ModelProcessingError.""" + error = ModelProcessingError("401 Unauthorized") + result = categorize_error(error, ValidationStage.MODEL_CREATION) + assert result == ValidationErrorType.AUTHENTICATION_ERROR + + +@pytest.mark.unit +def test_model_processing_error_404(): + """Test categorization of 404 errors from ModelProcessingError.""" + error = ModelProcessingError("404 Model not found") + result = categorize_error(error, ValidationStage.MODEL_CREATION) + assert result == ValidationErrorType.MODEL_NOT_FOUND + + +@pytest.mark.unit +def test_model_processing_error_429(): + """Test categorization of 429 errors from ModelProcessingError.""" + error = ModelProcessingError("429 Rate limit exceeded") + result = categorize_error(error, ValidationStage.MODEL_CALL) + assert result == ValidationErrorType.RATE_LIMIT_ERROR + + +@pytest.mark.unit +def test_model_processing_error_quota(): + """Test categorization of quota errors from ModelProcessingError.""" + error = ModelProcessingError("Insufficient quota") + result = categorize_error(error, ValidationStage.MODEL_CALL) + assert result == ValidationErrorType.QUOTA_EXCEEDED + + +@pytest.mark.unit +def test_unknown_error(): + """Test categorization of unknown errors.""" + error = ValueError("Some random error") + result = categorize_error(error, ValidationStage.MODEL_CALL) + assert result == ValidationErrorType.UNKNOWN_ERROR + + +@pytest.mark.unit +def test_http_status_code_401(): + """Test categorization based on HTTP status code 401.""" + error = Exception("401 Unauthorized access") + result = categorize_error(error, ValidationStage.MODEL_CREATION) + assert result == ValidationErrorType.AUTHENTICATION_ERROR + + +@pytest.mark.unit +def test_http_status_code_404(): + """Test categorization based on HTTP status code 404.""" + error = Exception("404 Not found") + result = categorize_error(error, ValidationStage.MODEL_CREATION) + assert result == ValidationErrorType.MODEL_NOT_FOUND + + +@pytest.mark.unit +def test_http_status_code_429(): + """Test categorization based on HTTP status code 429.""" + error = Exception("429 Too many requests") + result = categorize_error(error, ValidationStage.MODEL_CALL) + assert result == ValidationErrorType.RATE_LIMIT_ERROR + + +@pytest.mark.unit +@patch("app.component.model_validation.ModelFactory.create") +@patch("app.component.model_validation.ChatAgent") +def test_create_agent_success(mock_chat_agent, mock_model_factory): + """Test successful agent creation.""" + mock_model = MagicMock() + mock_model_factory.return_value = mock_model + mock_agent_instance = MagicMock() + mock_chat_agent.return_value = mock_agent_instance + + agent = create_agent( + model_platform="OPENAI", + model_type="GPT_4O_MINI", + api_key="test_key", + ) + + mock_model_factory.assert_called_once() + mock_chat_agent.assert_called_once() + assert agent == mock_agent_instance + + +@pytest.mark.unit +def test_create_agent_invalid_model_type(): + """Test agent creation with invalid model type.""" + with pytest.raises(ValueError, match="Invalid model_type"): + create_agent(model_platform="OPENAI", model_type=None) + + +@pytest.mark.unit +def test_create_agent_invalid_model_platform(): + """Test agent creation with invalid model platform.""" + with pytest.raises(ValueError, match="Invalid model_platform"): + create_agent(model_platform=None, model_type="GPT_4O_MINI") + + +@pytest.mark.unit +def test_validation_missing_model_type(): + """Test validation with missing model type.""" + result = validate_model_with_details( + model_platform="OPENAI", model_type="" + ) + assert result.is_valid is False + assert result.error_type == ValidationErrorType.INVALID_CONFIGURATION + assert result.failed_stage == ValidationStage.INITIALIZATION + assert "Model type is required" in result.error_message + + +@pytest.mark.unit +def test_validation_missing_model_platform(): + """Test validation with missing model platform.""" + result = validate_model_with_details( + model_platform="", model_type="GPT_4O_MINI" + ) + assert result.is_valid is False + assert result.error_type == ValidationErrorType.INVALID_CONFIGURATION + assert result.failed_stage == ValidationStage.INITIALIZATION + assert "Model platform is required" in result.error_message + + +@pytest.mark.unit +@patch("app.component.model_validation.ModelFactory.create") +def test_validation_model_creation_failure(mock_model_factory): + """Test validation when model creation fails.""" + mock_model_factory.side_effect = ModelProcessingError("401 Unauthorized") + + result = validate_model_with_details( + model_platform="OPENAI", + model_type="GPT_4O_MINI", + api_key="invalid_key", + ) + + assert result.is_valid is False + assert result.error_type == ValidationErrorType.AUTHENTICATION_ERROR + assert result.failed_stage == ValidationStage.MODEL_CREATION + assert result.validation_stages[ValidationStage.INITIALIZATION] is True + assert ValidationStage.INITIALIZATION in result.successful_stages + + +@pytest.mark.unit +@patch("app.component.model_validation.ModelFactory.create") +@patch("app.component.model_validation.ChatAgent") +def test_validation_agent_creation_failure( + mock_chat_agent, mock_model_factory +): + """Test validation when agent creation fails.""" + mock_model = MagicMock() + mock_model_factory.return_value = mock_model + mock_chat_agent.side_effect = Exception("Agent creation failed") + + result = validate_model_with_details( + model_platform="OPENAI", + model_type="GPT_4O_MINI", + api_key="test_key", + ) + + assert result.is_valid is False + assert result.failed_stage == ValidationStage.AGENT_CREATION + assert result.validation_stages[ValidationStage.MODEL_CREATION] is True + + +@pytest.mark.unit +@patch("app.component.model_validation.ModelFactory.create") +@patch("app.component.model_validation.ChatAgent") +def test_validation_model_call_failure(mock_chat_agent, mock_model_factory): + """Test validation when model call fails.""" + mock_model = MagicMock() + mock_model_factory.return_value = mock_model + mock_agent = MagicMock() + mock_agent.step.side_effect = Exception("Model call failed") + mock_chat_agent.return_value = mock_agent + + result = validate_model_with_details( + model_platform="OPENAI", + model_type="GPT_4O_MINI", + api_key="test_key", + ) + + assert result.is_valid is False + assert result.failed_stage == ValidationStage.MODEL_CALL + assert result.validation_stages[ValidationStage.AGENT_CREATION] is True + + +@pytest.mark.unit +@patch("app.component.model_validation.ModelFactory.create") +@patch("app.component.model_validation.ChatAgent") +def test_validation_no_tool_calls(mock_chat_agent, mock_model_factory): + """Test validation when model doesn't make tool calls.""" + mock_model = MagicMock() + mock_model_factory.return_value = mock_model + mock_agent = MagicMock() + mock_response = MagicMock() + mock_response.info = {"tool_calls": []} # No tool calls + mock_agent.step.return_value = mock_response + mock_chat_agent.return_value = mock_agent + + result = validate_model_with_details( + model_platform="OPENAI", + model_type="GPT_4O_MINI", + api_key="test_key", + ) + + assert result.is_valid is False + assert result.is_tool_calls is False + assert result.error_type == ValidationErrorType.TOOL_CALL_NOT_SUPPORTED + assert result.failed_stage == ValidationStage.TOOL_CALL_EXECUTION + assert result.validation_stages[ValidationStage.MODEL_CALL] is True + + +@pytest.mark.unit +@patch("app.component.model_validation.ModelFactory.create") +@patch("app.component.model_validation.ChatAgent") +def test_validation_tool_call_execution_failed( + mock_chat_agent, mock_model_factory +): + """Test validation when tool call execution fails.""" + mock_model = MagicMock() + mock_model_factory.return_value = mock_model + mock_agent = MagicMock() + mock_response = MagicMock() + tool_call = MagicMock() + tool_call.result = "Wrong result" # Wrong result + mock_response.info = {"tool_calls": [tool_call]} + mock_agent.step.return_value = mock_response + mock_chat_agent.return_value = mock_agent + + result = validate_model_with_details( + model_platform="OPENAI", + model_type="GPT_4O_MINI", + api_key="test_key", + ) + + assert result.is_valid is False + assert result.is_tool_calls is False + assert result.error_type == ValidationErrorType.TOOL_CALL_EXECUTION_FAILED + assert result.failed_stage == ValidationStage.TOOL_CALL_EXECUTION + + +@pytest.mark.unit +@patch("app.component.model_validation.ModelFactory.create") +@patch("app.component.model_validation.ChatAgent") +def test_validation_success_with_tool_calls( + mock_chat_agent, mock_model_factory +): + """Test successful validation with tool calls.""" + mock_model = MagicMock() + mock_model_factory.return_value = mock_model + mock_agent = MagicMock() + mock_response = MagicMock() + tool_call = MagicMock() + tool_call.result = EXPECTED_TOOL_RESULT # Correct result + mock_response.info = {"tool_calls": [tool_call]} + mock_agent.step.return_value = mock_response + mock_chat_agent.return_value = mock_agent + + result = validate_model_with_details( + model_platform="OPENAI", + model_type="GPT_4O_MINI", + api_key="test_key", + ) + + assert result.is_valid is True + assert result.is_tool_calls is True + assert result.error_type is None + assert result.failed_stage is None + assert ( + result.validation_stages[ValidationStage.TOOL_CALL_EXECUTION] is True + ) + assert ValidationStage.TOOL_CALL_EXECUTION in result.successful_stages + + +@pytest.mark.unit +@patch("app.component.model_validation.ModelFactory.create") +@patch("app.component.model_validation.ChatAgent") +def test_validation_diagnostic_info(mock_chat_agent, mock_model_factory): + """Test that diagnostic info is properly populated.""" + mock_model = MagicMock() + mock_model_factory.return_value = mock_model + mock_agent = MagicMock() + mock_response = MagicMock() + tool_call = MagicMock() + tool_call.result = EXPECTED_TOOL_RESULT + mock_response.info = {"tool_calls": [tool_call]} + mock_agent.step.return_value = mock_response + mock_chat_agent.return_value = mock_agent + + result = validate_model_with_details( + model_platform="OPENAI", + model_type="GPT_4O_MINI", + api_key="test_key", + ) + + assert "initialization" in result.diagnostic_info + assert "model_creation" in result.diagnostic_info + assert "agent_creation" in result.diagnostic_info + assert result.model_response_info is not None + assert result.tool_call_info is not None + assert result.tool_call_info["execution_successful"] is True diff --git a/backend/tests/app/controller/test_model_controller.py b/backend/tests/app/controller/test_model_controller.py index 4a17e3d92..4e8ecf830 100644 --- a/backend/tests/app/controller/test_model_controller.py +++ b/backend/tests/app/controller/test_model_controller.py @@ -15,9 +15,14 @@ from unittest.mock import MagicMock, patch import pytest -from fastapi import HTTPException from fastapi.testclient import TestClient +from app.component.model_validation import ( + EXPECTED_TOOL_RESULT, + ValidationErrorType, + ValidationResult, + ValidationStage, +) from app.controller.model_controller import ( ValidateModelRequest, ValidateModelResponse, @@ -26,331 +31,421 @@ @pytest.mark.unit -class TestModelController: - """Test cases for model controller endpoints.""" +class TestModelControllerEnhanced: + """Test cases for enhanced model controller with detailed validation.""" @pytest.mark.asyncio - async def test_validate_model_success(self): - """Test successful model validation.""" + async def test_validate_model_with_diagnostics_success(self): + """Test successful model validation with diagnostics enabled.""" request_data = ValidateModelRequest( model_platform="openai", model_type="gpt-4o", api_key="test_key", - url="https://api.openai.com/v1", - model_config_dict={"temperature": 0.7}, - extra_params={"max_tokens": 1000}, + include_diagnostics=True, ) mock_agent = MagicMock() mock_response = MagicMock() tool_call = MagicMock() - tool_call.result = "Tool execution completed successfully for https://www.camel-ai.org, Website Content: Welcome to CAMEL AI!" + tool_call.result = EXPECTED_TOOL_RESULT mock_response.info = {"tool_calls": [tool_call]} mock_agent.step.return_value = mock_response with patch( - "app.controller.model_controller.create_agent", - return_value=mock_agent, - ): + "app.controller.model_controller.validate_model_with_details" + ) as mock_validate: + validation_result = ValidationResult() + validation_result.is_valid = True + validation_result.is_tool_calls = True + validation_result.error_type = None + validation_result.failed_stage = None + validation_result.successful_stages = [ + ValidationStage.INITIALIZATION, + ValidationStage.MODEL_CREATION, + ValidationStage.AGENT_CREATION, + ValidationStage.MODEL_CALL, + ValidationStage.TOOL_CALL_EXECUTION, + ] + validation_result.validation_stages = { + ValidationStage.INITIALIZATION: True, + ValidationStage.MODEL_CREATION: True, + ValidationStage.AGENT_CREATION: True, + ValidationStage.MODEL_CALL: True, + ValidationStage.TOOL_CALL_EXECUTION: True, + } + validation_result.diagnostic_info = { + "initialization": {"model_platform": "openai"}, + "model_creation": {"model_created": True}, + } + validation_result.model_response_info = { + "has_response": True, + "tool_calls_count": 1, + } + validation_result.tool_call_info = { + "tool_calls_count": 1, + "execution_successful": True, + } + mock_validate.return_value = validation_result + response = await validate_model(request_data) assert isinstance(response, ValidateModelResponse) assert response.is_valid is True assert response.is_tool_calls is True - assert response.message == "Validation Success" - assert response.error_code is None - assert response.error is None + assert response.error_type is None + assert response.failed_stage is None + assert response.successful_stages == [ + "initialization", + "model_creation", + "agent_creation", + "model_call", + "tool_call_execution", + ] + assert response.diagnostic_info is not None + assert response.model_response_info is not None + assert response.tool_call_info is not None + assert response.validation_stages is not None + assert response.validation_stages["tool_call_execution"] is True @pytest.mark.asyncio - async def test_validate_model_creation_failure(self): - """Test model validation when agent creation fails.""" + async def test_validate_model_with_diagnostics_failure(self): + """Test model validation failure with diagnostics enabled.""" request_data = ValidateModelRequest( - model_platform="INVALID", - model_type="INVALID_MODEL", + model_platform="openai", + model_type="gpt-4o", api_key="invalid_key", + include_diagnostics=True, ) with patch( - "app.controller.model_controller.create_agent", - side_effect=Exception("Invalid model configuration"), - ): - with pytest.raises(HTTPException) as exc_info: - await validate_model(request_data) - assert exc_info.value.status_code == 400 + "app.controller.model_controller.validate_model_with_details" + ) as mock_validate: + validation_result = ValidationResult() + validation_result.is_valid = False + validation_result.is_tool_calls = False + validation_result.error_type = ( + ValidationErrorType.AUTHENTICATION_ERROR + ) + validation_result.failed_stage = ValidationStage.MODEL_CREATION + validation_result.raw_error_message = "401 Unauthorized" + validation_result.error_message = "401 Unauthorized" + validation_result.successful_stages = [ + ValidationStage.INITIALIZATION + ] + validation_result.validation_stages = { + ValidationStage.INITIALIZATION: True, + ValidationStage.MODEL_CREATION: False, + } + validation_result.diagnostic_info = { + "initialization": {"model_platform": "openai"}, + } + mock_validate.return_value = validation_result + + response = await validate_model(request_data) + + assert isinstance(response, ValidateModelResponse) + assert response.is_valid is False + assert response.is_tool_calls is False + assert response.error_type == "authentication_error" + assert response.failed_stage == "model_creation" + assert response.successful_stages == ["initialization"] + assert response.diagnostic_info is not None + assert response.message == "401 Unauthorized" @pytest.mark.asyncio - async def test_validate_model_step_failure(self): - """Test model validation when agent step fails.""" + async def test_validate_model_without_diagnostics(self): + """Test model validation without diagnostics (default).""" request_data = ValidateModelRequest( - model_platform="openai", model_type="gpt-4o", api_key="test_key" + model_platform="openai", + model_type="gpt-4o", + api_key="test_key", + include_diagnostics=False, ) - mock_agent = MagicMock() - mock_agent.step.side_effect = Exception("API call failed") - with patch( - "app.controller.model_controller.create_agent", - return_value=mock_agent, - ): - with pytest.raises(HTTPException) as exc_info: - await validate_model(request_data) - assert exc_info.value.status_code == 400 + "app.controller.model_controller.validate_model_with_details" + ) as mock_validate: + validation_result = ValidationResult() + validation_result.is_valid = True + validation_result.is_tool_calls = True + mock_validate.return_value = validation_result + + response = await validate_model(request_data) + + assert isinstance(response, ValidateModelResponse) + assert response.is_valid is True + assert response.is_tool_calls is True + # Diagnostic fields should not be included when include_diagnostics=False + assert response.error_type is None + assert response.failed_stage is None + assert response.successful_stages is None + assert response.diagnostic_info is None @pytest.mark.asyncio - async def test_validate_model_tool_calls_false(self): - """Test model validation when tool calls fail.""" + async def test_validate_model_tool_call_not_supported(self): + """Test model validation when tool calls are not supported.""" request_data = ValidateModelRequest( - model_platform="openai", model_type="gpt-4o", api_key="test_key" + model_platform="openai", + model_type="gpt-4o", + api_key="test_key", + include_diagnostics=True, ) - mock_agent = MagicMock() - mock_response = MagicMock() - tool_call = MagicMock() - tool_call.result = "Different response" - mock_response.info = {"tool_calls": [tool_call]} - mock_agent.step.return_value = mock_response - with patch( - "app.controller.model_controller.create_agent", - return_value=mock_agent, - ): + "app.controller.model_controller.validate_model_with_details" + ) as mock_validate: + validation_result = ValidationResult() + validation_result.is_valid = True + validation_result.is_tool_calls = False + validation_result.error_type = ( + ValidationErrorType.TOOL_CALL_NOT_SUPPORTED + ) + validation_result.failed_stage = ( + ValidationStage.TOOL_CALL_EXECUTION + ) + validation_result.successful_stages = [ + ValidationStage.INITIALIZATION, + ValidationStage.MODEL_CREATION, + ValidationStage.AGENT_CREATION, + ValidationStage.MODEL_CALL, + ] + mock_validate.return_value = validation_result + response = await validate_model(request_data) assert isinstance(response, ValidateModelResponse) assert response.is_valid is True assert response.is_tool_calls is False + assert response.error_type == "tool_call_not_supported" assert ( - response.message - == "This model doesn't support tool calls. please try with another model." + "does not support tool calling" in response.message + or "tool call" in response.message.lower() ) @pytest.mark.asyncio - async def test_validate_model_with_minimal_parameters(self): - """Test model validation with minimal parameters (no API key).""" - request_data = ( - ValidateModelRequest() - ) # Uses default values, api_key is None - - mock_agent = MagicMock() - mock_response = MagicMock() - tool_call = MagicMock() - tool_call.result = "Tool execution completed successfully for https://www.camel-ai.org, Website Content: Welcome to CAMEL AI!" - mock_response.info = {"tool_calls": [tool_call]} - mock_agent.step.return_value = mock_response + async def test_validate_model_tool_call_execution_failed(self): + """Test model validation when tool call execution fails.""" + request_data = ValidateModelRequest( + model_platform="openai", + model_type="gpt-4o", + api_key="test_key", + include_diagnostics=True, + ) with patch( - "app.controller.model_controller.create_agent", - return_value=mock_agent, - ): - # api_key is None by default, which passes the empty string check - # The agent step succeeds, so validation should pass + "app.controller.model_controller.validate_model_with_details" + ) as mock_validate: + validation_result = ValidationResult() + validation_result.is_valid = False + validation_result.is_tool_calls = False + validation_result.error_type = ( + ValidationErrorType.TOOL_CALL_EXECUTION_FAILED + ) + validation_result.failed_stage = ( + ValidationStage.TOOL_CALL_EXECUTION + ) + validation_result.raw_error_message = "Tool execution failed" + validation_result.error_message = "Tool execution failed" + mock_validate.return_value = validation_result + response = await validate_model(request_data) + assert isinstance(response, ValidateModelResponse) - assert response.is_valid is True - assert response.is_tool_calls is True + assert response.is_valid is False + assert response.is_tool_calls is False + assert response.error_type == "tool_call_execution_failed" + assert response.message == "Tool execution failed" @pytest.mark.asyncio - async def test_validate_model_no_response(self): - """Test model validation when no response is returned.""" + async def test_validate_model_network_error(self): + """Test model validation with network error.""" request_data = ValidateModelRequest( - model_platform="openai", model_type="gpt-4o" + model_platform="openai", + model_type="gpt-4o", + api_key="test_key", + include_diagnostics=True, ) - mock_agent = MagicMock() - mock_agent.step.return_value = None - - # When response is None, should return False - with patch( - "app.controller.model_controller.create_agent", - return_value=mock_agent, - ): - result = await validate_model(request_data) - assert result.is_valid is False - assert result.is_tool_calls is False - assert result.error_code is None - assert result.error is None - - -@pytest.mark.integration -class TestModelControllerIntegration: - """Integration tests for model controller.""" - - def test_validate_model_endpoint_integration(self, client: TestClient): - """Test validate model endpoint through FastAPI test client.""" - request_data = { - "model_platform": "openai", - "model_type": "gpt-4o", - "api_key": "test_key", - "url": "https://api.openai.com/v1", - "model_config_dict": {"temperature": 0.7}, - "extra_params": {"max_tokens": 1000}, - } - - mock_agent = MagicMock() - mock_response = MagicMock() - tool_call = MagicMock() - tool_call.result = "Tool execution completed successfully for https://www.camel-ai.org, Website Content: Welcome to CAMEL AI!" - mock_response.info = {"tool_calls": [tool_call]} - mock_agent.step.return_value = mock_response - with patch( - "app.controller.model_controller.create_agent", - return_value=mock_agent, - ): - response = client.post("/model/validate", json=request_data) - - assert response.status_code == 200 - response_data = response.json() - assert response_data["is_valid"] is True - assert response_data["is_tool_calls"] is True - assert response_data["message"] == "Validation Success" - - def test_validate_model_endpoint_error_integration( - self, client: TestClient - ): - """Test validate model endpoint error handling through FastAPI test client.""" - request_data = { - "model_platform": "INVALID", - "model_type": "INVALID_MODEL", - } - - with patch( - "app.controller.model_controller.create_agent", - side_effect=Exception("Test error"), - ): - response = client.post("/model/validate", json=request_data) - - assert response.status_code == 400 + "app.controller.model_controller.validate_model_with_details" + ) as mock_validate: + validation_result = ValidationResult() + validation_result.is_valid = False + validation_result.is_tool_calls = False + validation_result.error_type = ValidationErrorType.NETWORK_ERROR + validation_result.failed_stage = ValidationStage.MODEL_CREATION + validation_result.raw_error_message = "Connection timeout" + validation_result.error_message = "Connection timeout" + mock_validate.return_value = validation_result + response = await validate_model(request_data) -@pytest.mark.model_backend -class TestModelControllerWithRealModels: - """Tests that require real model backends (marked for selective running).""" + assert isinstance(response, ValidateModelResponse) + assert response.is_valid is False + assert response.error_type == "network_error" + assert response.message == "Connection timeout" @pytest.mark.asyncio - async def test_validate_model_with_real_openai_model(self): - """Test model validation with real OpenAI model (requires API key).""" - # This test would validate against real OpenAI API - # Marked as model_backend for selective execution - assert True # Placeholder + async def test_validate_model_rate_limit_error(self): + """Test model validation with rate limit error.""" + request_data = ValidateModelRequest( + model_platform="openai", + model_type="gpt-4o", + api_key="test_key", + include_diagnostics=True, + ) - @pytest.mark.very_slow - async def test_validate_multiple_model_platforms(self): - """Test validation across multiple model platforms (very slow test).""" - # This test would validate multiple different model platforms - # Marked as very_slow for execution only in full test mode - assert True # Placeholder + with patch( + "app.controller.model_controller.validate_model_with_details" + ) as mock_validate: + validation_result = ValidationResult() + validation_result.is_valid = False + validation_result.is_tool_calls = False + validation_result.error_type = ValidationErrorType.RATE_LIMIT_ERROR + validation_result.failed_stage = ValidationStage.MODEL_CALL + validation_result.raw_error_message = "429 Rate limit exceeded" + validation_result.error_message = "429 Rate limit exceeded" + mock_validate.return_value = validation_result + response = await validate_model(request_data) -@pytest.mark.unit -class TestModelControllerErrorCases: - """Test error cases and edge conditions for model controller.""" + assert isinstance(response, ValidateModelResponse) + assert response.is_valid is False + assert response.error_type == "rate_limit_error" + assert response.message == "429 Rate limit exceeded" @pytest.mark.asyncio - async def test_validate_model_with_invalid_json_config(self): - """Test model validation with invalid JSON configuration.""" + async def test_validate_model_empty_api_key(self): + """Test model validation with empty API key.""" request_data = ValidateModelRequest( model_platform="openai", model_type="gpt-4o", - model_config_dict={"invalid": float("inf")}, # Invalid JSON value + api_key="", # Empty API key ) - with patch( - "app.controller.model_controller.create_agent", - side_effect=ValueError("Invalid configuration"), - ): - with pytest.raises(HTTPException) as exc_info: - await validate_model(request_data) - assert exc_info.value.status_code == 400 + # Should raise HTTPException before calling validate_model_with_details + with pytest.raises(Exception): # HTTPException is raised + await validate_model(request_data) @pytest.mark.asyncio - async def test_validate_model_with_network_error(self): - """Test model validation with network connectivity issues.""" + async def test_validate_model_error_response_structure(self): + """Test that error response structure is correct.""" request_data = ValidateModelRequest( model_platform="openai", model_type="gpt-4o", - url="https://invalid-url.com", + api_key="test_key", + include_diagnostics=True, ) - mock_agent = MagicMock() - mock_agent.step.side_effect = ConnectionError("Network unreachable") - with patch( - "app.controller.model_controller.create_agent", - return_value=mock_agent, - ): - with pytest.raises(HTTPException) as exc_info: - await validate_model(request_data) - assert exc_info.value.status_code == 400 + "app.controller.model_controller.validate_model_with_details" + ) as mock_validate: + validation_result = ValidationResult() + validation_result.is_valid = False + validation_result.is_tool_calls = False + validation_result.error_type = ( + ValidationErrorType.AUTHENTICATION_ERROR + ) + validation_result.error_message = "Invalid API key" + validation_result.error_details = {"code": "invalid_key"} + mock_validate.return_value = validation_result - @pytest.mark.asyncio - async def test_validate_model_with_malformed_tool_calls_response(self): - """Test model validation with malformed tool calls in response.""" - request_data = ValidateModelRequest( - model_platform="openai", model_type="gpt-4o" - ) + response = await validate_model(request_data) - mock_agent = MagicMock() - mock_response = MagicMock() - mock_response.info = { - "tool_calls": [] # Empty tool calls - } - mock_agent.step.return_value = mock_response + assert response.error_code == "authentication_error" + assert response.error is not None + assert response.error["type"] == "invalid_request_error" + assert response.error["code"] == "authentication_error" + assert response.error["message"] == "Invalid API key" + assert "details" in response.error + assert response.error["details"]["code"] == "invalid_key" - with patch( - "app.controller.model_controller.create_agent", - return_value=mock_agent, - ): - # Should handle empty tool calls gracefully - result = await validate_model(request_data) - assert result.is_valid is True # Response exists - assert result.is_tool_calls is False # No valid tool calls - @pytest.mark.asyncio - async def test_validate_model_with_missing_info_field(self): - """Test model validation with missing info field in response.""" - request_data = ValidateModelRequest( - model_platform="openai", model_type="gpt-4o" - ) +@pytest.mark.integration +class TestModelControllerIntegrationEnhanced: + """Integration tests for enhanced model controller.""" - mock_agent = MagicMock() - mock_response = MagicMock() - mock_response.info = {} # Missing tool_calls - mock_agent.step.return_value = mock_response + def test_validate_model_endpoint_with_diagnostics( + self, client: TestClient + ): + """Test validate model endpoint with diagnostics through FastAPI test client.""" + request_data = { + "model_platform": "openai", + "model_type": "gpt-4o", + "api_key": "test_key", + "include_diagnostics": True, + } with patch( - "app.controller.model_controller.create_agent", - return_value=mock_agent, - ): - # Should handle missing tool_calls key gracefully - result = await validate_model(request_data) - assert result.is_valid is True # Response exists - assert result.is_tool_calls is False # No tool_calls key + "app.controller.model_controller.validate_model_with_details" + ) as mock_validate: + validation_result = ValidationResult() + validation_result.is_valid = True + validation_result.is_tool_calls = True + validation_result.successful_stages = [ + ValidationStage.INITIALIZATION, + ValidationStage.MODEL_CREATION, + ValidationStage.AGENT_CREATION, + ValidationStage.MODEL_CALL, + ValidationStage.TOOL_CALL_EXECUTION, + ] + validation_result.validation_stages = { + ValidationStage.INITIALIZATION: True, + ValidationStage.MODEL_CREATION: True, + ValidationStage.AGENT_CREATION: True, + ValidationStage.MODEL_CALL: True, + ValidationStage.TOOL_CALL_EXECUTION: True, + } + validation_result.diagnostic_info = {"test": "info"} + validation_result.model_response_info = {"has_response": True} + validation_result.tool_call_info = {"tool_calls_count": 1} + mock_validate.return_value = validation_result - @pytest.mark.asyncio - async def test_validate_model_empty_api_key(self): - """Test model validation with empty API key.""" - request_data = ValidateModelRequest( - model_platform="openai", - model_type="gpt-4o", - api_key="", # Empty API key - ) + response = client.post("/model/validate", json=request_data) - with pytest.raises(HTTPException) as exc_info: - await validate_model(request_data) - assert exc_info.value.status_code == 400 - detail = exc_info.value.detail - assert detail["error_code"] == "invalid_api_key" + assert response.status_code == 200 + response_data = response.json() + assert response_data["is_valid"] is True + assert response_data["is_tool_calls"] is True + assert "error_type" in response_data + assert "failed_stage" in response_data + assert "successful_stages" in response_data + assert "diagnostic_info" in response_data + assert "model_response_info" in response_data + assert "tool_call_info" in response_data + assert "validation_stages" in response_data + + def test_validate_model_endpoint_without_diagnostics( + self, client: TestClient + ): + """Test validate model endpoint without diagnostics through FastAPI test client.""" + request_data = { + "model_platform": "openai", + "model_type": "gpt-4o", + "api_key": "test_key", + "include_diagnostics": False, + } - @pytest.mark.asyncio - async def test_validate_model_invalid_model_type(self): - """Test model validation with invalid model type raises HTTPException.""" - request_data = ValidateModelRequest( - model_platform="openai", - model_type="INVALID_MODEL_TYPE", - api_key="test_key", - ) + with patch( + "app.controller.model_controller.validate_model_with_details" + ) as mock_validate: + validation_result = ValidationResult() + validation_result.is_valid = True + validation_result.is_tool_calls = True + mock_validate.return_value = validation_result - with pytest.raises(HTTPException) as exc_info: - await validate_model(request_data) - assert exc_info.value.status_code == 400 + response = client.post("/model/validate", json=request_data) + + assert response.status_code == 200 + response_data = response.json() + assert response_data["is_valid"] is True + assert response_data["is_tool_calls"] is True + # Diagnostic fields should not be present when include_diagnostics=False + assert ( + "error_type" not in response_data + or response_data.get("error_type") is None + ) + assert ( + "failed_stage" not in response_data + or response_data.get("failed_stage") is None + )